diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2e4fa7c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Source code - enforce LF line endings +*.py text eol=lf +*.ts text eol=lf +*.tsx text eol=lf +*.js text eol=lf +*.jsx text eol=lf +*.json text eol=lf +*.css text eol=lf +*.scss text eol=lf +*.html text eol=lf +*.md text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.toml text eol=lf +*.sql text eol=lf +*.sh text eol=lf +*.bat text eol=lf +*.conf text eol=lf +*.cfg text eol=lf +*.txt text eol=lf +*.env text eol=lf + +# Binary files - don't touch +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.pdf binary +*.zip binary +*.woff binary +*.woff2 binary +*.ttf binary +*.eot binary diff --git a/.gitignore b/.gitignore index 6c983ad..2947302 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,7 @@ Desktop.ini # 其他 *.tmp *.temp +*.tar.gz .cache/ *.coverage .coverage diff --git a/backend/main.py b/backend/main.py index c5dec9e..52f8a1e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -4,14 +4,17 @@ FastAPI应用入口 """ import os import sys +import logging from pathlib import Path from datetime import datetime -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, JSONResponse import uvicorn +logger = logging.getLogger(__name__) + # 设置系统时区为北京时间 os.environ['TZ'] = 'Asia/Shanghai' @@ -52,9 +55,9 @@ def startup_knowledge_base(): try: from src.services.file_watcher_service import start_file_watcher start_file_watcher() - print("知识库文件监控服务启动成功") + logger.info("知识库文件监控服务启动成功") except Exception as e: - print(f"启动知识库文件监控服务失败: {str(e)}") + logger.error(f"启动知识库文件监控服务失败: {str(e)}") def seed_forum_categories(): @@ -72,17 +75,17 @@ def seed_forum_categories(): try: existing_count = db.query(ForumCategory).count() if existing_count > 0: - print(f"论坛分类已存在（{existing_count} 个），跳过初始化") + logger.info(f"论坛分类已存在（{existing_count} 个），跳过初始化") return for cat_data in DEFAULT_CATEGORIES: category = ForumCategory(**cat_data) db.add(category) db.commit() - print(f"已创建 {len(DEFAULT_CATEGORIES)} 个论坛分类") + logger.info(f"已创建 {len(DEFAULT_CATEGORIES)} 个论坛分类") except Exception as e: db.rollback() - print(f"初始化论坛分类失败: {e}") + logger.error(f"初始化论坛分类失败: {e}") finally: db.close() @@ -94,10 +97,9 @@ async def startup_event(): import asyncio import sys - print("=" * 50, file=sys.stderr) - print("应用启动事件开始", file=sys.stderr) - print("=" * 50, file=sys.stderr) - sys.stderr.flush() + logger.info("=" * 50) + logger.info("应用启动事件开始") + logger.info("=" * 50) # 等待数据库可用（重试机制，处理 DNS 解析延迟） max_retries = 10 @@ -105,28 +107,23 @@ async def startup_event(): for attempt in range(max_retries): try: - print(f"尝试连接数据库（尝试 {attempt + 1}/{max_retries}）...", file=sys.stderr) - sys.stderr.flush() + logger.info(f"尝试连接数据库（尝试 {attempt + 1}/{max_retries}）...") # 尝试创建数据库表 create_tables() - print("数据库表创建成功", file=sys.stderr) - sys.stderr.flush() + logger.info("数据库表创建成功") break except Exception as e: if attempt < max_retries - 1: - print(f"数据库连接失败（尝试 {attempt + 1}/{max_retries}）: {e}", file=sys.stderr) - print(f"等待 {retry_delay} 秒后重试...", file=sys.stderr) - sys.stderr.flush() + logger.warning(f"数据库连接失败（尝试 {attempt + 1}/{max_retries}）: {e}") await asyncio.sleep(retry_delay) else: - print(f"数据库连接失败，已达到最大重试次数: {e}", file=sys.stderr) - sys.stderr.flush() + logger.error(f"数据库连接失败，已达到最大重试次数: {e}") # 不抛出异常，让应用继续启动，但数据库操作会失败 try: startup_knowledge_base() except Exception as e: - print(f"启动知识库服务失败: {e}", file=sys.stderr) + logger.error(f"启动知识库服务失败: {e}") # 确保系统知识库与目录同步 try: @@ -137,17 +134,15 @@ async def startup_event(): kb_service.ensure_system_knowledge_bases() db.close() except Exception as e: - print(f"同步系统知识库失败: {e}", file=sys.stderr) + logger.error(f"同步系统知识库失败: {e}") # 初始化论坛分类 try: seed_forum_categories() except Exception as e: - print(f"初始化论坛分类失败: {e}", file=sys.stderr) - sys.stderr.flush() - - print("应用启动事件完成", file=sys.stderr) - sys.stderr.flush() + logger.error(f"初始化论坛分类失败: {e}") + + logger.info("应用启动事件完成") # 应用关闭事件 @app.on_event("shutdown") @@ -156,9 +151,18 @@ async def shutdown_event(): try: from src.services.file_watcher_service import stop_file_watcher stop_file_watcher() - print("知识库文件监控服务已停止") + logger.info("知识库文件监控服务已停止") except Exception as e: - print(f"停止知识库文件监控服务失败: {str(e)}") + logger.error(f"停止知识库文件监控服务失败: {str(e)}") + +# 全局异常处理器 +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + logger.error(f"未处理的异常: {exc}", exc_info=True) + return JSONResponse( + status_code=500, + content={"detail": "服务器内部错误，请稍后重试"} + ) # 注册路由 app.include_router(auth.router) @@ -315,11 +319,8 @@ async def list_services(): if __name__ == "__main__": import sys - print("=" * 50, file=sys.stderr) - print("启动 Uvicorn 服务器...", file=sys.stderr) - print(f"Host: {settings.host}, Port: {settings.port}", file=sys.stderr) - print("=" * 50, file=sys.stderr) - sys.stderr.flush() + logger.info("启动 Uvicorn 服务器...") + logger.info(f"Host: {settings.host}, Port: {settings.port}") uvicorn.run( "main:app", diff --git a/backend/src/api/document.py b/backend/src/api/document.py index 8070a57..2189654 100644 --- a/backend/src/api/document.py +++ b/backend/src/api/document.py @@ -1,6 +1,7 @@ """ 文档管理API """ +import logging import asyncio import os import uuid @@ -15,6 +16,8 @@ from ..core.security import get_current_user from ..models.document import Document, DocumentChunk from ..services.document_service import DocumentService +logger = logging.getLogger(__name__) + router = APIRouter(prefix="/documents", tags=["文档管理"]) @@ -186,32 +189,30 @@ async def delete_document( # 1. 先删除向量数据和文档块 try: - print(f"开始删除文档向量数据: {document.filename} (ID: {document.id})") + logger.info(f"开始删除文档向量数据: {document.filename} (ID: {document.id})") document_service = DocumentService(db) vector_deleted = document_service.delete_document_chunks(document.id) if vector_deleted: - print(f"成功删除文档向量数据: {document.filename}") + logger.info(f"成功删除文档向量数据: {document.filename}") else: - print(f"警告：删除文档向量数据失败: {document.filename}") + logger.warning(f"警告：删除文档向量数据失败: {document.filename}") except Exception as e: - print(f"删除向量数据时发生错误: {str(e)}") - import traceback - print(f"详细错误信息: {traceback.format_exc()}") + logger.error(f"删除向量数据时发生错误: {str(e)}", exc_info=True) # 2. 删除物理文件 try: if os.path.exists(document.file_path): os.remove(document.file_path) - print(f"成功删除物理文件: {document.file_path}") + logger.info(f"成功删除物理文件: {document.file_path}") else: - print(f"物理文件不存在: {document.file_path}") + logger.info(f"物理文件不存在: {document.file_path}") except Exception as e: - print(f"删除物理文件时发生错误: {str(e)}") + logger.error(f"删除物理文件时发生错误: {str(e)}") # 3. 删除数据库记录 db.delete(document) db.commit() - print(f"成功删除文档数据库记录: {document.filename}") + logger.info(f"成功删除文档数据库记录: {document.filename}") return {"message": "文档删除成功"} diff --git a/backend/src/api/knowledge_base.py b/backend/src/api/knowledge_base.py index ecb1895..2a95048 100644 --- a/backend/src/api/knowledge_base.py +++ b/backend/src/api/knowledge_base.py @@ -1,6 +1,7 @@ """ 知识库CRUD API """ +import logging import os import uuid from pathlib import Path @@ -18,6 +19,8 @@ from ..models.document import Document from ..models.user import User from ..services.document_service import DocumentService +logger = logging.getLogger(__name__) + router = APIRouter(prefix="/knowledge-bases", tags=["知识库管理"]) @@ -64,7 +67,7 @@ async def get_knowledge_bases( db: Session = Depends(get_db) ): """获取用户的所有知识库""" - print(f"DEBUG: get_knowledge_bases called for user: {current_user}") + logger.debug(f"get_knowledge_bases called for user: {current_user}") try: # 获取用户ID user = db.query(User).filter(User.username == current_user).first() @@ -118,7 +121,7 @@ async def create_knowledge_base( db: Session = Depends(get_db) ): """创建新知识库""" - print(f"DEBUG: create_knowledge_base called for user: {current_user}, data: {data}") + logger.debug(f"create_knowledge_base called for user: {current_user}, data: {data}") try: # 获取用户ID user = db.query(User).filter(User.username == current_user).first() @@ -368,7 +371,7 @@ async def delete_knowledge_base( Document.knowledge_base_id == knowledge_base_id ).all() - print(f"开始删除知识库 '{knowledge_base.name}'，包含 {len(documents)} 个文档") + logger.info(f"开始删除知识库 '{knowledge_base.name}'，包含 {len(documents)} 个文档") # 2. 逐个删除文档的向量数据和物理文件 document_service = DocumentService(db) @@ -380,33 +383,31 @@ async def delete_knowledge_base( # 删除向量数据 vector_deleted = document_service.delete_document_chunks(document.id) if vector_deleted: - print(f"成功删除文档向量数据: {document.filename}") + logger.info(f"成功删除文档向量数据: {document.filename}") else: - print(f"警告：删除文档向量数据失败: {document.filename}") + logger.warning(f"警告：删除文档向量数据失败: {document.filename}") error_count += 1 # 删除物理文件 if os.path.exists(document.file_path): os.remove(document.file_path) - print(f"成功删除物理文件: {document.file_path}") + logger.info(f"成功删除物理文件: {document.file_path}") else: - print(f"物理文件不存在: {document.file_path}") + logger.info(f"物理文件不存在: {document.file_path}") success_count += 1 except Exception as e: - print(f"删除文档 {document.filename} 的资源时出错: {str(e)}") - import traceback - print(f"详细错误信息: {traceback.format_exc()}") + logger.error(f"删除文档 {document.filename} 的资源时出错: {str(e)}", exc_info=True) error_count += 1 # 继续处理其他文档 - print(f"文档清理完成: 成功 {success_count} 个，失败 {error_count} 个") + logger.info(f"文档清理完成: 成功 {success_count} 个，失败 {error_count} 个") # 3. 删除知识库（级联删除文档记录） db.delete(knowledge_base) db.commit() - print(f"成功删除知识库数据库记录: {knowledge_base.name}") + logger.info(f"成功删除知识库数据库记录: {knowledge_base.name}") return {"message": "知识库删除成功"} @@ -528,20 +529,18 @@ async def upload_document_to_knowledge_base( # 自动处理文档向量化 try: - print(f"开始处理文档向量化: {document.filename} (ID: {document.id})") + logger.info(f"开始处理文档向量化: {document.filename} (ID: {document.id})") document_service = DocumentService(db) success = await document_service.process_document(document.id) - + if success: - print(f"文档向量化处理成功: {document.filename}") + logger.info(f"文档向量化处理成功: {document.filename}") message = "文档上传并处理成功" else: - print(f"文档向量化处理失败: {document.filename}") + logger.warning(f"文档向量化处理失败: {document.filename}") message = "文档上传成功，但向量化处理失败" except Exception as e: - print(f"文档向量化处理异常: {document.filename}, 错误: {str(e)}") - import traceback - print(f"详细错误信息: {traceback.format_exc()}") + logger.error(f"文档向量化处理异常: {document.filename}, 错误: {str(e)}", exc_info=True) message = "文档上传成功，但向量化处理失败" return DocumentUploadResponse( diff --git a/backend/src/core/security.py b/backend/src/core/security.py index a13eed8..e5c83d6 100644 --- a/backend/src/core/security.py +++ b/backend/src/core/security.py @@ -8,6 +8,7 @@ from passlib.context import CryptContext from fastapi import HTTPException, status, Depends from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from pydantic import BaseModel +from sqlalchemy.orm import Session from .config import get_settings @@ -82,3 +83,24 @@ async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(s return username except JWTError: raise credentials_exception + + +async def get_current_user_obj(current_user: str = Depends(get_current_user)): + """获取当前用户的完整User对象 + + 用法: current_user: User = Depends(get_current_user_obj) + 替代: current_user: str = Depends(get_current_user) + 手动 db.query(User) + """ + from ..models.user import User + from .database import get_db + db = next(get_db()) + try: + user = db.query(User).filter(User.username == current_user).first() + if not user: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="用户不存在" + ) + return user + finally: + db.close() diff --git a/backend/src/models/chat.py b/backend/src/models/chat.py index 208e93b..aaae9dd 100644 --- a/backend/src/models/chat.py +++ b/backend/src/models/chat.py @@ -13,7 +13,7 @@ class ChatSession(Base): __tablename__ = "chat_sessions" id = Column(Integer, primary_key=True, index=True) - user_id = Column(Integer, ForeignKey("users.id"), nullable=False) + user_id = Column(Integer, ForeignKey("users.id"), nullable=False, index=True) title = Column(String(200), nullable=True) is_active = Column(Boolean, default=True) created_at = Column(DateTime(timezone=True), server_default=func.now()) @@ -32,7 +32,7 @@ class ChatMessage(Base): __tablename__ = "chat_messages" id = Column(Integer, primary_key=True, index=True) - session_id = Column(Integer, ForeignKey("chat_sessions.id"), nullable=False) + session_id = Column(Integer, ForeignKey("chat_sessions.id"), nullable=False, index=True) role = Column(String(20), nullable=False) # user, assistant, system content = Column(Text, nullable=False) message_metadata = Column(Text, nullable=True) # JSON格式的元数据 diff --git a/backend/src/models/document.py b/backend/src/models/document.py index a80dceb..234c2bc 100644 --- a/backend/src/models/document.py +++ b/backend/src/models/document.py @@ -14,7 +14,7 @@ class Document(Base): id = Column(Integer, primary_key=True, index=True) user_id = Column(Integer, ForeignKey("users.id"), nullable=True) # None表示系统文档 - knowledge_base_id = Column(Integer, ForeignKey("knowledge_bases.id"), nullable=True) # 所属知识库 + knowledge_base_id = Column(Integer, ForeignKey("knowledge_bases.id"), nullable=True, index=True) filename = Column(String(255), nullable=False) original_filename = Column(String(255), nullable=False) file_path = Column(String(500), nullable=False) @@ -47,7 +47,7 @@ class DocumentChunk(Base): __tablename__ = "document_chunks" id = Column(Integer, primary_key=True, index=True) - document_id = Column(Integer, ForeignKey("documents.id"), nullable=False) + document_id = Column(Integer, ForeignKey("documents.id"), nullable=False, index=True) chunk_index = Column(Integer, nullable=False) content = Column(Text, nullable=False) content_hash = Column(String(64), nullable=False) # 内容哈希 diff --git a/backend/src/rag/score_utils.py b/backend/src/rag/score_utils.py new file mode 100644 index 0000000..f17c426 --- /dev/null +++ b/backend/src/rag/score_utils.py @@ -0,0 +1,13 @@ +""" +分数转换工具 +""" +import math + + +def convert_distance_to_score(distance: float) -> float: + """将ChromaDB的distance转换为0-1范围的相似度分数""" + if distance < 0: + return (1 + distance) / 2 + if distance > 100: + return 1 / (1 + math.log(distance)) + return 1 / (1 + distance) diff --git a/backend/src/services/document_service.py b/backend/src/services/document_service.py index 3811fd4..2943d13 100644 --- a/backend/src/services/document_service.py +++ b/backend/src/services/document_service.py @@ -3,15 +3,19 @@ """ import os import asyncio +import logging from pathlib import Path from typing import List, Dict, Any, Optional from sqlalchemy.orm import Session from langchain_core.documents import Document as LangChainDocument from ..models.document import Document, DocumentChunk + +logger = logging.getLogger(__name__) from ..rag.vector_store import get_vector_store from ..rag.document_loaders import DocumentLoaderFactory, PDFImageExtractor from ..rag.text_splitters import get_text_splitter +from ..rag.score_utils import convert_distance_to_score from ..llm.siliconflow import get_llm_client IMAGES_DIR = Path(__file__).parent.parent.parent.parent / "data" / "images" @@ -65,14 +69,14 @@ class DocumentService: if success: document.is_processed = True self.db.commit() - print(f"[DocumentService] 文档 {document.filename} 处理完成: " + logger.info(f"[DocumentService] 文档 {document.filename} 处理完成: " f"{len(splits)} 个文本块, {len(image_chunks)} 个图片描述块") return True return False except Exception as e: - print(f"处理文档失败: {str(e)}") + logger.error(f"处理文档失败: {str(e)}") self.db.rollback() return False @@ -94,56 +98,58 @@ class DocumentService: images = PDFImageExtractor.extract_images(str(file_path), str(img_output_dir)) if not images: - print(f"[DocumentService] 未发现可提取的图片: {filename}") + logger.info(f"[DocumentService] 未发现可提取的图片: {filename}") return [] - print(f"[DocumentService] 提取到 {len(images)} 张图片, 开始VLM描述生成...") + logger.info(f"[DocumentService] 提取到 {len(images)} 张图片, 开始VLM描述生成...") - # 批量调用VLM生成描述 + # 并发调用VLM生成描述（限制并发度为5） llm_client = get_llm_client() - for idx, img in enumerate(images): - try: - description = await llm_client.describe_image( - img["path"], - img.get("context_text", "") - ) + semaphore = asyncio.Semaphore(5) - if description: - # 相对路径用于URL访问 - rel_path = f"{document_id}/{img['filename']}" - image_url = f"/images/{rel_path}" - - # 构建图片描述文本块（URL写入内容，LLM可直接引用） - chunk_content = ( - f"[图片描述 - 第{img['page']}页]\n" - f"图片URL: {image_url}\n" - f"图片内容：{description}" + async def describe_single_image(idx, img): + async with semaphore: + try: + description = await llm_client.describe_image( + img["path"], + img.get("context_text", "") ) + if description: + rel_path = f"{document_id}/{img['filename']}" + image_url = f"/images/{rel_path}" + chunk_content = ( + f"[图片描述 - 第{img['page']}页]\n" + f"图片URL: {image_url}\n" + f"图片内容：{description}" + ) + chunk = LangChainDocument( + page_content=chunk_content, + metadata={ + "document_id": document_id, + "knowledge_base_id": knowledge_base_id, + "title": title, + "filename": filename, + "source_type": "image", + "image_path": str(img["path"]), + "image_url": image_url, + "page": img["page"], + } + ) + logger.info(f"[DocumentService] 图片描述成功 {idx+1}/{len(images)}: {img['filename']}") + return chunk + else: + logger.warning(f"[DocumentService] 图片描述为空 {idx+1}/{len(images)}: {img['filename']}") + return None + except Exception as e: + logger.error(f"[DocumentService] 图片处理失败 {img['filename']}: {e}") + return None - chunk = LangChainDocument( - page_content=chunk_content, - metadata={ - "document_id": document_id, - "knowledge_base_id": knowledge_base_id, - "title": title, - "filename": filename, - "source_type": "image", - "image_path": str(img["path"]), - "image_url": image_url, - "page": img["page"], - } - ) - image_chunks.append(chunk) - print(f"[DocumentService] 图片描述成功 {idx+1}/{len(images)}: {img['filename']}") - else: - print(f"[DocumentService] 图片描述为空 {idx+1}/{len(images)}: {img['filename']}") - - except Exception as e: - print(f"[DocumentService] 图片处理失败 {img['filename']}: {e}") - continue + tasks = [describe_single_image(idx, img) for idx, img in enumerate(images)] + results = await asyncio.gather(*tasks) + image_chunks = [r for r in results if r is not None] except Exception as e: - print(f"[DocumentService] PDF图片处理失败: {e}") + logger.error(f"[DocumentService] PDF图片处理失败: {e}") return image_chunks @@ -166,7 +172,7 @@ class DocumentService: search_results = [] for doc, distance in results: metadata = doc.metadata if hasattr(doc, 'metadata') else {} - score = self._convert_distance_to_score(distance) + score = convert_distance_to_score(distance) search_results.append({ "content": doc.page_content, @@ -178,24 +184,9 @@ class DocumentService: return search_results except Exception as e: - print(f"搜索文档失败: {str(e)}") + logger.error(f"搜索文档失败: {str(e)}") return [] - def _convert_distance_to_score(self, distance: float) -> float: - """将ChromaDB的distance转换为0-1范围的相似度分数""" - import math - - # 内积距离（负值） - if distance < 0: - return (1 + distance) / 2 - - # 大距离使用对数缩放 - if distance > 100: - return 1 / (1 + math.log(distance)) - - # 标准距离转换 - return 1 / (1 + distance) - def get_document_chunks(self, document_id: int) -> List[DocumentChunk]: """获取文档的所有块""" return self.db.query(DocumentChunk).filter( @@ -214,6 +205,6 @@ class DocumentService: self.db.commit() return True except Exception as e: - print(f"删除文档块失败: {str(e)}") + logger.error(f"删除文档块失败: {str(e)}") self.db.rollback() return False \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 91e8465..7abc566 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: POSTGRES_DB: course_agent_db POSTGRES_USER: user - POSTGRES_PASSWORD: password + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C" volumes: - postgres_data:/var/lib/postgresql/data @@ -35,7 +35,7 @@ services: - "8001:8001" # 前端应用 environment: # 数据库配置 - DATABASE_URL: postgresql+psycopg://user:password@db:5432/course_agent_db + DATABASE_URL: postgresql+psycopg://user:${POSTGRES_PASSWORD:-password}@db:5432/course_agent_db # JWT配置 SECRET_KEY: ${SECRET_KEY:-your-super-secret-key-change-in-production} @@ -43,7 +43,7 @@ services: ACCESS_TOKEN_EXPIRE_MINUTES: 30 # 硅基流动API配置 - SILICONFLOW_API_KEY: ${SILICONFLOW_API_KEY:-sk-pvvtosiglncktlucwarxilvsypqcttqizgpcfdvodgcuaezn} + SILICONFLOW_API_KEY: ${SILICONFLOW_API_KEY:?请设置 SILICONFLOW_API_KEY 环境变量} SILICONFLOW_BASE_URL: ${SILICONFLOW_BASE_URL:-https://api.siliconflow.cn/v1} SILICONFLOW_MODEL: ${SILICONFLOW_MODEL:-Qwen/Qwen3-30B-A3B-Thinking-2507}