feat: global exception handler, concurrent VLM, and docker security

- Add global FastAPI exception handler for unhandled errors - Add get_current_user_obj() dependency for cleaner auth patterns - Switch VLM image description from serial to asyncio.gather concurrency - Extract score conversion to shared score_utils module - Docker: use env vars for passwords, remove hardcoded API key default - Add .gitattributes and update .gitignore for tar.gz and tsbuildinfo Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-28 15:53:18 +08:00
parent 37c0364e7e
commit b8253b86a0
11 changed files with 196 additions and 131 deletions
@@ -1,6 +1,7 @@
 """
 文档管理API
 """
+import logging
 import asyncio
 import os
 import uuid
@@ -15,6 +16,8 @@ from ..core.security import get_current_user
 from ..models.document import Document, DocumentChunk
 from ..services.document_service import DocumentService

+logger = logging.getLogger(__name__)
+
 router = APIRouter(prefix="/documents", tags=["文档管理"])


@@ -186,32 +189,30 @@ async def delete_document(
        
        # 1. 先删除向量数据和文档块
        try:
-            print(f"开始删除文档向量数据: {document.filename} (ID: {document.id})")
+            logger.info(f"开始删除文档向量数据: {document.filename} (ID: {document.id})")
            document_service = DocumentService(db)
            vector_deleted = document_service.delete_document_chunks(document.id)
            if vector_deleted:
-                print(f"成功删除文档向量数据: {document.filename}")
+                logger.info(f"成功删除文档向量数据: {document.filename}")
            else:
-                print(f"警告：删除文档向量数据失败: {document.filename}")
+                logger.warning(f"警告：删除文档向量数据失败: {document.filename}")
        except Exception as e:
-            print(f"删除向量数据时发生错误: {str(e)}")
-            import traceback
-            print(f"详细错误信息: {traceback.format_exc()}")
+            logger.error(f"删除向量数据时发生错误: {str(e)}", exc_info=True)
        
        # 2. 删除物理文件
        try:
            if os.path.exists(document.file_path):
                os.remove(document.file_path)
-                print(f"成功删除物理文件: {document.file_path}")
+                logger.info(f"成功删除物理文件: {document.file_path}")
            else:
-                print(f"物理文件不存在: {document.file_path}")
+                logger.info(f"物理文件不存在: {document.file_path}")
        except Exception as e:
-            print(f"删除物理文件时发生错误: {str(e)}")
+            logger.error(f"删除物理文件时发生错误: {str(e)}")
        
        # 3. 删除数据库记录
        db.delete(document)
        db.commit()
-        print(f"成功删除文档数据库记录: {document.filename}")
+        logger.info(f"成功删除文档数据库记录: {document.filename}")
        
        return {"message": "文档删除成功"}
        
@@ -1,6 +1,7 @@
 """
 知识库CRUD API
 """
+import logging
 import os
 import uuid
 from pathlib import Path
@@ -18,6 +19,8 @@ from ..models.document import Document
 from ..models.user import User
 from ..services.document_service import DocumentService

+logger = logging.getLogger(__name__)
+
 router = APIRouter(prefix="/knowledge-bases", tags=["知识库管理"])


@@ -64,7 +67,7 @@ async def get_knowledge_bases(
    db: Session = Depends(get_db)
 ):
    """获取用户的所有知识库"""
-    print(f"DEBUG: get_knowledge_bases called for user: {current_user}")
+    logger.debug(f"get_knowledge_bases called for user: {current_user}")
    try:
        # 获取用户ID
        user = db.query(User).filter(User.username == current_user).first()
@@ -118,7 +121,7 @@ async def create_knowledge_base(
    db: Session = Depends(get_db)
 ):
    """创建新知识库"""
-    print(f"DEBUG: create_knowledge_base called for user: {current_user}, data: {data}")
+    logger.debug(f"create_knowledge_base called for user: {current_user}, data: {data}")
    try:
        # 获取用户ID
        user = db.query(User).filter(User.username == current_user).first()
@@ -368,7 +371,7 @@ async def delete_knowledge_base(
            Document.knowledge_base_id == knowledge_base_id
        ).all()
        
-        print(f"开始删除知识库 '{knowledge_base.name}'，包含 {len(documents)} 个文档")
+        logger.info(f"开始删除知识库 '{knowledge_base.name}'，包含 {len(documents)} 个文档")
        
        # 2. 逐个删除文档的向量数据和物理文件
        document_service = DocumentService(db)
@@ -380,33 +383,31 @@ async def delete_knowledge_base(
                # 删除向量数据
                vector_deleted = document_service.delete_document_chunks(document.id)
                if vector_deleted:
-                    print(f"成功删除文档向量数据: {document.filename}")
+                    logger.info(f"成功删除文档向量数据: {document.filename}")
                else:
-                    print(f"警告：删除文档向量数据失败: {document.filename}")
+                    logger.warning(f"警告：删除文档向量数据失败: {document.filename}")
                    error_count += 1
                
                # 删除物理文件
                if os.path.exists(document.file_path):
                    os.remove(document.file_path)
-                    print(f"成功删除物理文件: {document.file_path}")
+                    logger.info(f"成功删除物理文件: {document.file_path}")
                else:
-                    print(f"物理文件不存在: {document.file_path}")
+                    logger.info(f"物理文件不存在: {document.file_path}")
                
                success_count += 1
                
            except Exception as e:
-                print(f"删除文档 {document.filename} 的资源时出错: {str(e)}")
-                import traceback
-                print(f"详细错误信息: {traceback.format_exc()}")
+                logger.error(f"删除文档 {document.filename} 的资源时出错: {str(e)}", exc_info=True)
                error_count += 1
                # 继续处理其他文档
        
-        print(f"文档清理完成: 成功 {success_count} 个，失败 {error_count} 个")
+        logger.info(f"文档清理完成: 成功 {success_count} 个，失败 {error_count} 个")
        
        # 3. 删除知识库（级联删除文档记录）
        db.delete(knowledge_base)
        db.commit()
-        print(f"成功删除知识库数据库记录: {knowledge_base.name}")
+        logger.info(f"成功删除知识库数据库记录: {knowledge_base.name}")
        
        return {"message": "知识库删除成功"}
        
@@ -528,20 +529,18 @@ async def upload_document_to_knowledge_base(
        
        # 自动处理文档向量化
        try:
-            print(f"开始处理文档向量化: {document.filename} (ID: {document.id})")
+            logger.info(f"开始处理文档向量化: {document.filename} (ID: {document.id})")
            document_service = DocumentService(db)
            success = await document_service.process_document(document.id)
-            
+
            if success:
-                print(f"文档向量化处理成功: {document.filename}")
+                logger.info(f"文档向量化处理成功: {document.filename}")
                message = "文档上传并处理成功"
            else:
-                print(f"文档向量化处理失败: {document.filename}")
+                logger.warning(f"文档向量化处理失败: {document.filename}")
                message = "文档上传成功，但向量化处理失败"
        except Exception as e:
-            print(f"文档向量化处理异常: {document.filename}, 错误: {str(e)}")
-            import traceback
-            print(f"详细错误信息: {traceback.format_exc()}")
+            logger.error(f"文档向量化处理异常: {document.filename}, 错误: {str(e)}", exc_info=True)
            message = "文档上传成功，但向量化处理失败"
        
        return DocumentUploadResponse(
@@ -8,6 +8,7 @@ from passlib.context import CryptContext
 from fastapi import HTTPException, status, Depends
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 from pydantic import BaseModel
+from sqlalchemy.orm import Session

 from .config import get_settings

@@ -82,3 +83,24 @@ async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(s
        return username
    except JWTError:
        raise credentials_exception
+
+
+async def get_current_user_obj(current_user: str = Depends(get_current_user)):
+    """获取当前用户的完整User对象
+
+    用法: current_user: User = Depends(get_current_user_obj)
+    替代: current_user: str = Depends(get_current_user) + 手动 db.query(User)
+    """
+    from ..models.user import User
+    from .database import get_db
+    db = next(get_db())
+    try:
+        user = db.query(User).filter(User.username == current_user).first()
+        if not user:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail="用户不存在"
+            )
+        return user
+    finally:
+        db.close()
@@ -13,7 +13,7 @@ class ChatSession(Base):
    __tablename__ = "chat_sessions"
    
    id = Column(Integer, primary_key=True, index=True)
-    user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
+    user_id = Column(Integer, ForeignKey("users.id"), nullable=False, index=True)
    title = Column(String(200), nullable=True)
    is_active = Column(Boolean, default=True)
    created_at = Column(DateTime(timezone=True), server_default=func.now())
@@ -32,7 +32,7 @@ class ChatMessage(Base):
    __tablename__ = "chat_messages"
    
    id = Column(Integer, primary_key=True, index=True)
-    session_id = Column(Integer, ForeignKey("chat_sessions.id"), nullable=False)
+    session_id = Column(Integer, ForeignKey("chat_sessions.id"), nullable=False, index=True)
    role = Column(String(20), nullable=False)  # user, assistant, system
    content = Column(Text, nullable=False)
    message_metadata = Column(Text, nullable=True)  # JSON格式的元数据
@@ -14,7 +14,7 @@ class Document(Base):
    
    id = Column(Integer, primary_key=True, index=True)
    user_id = Column(Integer, ForeignKey("users.id"), nullable=True)  # None表示系统文档
-    knowledge_base_id = Column(Integer, ForeignKey("knowledge_bases.id"), nullable=True)  # 所属知识库
+    knowledge_base_id = Column(Integer, ForeignKey("knowledge_bases.id"), nullable=True, index=True)
    filename = Column(String(255), nullable=False)
    original_filename = Column(String(255), nullable=False)
    file_path = Column(String(500), nullable=False)
@@ -47,7 +47,7 @@ class DocumentChunk(Base):
    __tablename__ = "document_chunks"
    
    id = Column(Integer, primary_key=True, index=True)
-    document_id = Column(Integer, ForeignKey("documents.id"), nullable=False)
+    document_id = Column(Integer, ForeignKey("documents.id"), nullable=False, index=True)
    chunk_index = Column(Integer, nullable=False)
    content = Column(Text, nullable=False)
    content_hash = Column(String(64), nullable=False)  # 内容哈希
@@ -0,0 +1,13 @@
+"""
+分数转换工具
+"""
+import math
+
+
+def convert_distance_to_score(distance: float) -> float:
+    """将ChromaDB的distance转换为0-1范围的相似度分数"""
+    if distance < 0:
+        return (1 + distance) / 2
+    if distance > 100:
+        return 1 / (1 + math.log(distance))
+    return 1 / (1 + distance)
@@ -3,15 +3,19 @@
 """
 import os
 import asyncio
+import logging
 from pathlib import Path
 from typing import List, Dict, Any, Optional
 from sqlalchemy.orm import Session
 from langchain_core.documents import Document as LangChainDocument

 from ..models.document import Document, DocumentChunk
+
+logger = logging.getLogger(__name__)
 from ..rag.vector_store import get_vector_store
 from ..rag.document_loaders import DocumentLoaderFactory, PDFImageExtractor
 from ..rag.text_splitters import get_text_splitter
+from ..rag.score_utils import convert_distance_to_score
 from ..llm.siliconflow import get_llm_client

 IMAGES_DIR = Path(__file__).parent.parent.parent.parent / "data" / "images"
@@ -65,14 +69,14 @@ class DocumentService:
            if success:
                document.is_processed = True
                self.db.commit()
-                print(f"[DocumentService] 文档 {document.filename} 处理完成: "
+                logger.info(f"[DocumentService] 文档 {document.filename} 处理完成: "
                      f"{len(splits)} 个文本块, {len(image_chunks)} 个图片描述块")
                return True

            return False

        except Exception as e:
-            print(f"处理文档失败: {str(e)}")
+            logger.error(f"处理文档失败: {str(e)}")
            self.db.rollback()
            return False

@@ -94,56 +98,58 @@ class DocumentService:
            images = PDFImageExtractor.extract_images(str(file_path), str(img_output_dir))

            if not images:
-                print(f"[DocumentService] 未发现可提取的图片: {filename}")
+                logger.info(f"[DocumentService] 未发现可提取的图片: {filename}")
                return []

-            print(f"[DocumentService] 提取到 {len(images)} 张图片, 开始VLM描述生成...")
+            logger.info(f"[DocumentService] 提取到 {len(images)} 张图片, 开始VLM描述生成...")

-            # 批量调用VLM生成描述
+            # 并发调用VLM生成描述（限制并发度为5）
            llm_client = get_llm_client()
-            for idx, img in enumerate(images):
-                try:
-                    description = await llm_client.describe_image(
-                        img["path"],
-                        img.get("context_text", "")
-                    )
+            semaphore = asyncio.Semaphore(5)

-                    if description:
-                        # 相对路径用于URL访问
-                        rel_path = f"{document_id}/{img['filename']}"
-                        image_url = f"/images/{rel_path}"
-
-                        # 构建图片描述文本块（URL写入内容，LLM可直接引用）
-                        chunk_content = (
-                            f"[图片描述 - 第{img['page']}页]\n"
-                            f"图片URL: {image_url}\n"
-                            f"图片内容：{description}"
+            async def describe_single_image(idx, img):
+                async with semaphore:
+                    try:
+                        description = await llm_client.describe_image(
+                            img["path"],
+                            img.get("context_text", "")
                        )
+                        if description:
+                            rel_path = f"{document_id}/{img['filename']}"
+                            image_url = f"/images/{rel_path}"
+                            chunk_content = (
+                                f"[图片描述 - 第{img['page']}页]\n"
+                                f"图片URL: {image_url}\n"
+                                f"图片内容：{description}"
+                            )
+                            chunk = LangChainDocument(
+                                page_content=chunk_content,
+                                metadata={
+                                    "document_id": document_id,
+                                    "knowledge_base_id": knowledge_base_id,
+                                    "title": title,
+                                    "filename": filename,
+                                    "source_type": "image",
+                                    "image_path": str(img["path"]),
+                                    "image_url": image_url,
+                                    "page": img["page"],
+                                }
+                            )
+                            logger.info(f"[DocumentService] 图片描述成功 {idx+1}/{len(images)}: {img['filename']}")
+                            return chunk
+                        else:
+                            logger.warning(f"[DocumentService] 图片描述为空 {idx+1}/{len(images)}: {img['filename']}")
+                            return None
+                    except Exception as e:
+                        logger.error(f"[DocumentService] 图片处理失败 {img['filename']}: {e}")
+                        return None

-                        chunk = LangChainDocument(
-                            page_content=chunk_content,
-                            metadata={
-                                "document_id": document_id,
-                                "knowledge_base_id": knowledge_base_id,
-                                "title": title,
-                                "filename": filename,
-                                "source_type": "image",
-                                "image_path": str(img["path"]),
-                                "image_url": image_url,
-                                "page": img["page"],
-                            }
-                        )
-                        image_chunks.append(chunk)
-                        print(f"[DocumentService] 图片描述成功 {idx+1}/{len(images)}: {img['filename']}")
-                    else:
-                        print(f"[DocumentService] 图片描述为空 {idx+1}/{len(images)}: {img['filename']}")
-
-                except Exception as e:
-                    print(f"[DocumentService] 图片处理失败 {img['filename']}: {e}")
-                    continue
+            tasks = [describe_single_image(idx, img) for idx, img in enumerate(images)]
+            results = await asyncio.gather(*tasks)
+            image_chunks = [r for r in results if r is not None]

        except Exception as e:
-            print(f"[DocumentService] PDF图片处理失败: {e}")
+            logger.error(f"[DocumentService] PDF图片处理失败: {e}")

        return image_chunks
    
@@ -166,7 +172,7 @@ class DocumentService:
            search_results = []
            for doc, distance in results:
                metadata = doc.metadata if hasattr(doc, 'metadata') else {}
-                score = self._convert_distance_to_score(distance)
+                score = convert_distance_to_score(distance)
                
                search_results.append({
                    "content": doc.page_content,
@@ -178,24 +184,9 @@ class DocumentService:
            return search_results
            
        except Exception as e:
-            print(f"搜索文档失败: {str(e)}")
+            logger.error(f"搜索文档失败: {str(e)}")
            return []
    
-    def _convert_distance_to_score(self, distance: float) -> float:
-        """将ChromaDB的distance转换为0-1范围的相似度分数"""
-        import math
-        
-        # 内积距离（负值）
-        if distance < 0:
-            return (1 + distance) / 2
-        
-        # 大距离使用对数缩放
-        if distance > 100:
-            return 1 / (1 + math.log(distance))
-        
-        # 标准距离转换
-        return 1 / (1 + distance)
-    
    def get_document_chunks(self, document_id: int) -> List[DocumentChunk]:
        """获取文档的所有块"""
        return self.db.query(DocumentChunk).filter(
@@ -214,6 +205,6 @@ class DocumentService:
            self.db.commit()
            return True
        except Exception as e:
-            print(f"删除文档块失败: {str(e)}")
+            logger.error(f"删除文档块失败: {str(e)}")
            self.db.rollback()
            return False