feat: global exception handler, concurrent VLM, and docker security

- Add global FastAPI exception handler for unhandled errors
- Add get_current_user_obj() dependency for cleaner auth patterns
- Switch VLM image description from serial to asyncio.gather concurrency
- Extract score conversion to shared score_utils module
- Docker: use env vars for passwords, remove hardcoded API key default
- Add .gitattributes and update .gitignore for tar.gz and tsbuildinfo

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-05-28 15:53:18 +08:00
parent 37c0364e7e
commit b8253b86a0
11 changed files with 196 additions and 131 deletions
+37
View File
@@ -0,0 +1,37 @@
# Auto detect text files and perform LF normalization
* text=auto
# Source code - enforce LF line endings
*.py text eol=lf
*.ts text eol=lf
*.tsx text eol=lf
*.js text eol=lf
*.jsx text eol=lf
*.json text eol=lf
*.css text eol=lf
*.scss text eol=lf
*.html text eol=lf
*.md text eol=lf
*.yml text eol=lf
*.yaml text eol=lf
*.toml text eol=lf
*.sql text eol=lf
*.sh text eol=lf
*.bat text eol=lf
*.conf text eol=lf
*.cfg text eol=lf
*.txt text eol=lf
*.env text eol=lf
# Binary files - don't touch
*.png binary
*.jpg binary
*.jpeg binary
*.gif binary
*.ico binary
*.pdf binary
*.zip binary
*.woff binary
*.woff2 binary
*.ttf binary
*.eot binary
+1
View File
@@ -64,6 +64,7 @@ Desktop.ini
# 其他 # 其他
*.tmp *.tmp
*.temp *.temp
*.tar.gz
.cache/ .cache/
*.coverage *.coverage
.coverage .coverage
+34 -33
View File
@@ -4,14 +4,17 @@ FastAPI应用入口
""" """
import os import os
import sys import sys
import logging
from pathlib import Path from pathlib import Path
from datetime import datetime from datetime import datetime
from fastapi import FastAPI, HTTPException from fastapi import FastAPI, HTTPException, Request
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse from fastapi.responses import FileResponse, JSONResponse
import uvicorn import uvicorn
logger = logging.getLogger(__name__)
# 设置系统时区为北京时间 # 设置系统时区为北京时间
os.environ['TZ'] = 'Asia/Shanghai' os.environ['TZ'] = 'Asia/Shanghai'
@@ -52,9 +55,9 @@ def startup_knowledge_base():
try: try:
from src.services.file_watcher_service import start_file_watcher from src.services.file_watcher_service import start_file_watcher
start_file_watcher() start_file_watcher()
print("知识库文件监控服务启动成功") logger.info("知识库文件监控服务启动成功")
except Exception as e: except Exception as e:
print(f"启动知识库文件监控服务失败: {str(e)}") logger.error(f"启动知识库文件监控服务失败: {str(e)}")
def seed_forum_categories(): def seed_forum_categories():
@@ -72,17 +75,17 @@ def seed_forum_categories():
try: try:
existing_count = db.query(ForumCategory).count() existing_count = db.query(ForumCategory).count()
if existing_count > 0: if existing_count > 0:
print(f"论坛分类已存在({existing_count} 个),跳过初始化") logger.info(f"论坛分类已存在({existing_count} 个),跳过初始化")
return return
for cat_data in DEFAULT_CATEGORIES: for cat_data in DEFAULT_CATEGORIES:
category = ForumCategory(**cat_data) category = ForumCategory(**cat_data)
db.add(category) db.add(category)
db.commit() db.commit()
print(f"已创建 {len(DEFAULT_CATEGORIES)} 个论坛分类") logger.info(f"已创建 {len(DEFAULT_CATEGORIES)} 个论坛分类")
except Exception as e: except Exception as e:
db.rollback() db.rollback()
print(f"初始化论坛分类失败: {e}") logger.error(f"初始化论坛分类失败: {e}")
finally: finally:
db.close() db.close()
@@ -94,10 +97,9 @@ async def startup_event():
import asyncio import asyncio
import sys import sys
print("=" * 50, file=sys.stderr) logger.info("=" * 50)
print("应用启动事件开始", file=sys.stderr) logger.info("应用启动事件开始")
print("=" * 50, file=sys.stderr) logger.info("=" * 50)
sys.stderr.flush()
# 等待数据库可用(重试机制,处理 DNS 解析延迟) # 等待数据库可用(重试机制,处理 DNS 解析延迟)
max_retries = 10 max_retries = 10
@@ -105,28 +107,23 @@ async def startup_event():
for attempt in range(max_retries): for attempt in range(max_retries):
try: try:
print(f"尝试连接数据库(尝试 {attempt + 1}/{max_retries}...", file=sys.stderr) logger.info(f"尝试连接数据库(尝试 {attempt + 1}/{max_retries}...")
sys.stderr.flush()
# 尝试创建数据库表 # 尝试创建数据库表
create_tables() create_tables()
print("数据库表创建成功", file=sys.stderr) logger.info("数据库表创建成功")
sys.stderr.flush()
break break
except Exception as e: except Exception as e:
if attempt < max_retries - 1: if attempt < max_retries - 1:
print(f"数据库连接失败(尝试 {attempt + 1}/{max_retries}: {e}", file=sys.stderr) logger.warning(f"数据库连接失败(尝试 {attempt + 1}/{max_retries}: {e}")
print(f"等待 {retry_delay} 秒后重试...", file=sys.stderr)
sys.stderr.flush()
await asyncio.sleep(retry_delay) await asyncio.sleep(retry_delay)
else: else:
print(f"数据库连接失败,已达到最大重试次数: {e}", file=sys.stderr) logger.error(f"数据库连接失败,已达到最大重试次数: {e}")
sys.stderr.flush()
# 不抛出异常,让应用继续启动,但数据库操作会失败 # 不抛出异常,让应用继续启动,但数据库操作会失败
try: try:
startup_knowledge_base() startup_knowledge_base()
except Exception as e: except Exception as e:
print(f"启动知识库服务失败: {e}", file=sys.stderr) logger.error(f"启动知识库服务失败: {e}")
# 确保系统知识库与目录同步 # 确保系统知识库与目录同步
try: try:
@@ -137,17 +134,15 @@ async def startup_event():
kb_service.ensure_system_knowledge_bases() kb_service.ensure_system_knowledge_bases()
db.close() db.close()
except Exception as e: except Exception as e:
print(f"同步系统知识库失败: {e}", file=sys.stderr) logger.error(f"同步系统知识库失败: {e}")
# 初始化论坛分类 # 初始化论坛分类
try: try:
seed_forum_categories() seed_forum_categories()
except Exception as e: except Exception as e:
print(f"初始化论坛分类失败: {e}", file=sys.stderr) logger.error(f"初始化论坛分类失败: {e}")
sys.stderr.flush()
print("应用启动事件完成", file=sys.stderr) logger.info("应用启动事件完成")
sys.stderr.flush()
# 应用关闭事件 # 应用关闭事件
@app.on_event("shutdown") @app.on_event("shutdown")
@@ -156,9 +151,18 @@ async def shutdown_event():
try: try:
from src.services.file_watcher_service import stop_file_watcher from src.services.file_watcher_service import stop_file_watcher
stop_file_watcher() stop_file_watcher()
print("知识库文件监控服务已停止") logger.info("知识库文件监控服务已停止")
except Exception as e: except Exception as e:
print(f"停止知识库文件监控服务失败: {str(e)}") logger.error(f"停止知识库文件监控服务失败: {str(e)}")
# 全局异常处理器
@app.exception_handler(Exception)
async def generic_exception_handler(request: Request, exc: Exception):
logger.error(f"未处理的异常: {exc}", exc_info=True)
return JSONResponse(
status_code=500,
content={"detail": "服务器内部错误,请稍后重试"}
)
# 注册路由 # 注册路由
app.include_router(auth.router) app.include_router(auth.router)
@@ -315,11 +319,8 @@ async def list_services():
if __name__ == "__main__": if __name__ == "__main__":
import sys import sys
print("=" * 50, file=sys.stderr) logger.info("启动 Uvicorn 服务器...")
print("启动 Uvicorn 服务器...", file=sys.stderr) logger.info(f"Host: {settings.host}, Port: {settings.port}")
print(f"Host: {settings.host}, Port: {settings.port}", file=sys.stderr)
print("=" * 50, file=sys.stderr)
sys.stderr.flush()
uvicorn.run( uvicorn.run(
"main:app", "main:app",
+11 -10
View File
@@ -1,6 +1,7 @@
""" """
文档管理API 文档管理API
""" """
import logging
import asyncio import asyncio
import os import os
import uuid import uuid
@@ -15,6 +16,8 @@ from ..core.security import get_current_user
from ..models.document import Document, DocumentChunk from ..models.document import Document, DocumentChunk
from ..services.document_service import DocumentService from ..services.document_service import DocumentService
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/documents", tags=["文档管理"]) router = APIRouter(prefix="/documents", tags=["文档管理"])
@@ -186,32 +189,30 @@ async def delete_document(
# 1. 先删除向量数据和文档块 # 1. 先删除向量数据和文档块
try: try:
print(f"开始删除文档向量数据: {document.filename} (ID: {document.id})") logger.info(f"开始删除文档向量数据: {document.filename} (ID: {document.id})")
document_service = DocumentService(db) document_service = DocumentService(db)
vector_deleted = document_service.delete_document_chunks(document.id) vector_deleted = document_service.delete_document_chunks(document.id)
if vector_deleted: if vector_deleted:
print(f"成功删除文档向量数据: {document.filename}") logger.info(f"成功删除文档向量数据: {document.filename}")
else: else:
print(f"警告:删除文档向量数据失败: {document.filename}") logger.warning(f"警告:删除文档向量数据失败: {document.filename}")
except Exception as e: except Exception as e:
print(f"删除向量数据时发生错误: {str(e)}") logger.error(f"删除向量数据时发生错误: {str(e)}", exc_info=True)
import traceback
print(f"详细错误信息: {traceback.format_exc()}")
# 2. 删除物理文件 # 2. 删除物理文件
try: try:
if os.path.exists(document.file_path): if os.path.exists(document.file_path):
os.remove(document.file_path) os.remove(document.file_path)
print(f"成功删除物理文件: {document.file_path}") logger.info(f"成功删除物理文件: {document.file_path}")
else: else:
print(f"物理文件不存在: {document.file_path}") logger.info(f"物理文件不存在: {document.file_path}")
except Exception as e: except Exception as e:
print(f"删除物理文件时发生错误: {str(e)}") logger.error(f"删除物理文件时发生错误: {str(e)}")
# 3. 删除数据库记录 # 3. 删除数据库记录
db.delete(document) db.delete(document)
db.commit() db.commit()
print(f"成功删除文档数据库记录: {document.filename}") logger.info(f"成功删除文档数据库记录: {document.filename}")
return {"message": "文档删除成功"} return {"message": "文档删除成功"}
+17 -18
View File
@@ -1,6 +1,7 @@
""" """
知识库CRUD API 知识库CRUD API
""" """
import logging
import os import os
import uuid import uuid
from pathlib import Path from pathlib import Path
@@ -18,6 +19,8 @@ from ..models.document import Document
from ..models.user import User from ..models.user import User
from ..services.document_service import DocumentService from ..services.document_service import DocumentService
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/knowledge-bases", tags=["知识库管理"]) router = APIRouter(prefix="/knowledge-bases", tags=["知识库管理"])
@@ -64,7 +67,7 @@ async def get_knowledge_bases(
db: Session = Depends(get_db) db: Session = Depends(get_db)
): ):
"""获取用户的所有知识库""" """获取用户的所有知识库"""
print(f"DEBUG: get_knowledge_bases called for user: {current_user}") logger.debug(f"get_knowledge_bases called for user: {current_user}")
try: try:
# 获取用户ID # 获取用户ID
user = db.query(User).filter(User.username == current_user).first() user = db.query(User).filter(User.username == current_user).first()
@@ -118,7 +121,7 @@ async def create_knowledge_base(
db: Session = Depends(get_db) db: Session = Depends(get_db)
): ):
"""创建新知识库""" """创建新知识库"""
print(f"DEBUG: create_knowledge_base called for user: {current_user}, data: {data}") logger.debug(f"create_knowledge_base called for user: {current_user}, data: {data}")
try: try:
# 获取用户ID # 获取用户ID
user = db.query(User).filter(User.username == current_user).first() user = db.query(User).filter(User.username == current_user).first()
@@ -368,7 +371,7 @@ async def delete_knowledge_base(
Document.knowledge_base_id == knowledge_base_id Document.knowledge_base_id == knowledge_base_id
).all() ).all()
print(f"开始删除知识库 '{knowledge_base.name}',包含 {len(documents)} 个文档") logger.info(f"开始删除知识库 '{knowledge_base.name}',包含 {len(documents)} 个文档")
# 2. 逐个删除文档的向量数据和物理文件 # 2. 逐个删除文档的向量数据和物理文件
document_service = DocumentService(db) document_service = DocumentService(db)
@@ -380,33 +383,31 @@ async def delete_knowledge_base(
# 删除向量数据 # 删除向量数据
vector_deleted = document_service.delete_document_chunks(document.id) vector_deleted = document_service.delete_document_chunks(document.id)
if vector_deleted: if vector_deleted:
print(f"成功删除文档向量数据: {document.filename}") logger.info(f"成功删除文档向量数据: {document.filename}")
else: else:
print(f"警告:删除文档向量数据失败: {document.filename}") logger.warning(f"警告:删除文档向量数据失败: {document.filename}")
error_count += 1 error_count += 1
# 删除物理文件 # 删除物理文件
if os.path.exists(document.file_path): if os.path.exists(document.file_path):
os.remove(document.file_path) os.remove(document.file_path)
print(f"成功删除物理文件: {document.file_path}") logger.info(f"成功删除物理文件: {document.file_path}")
else: else:
print(f"物理文件不存在: {document.file_path}") logger.info(f"物理文件不存在: {document.file_path}")
success_count += 1 success_count += 1
except Exception as e: except Exception as e:
print(f"删除文档 {document.filename} 的资源时出错: {str(e)}") logger.error(f"删除文档 {document.filename} 的资源时出错: {str(e)}", exc_info=True)
import traceback
print(f"详细错误信息: {traceback.format_exc()}")
error_count += 1 error_count += 1
# 继续处理其他文档 # 继续处理其他文档
print(f"文档清理完成: 成功 {success_count} 个,失败 {error_count}") logger.info(f"文档清理完成: 成功 {success_count} 个,失败 {error_count}")
# 3. 删除知识库(级联删除文档记录) # 3. 删除知识库(级联删除文档记录)
db.delete(knowledge_base) db.delete(knowledge_base)
db.commit() db.commit()
print(f"成功删除知识库数据库记录: {knowledge_base.name}") logger.info(f"成功删除知识库数据库记录: {knowledge_base.name}")
return {"message": "知识库删除成功"} return {"message": "知识库删除成功"}
@@ -528,20 +529,18 @@ async def upload_document_to_knowledge_base(
# 自动处理文档向量化 # 自动处理文档向量化
try: try:
print(f"开始处理文档向量化: {document.filename} (ID: {document.id})") logger.info(f"开始处理文档向量化: {document.filename} (ID: {document.id})")
document_service = DocumentService(db) document_service = DocumentService(db)
success = await document_service.process_document(document.id) success = await document_service.process_document(document.id)
if success: if success:
print(f"文档向量化处理成功: {document.filename}") logger.info(f"文档向量化处理成功: {document.filename}")
message = "文档上传并处理成功" message = "文档上传并处理成功"
else: else:
print(f"文档向量化处理失败: {document.filename}") logger.warning(f"文档向量化处理失败: {document.filename}")
message = "文档上传成功,但向量化处理失败" message = "文档上传成功,但向量化处理失败"
except Exception as e: except Exception as e:
print(f"文档向量化处理异常: {document.filename}, 错误: {str(e)}") logger.error(f"文档向量化处理异常: {document.filename}, 错误: {str(e)}", exc_info=True)
import traceback
print(f"详细错误信息: {traceback.format_exc()}")
message = "文档上传成功,但向量化处理失败" message = "文档上传成功,但向量化处理失败"
return DocumentUploadResponse( return DocumentUploadResponse(
+22
View File
@@ -8,6 +8,7 @@ from passlib.context import CryptContext
from fastapi import HTTPException, status, Depends from fastapi import HTTPException, status, Depends
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from pydantic import BaseModel from pydantic import BaseModel
from sqlalchemy.orm import Session
from .config import get_settings from .config import get_settings
@@ -82,3 +83,24 @@ async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(s
return username return username
except JWTError: except JWTError:
raise credentials_exception raise credentials_exception
async def get_current_user_obj(current_user: str = Depends(get_current_user)):
"""获取当前用户的完整User对象
用法: current_user: User = Depends(get_current_user_obj)
替代: current_user: str = Depends(get_current_user) + 手动 db.query(User)
"""
from ..models.user import User
from .database import get_db
db = next(get_db())
try:
user = db.query(User).filter(User.username == current_user).first()
if not user:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="用户不存在"
)
return user
finally:
db.close()
+2 -2
View File
@@ -13,7 +13,7 @@ class ChatSession(Base):
__tablename__ = "chat_sessions" __tablename__ = "chat_sessions"
id = Column(Integer, primary_key=True, index=True) id = Column(Integer, primary_key=True, index=True)
user_id = Column(Integer, ForeignKey("users.id"), nullable=False) user_id = Column(Integer, ForeignKey("users.id"), nullable=False, index=True)
title = Column(String(200), nullable=True) title = Column(String(200), nullable=True)
is_active = Column(Boolean, default=True) is_active = Column(Boolean, default=True)
created_at = Column(DateTime(timezone=True), server_default=func.now()) created_at = Column(DateTime(timezone=True), server_default=func.now())
@@ -32,7 +32,7 @@ class ChatMessage(Base):
__tablename__ = "chat_messages" __tablename__ = "chat_messages"
id = Column(Integer, primary_key=True, index=True) id = Column(Integer, primary_key=True, index=True)
session_id = Column(Integer, ForeignKey("chat_sessions.id"), nullable=False) session_id = Column(Integer, ForeignKey("chat_sessions.id"), nullable=False, index=True)
role = Column(String(20), nullable=False) # user, assistant, system role = Column(String(20), nullable=False) # user, assistant, system
content = Column(Text, nullable=False) content = Column(Text, nullable=False)
message_metadata = Column(Text, nullable=True) # JSON格式的元数据 message_metadata = Column(Text, nullable=True) # JSON格式的元数据
+2 -2
View File
@@ -14,7 +14,7 @@ class Document(Base):
id = Column(Integer, primary_key=True, index=True) id = Column(Integer, primary_key=True, index=True)
user_id = Column(Integer, ForeignKey("users.id"), nullable=True) # None表示系统文档 user_id = Column(Integer, ForeignKey("users.id"), nullable=True) # None表示系统文档
knowledge_base_id = Column(Integer, ForeignKey("knowledge_bases.id"), nullable=True) # 所属知识库 knowledge_base_id = Column(Integer, ForeignKey("knowledge_bases.id"), nullable=True, index=True)
filename = Column(String(255), nullable=False) filename = Column(String(255), nullable=False)
original_filename = Column(String(255), nullable=False) original_filename = Column(String(255), nullable=False)
file_path = Column(String(500), nullable=False) file_path = Column(String(500), nullable=False)
@@ -47,7 +47,7 @@ class DocumentChunk(Base):
__tablename__ = "document_chunks" __tablename__ = "document_chunks"
id = Column(Integer, primary_key=True, index=True) id = Column(Integer, primary_key=True, index=True)
document_id = Column(Integer, ForeignKey("documents.id"), nullable=False) document_id = Column(Integer, ForeignKey("documents.id"), nullable=False, index=True)
chunk_index = Column(Integer, nullable=False) chunk_index = Column(Integer, nullable=False)
content = Column(Text, nullable=False) content = Column(Text, nullable=False)
content_hash = Column(String(64), nullable=False) # 内容哈希 content_hash = Column(String(64), nullable=False) # 内容哈希
+13
View File
@@ -0,0 +1,13 @@
"""
分数转换工具
"""
import math
def convert_distance_to_score(distance: float) -> float:
"""将ChromaDB的distance转换为0-1范围的相似度分数"""
if distance < 0:
return (1 + distance) / 2
if distance > 100:
return 1 / (1 + math.log(distance))
return 1 / (1 + distance)
+52 -61
View File
@@ -3,15 +3,19 @@
""" """
import os import os
import asyncio import asyncio
import logging
from pathlib import Path from pathlib import Path
from typing import List, Dict, Any, Optional from typing import List, Dict, Any, Optional
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from langchain_core.documents import Document as LangChainDocument from langchain_core.documents import Document as LangChainDocument
from ..models.document import Document, DocumentChunk from ..models.document import Document, DocumentChunk
logger = logging.getLogger(__name__)
from ..rag.vector_store import get_vector_store from ..rag.vector_store import get_vector_store
from ..rag.document_loaders import DocumentLoaderFactory, PDFImageExtractor from ..rag.document_loaders import DocumentLoaderFactory, PDFImageExtractor
from ..rag.text_splitters import get_text_splitter from ..rag.text_splitters import get_text_splitter
from ..rag.score_utils import convert_distance_to_score
from ..llm.siliconflow import get_llm_client from ..llm.siliconflow import get_llm_client
IMAGES_DIR = Path(__file__).parent.parent.parent.parent / "data" / "images" IMAGES_DIR = Path(__file__).parent.parent.parent.parent / "data" / "images"
@@ -65,14 +69,14 @@ class DocumentService:
if success: if success:
document.is_processed = True document.is_processed = True
self.db.commit() self.db.commit()
print(f"[DocumentService] 文档 {document.filename} 处理完成: " logger.info(f"[DocumentService] 文档 {document.filename} 处理完成: "
f"{len(splits)} 个文本块, {len(image_chunks)} 个图片描述块") f"{len(splits)} 个文本块, {len(image_chunks)} 个图片描述块")
return True return True
return False return False
except Exception as e: except Exception as e:
print(f"处理文档失败: {str(e)}") logger.error(f"处理文档失败: {str(e)}")
self.db.rollback() self.db.rollback()
return False return False
@@ -94,56 +98,58 @@ class DocumentService:
images = PDFImageExtractor.extract_images(str(file_path), str(img_output_dir)) images = PDFImageExtractor.extract_images(str(file_path), str(img_output_dir))
if not images: if not images:
print(f"[DocumentService] 未发现可提取的图片: {filename}") logger.info(f"[DocumentService] 未发现可提取的图片: {filename}")
return [] return []
print(f"[DocumentService] 提取到 {len(images)} 张图片, 开始VLM描述生成...") logger.info(f"[DocumentService] 提取到 {len(images)} 张图片, 开始VLM描述生成...")
# 批量调用VLM生成描述 # 并发调用VLM生成描述(限制并发度为5
llm_client = get_llm_client() llm_client = get_llm_client()
for idx, img in enumerate(images): semaphore = asyncio.Semaphore(5)
try:
description = await llm_client.describe_image(
img["path"],
img.get("context_text", "")
)
if description: async def describe_single_image(idx, img):
# 相对路径用于URL访问 async with semaphore:
rel_path = f"{document_id}/{img['filename']}" try:
image_url = f"/images/{rel_path}" description = await llm_client.describe_image(
img["path"],
# 构建图片描述文本块(URL写入内容,LLM可直接引用) img.get("context_text", "")
chunk_content = (
f"[图片描述 - 第{img['page']}页]\n"
f"图片URL: {image_url}\n"
f"图片内容:{description}"
) )
if description:
rel_path = f"{document_id}/{img['filename']}"
image_url = f"/images/{rel_path}"
chunk_content = (
f"[图片描述 - 第{img['page']}页]\n"
f"图片URL: {image_url}\n"
f"图片内容:{description}"
)
chunk = LangChainDocument(
page_content=chunk_content,
metadata={
"document_id": document_id,
"knowledge_base_id": knowledge_base_id,
"title": title,
"filename": filename,
"source_type": "image",
"image_path": str(img["path"]),
"image_url": image_url,
"page": img["page"],
}
)
logger.info(f"[DocumentService] 图片描述成功 {idx+1}/{len(images)}: {img['filename']}")
return chunk
else:
logger.warning(f"[DocumentService] 图片描述为空 {idx+1}/{len(images)}: {img['filename']}")
return None
except Exception as e:
logger.error(f"[DocumentService] 图片处理失败 {img['filename']}: {e}")
return None
chunk = LangChainDocument( tasks = [describe_single_image(idx, img) for idx, img in enumerate(images)]
page_content=chunk_content, results = await asyncio.gather(*tasks)
metadata={ image_chunks = [r for r in results if r is not None]
"document_id": document_id,
"knowledge_base_id": knowledge_base_id,
"title": title,
"filename": filename,
"source_type": "image",
"image_path": str(img["path"]),
"image_url": image_url,
"page": img["page"],
}
)
image_chunks.append(chunk)
print(f"[DocumentService] 图片描述成功 {idx+1}/{len(images)}: {img['filename']}")
else:
print(f"[DocumentService] 图片描述为空 {idx+1}/{len(images)}: {img['filename']}")
except Exception as e:
print(f"[DocumentService] 图片处理失败 {img['filename']}: {e}")
continue
except Exception as e: except Exception as e:
print(f"[DocumentService] PDF图片处理失败: {e}") logger.error(f"[DocumentService] PDF图片处理失败: {e}")
return image_chunks return image_chunks
@@ -166,7 +172,7 @@ class DocumentService:
search_results = [] search_results = []
for doc, distance in results: for doc, distance in results:
metadata = doc.metadata if hasattr(doc, 'metadata') else {} metadata = doc.metadata if hasattr(doc, 'metadata') else {}
score = self._convert_distance_to_score(distance) score = convert_distance_to_score(distance)
search_results.append({ search_results.append({
"content": doc.page_content, "content": doc.page_content,
@@ -178,24 +184,9 @@ class DocumentService:
return search_results return search_results
except Exception as e: except Exception as e:
print(f"搜索文档失败: {str(e)}") logger.error(f"搜索文档失败: {str(e)}")
return [] return []
def _convert_distance_to_score(self, distance: float) -> float:
"""将ChromaDB的distance转换为0-1范围的相似度分数"""
import math
# 内积距离(负值)
if distance < 0:
return (1 + distance) / 2
# 大距离使用对数缩放
if distance > 100:
return 1 / (1 + math.log(distance))
# 标准距离转换
return 1 / (1 + distance)
def get_document_chunks(self, document_id: int) -> List[DocumentChunk]: def get_document_chunks(self, document_id: int) -> List[DocumentChunk]:
"""获取文档的所有块""" """获取文档的所有块"""
return self.db.query(DocumentChunk).filter( return self.db.query(DocumentChunk).filter(
@@ -214,6 +205,6 @@ class DocumentService:
self.db.commit() self.db.commit()
return True return True
except Exception as e: except Exception as e:
print(f"删除文档块失败: {str(e)}") logger.error(f"删除文档块失败: {str(e)}")
self.db.rollback() self.db.rollback()
return False return False
+3 -3
View File
@@ -6,7 +6,7 @@ services:
environment: environment:
POSTGRES_DB: course_agent_db POSTGRES_DB: course_agent_db
POSTGRES_USER: user POSTGRES_USER: user
POSTGRES_PASSWORD: password POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C" POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C"
volumes: volumes:
- postgres_data:/var/lib/postgresql/data - postgres_data:/var/lib/postgresql/data
@@ -35,7 +35,7 @@ services:
- "8001:8001" # 前端应用 - "8001:8001" # 前端应用
environment: environment:
# 数据库配置 # 数据库配置
DATABASE_URL: postgresql+psycopg://user:password@db:5432/course_agent_db DATABASE_URL: postgresql+psycopg://user:${POSTGRES_PASSWORD:-password}@db:5432/course_agent_db
# JWT配置 # JWT配置
SECRET_KEY: ${SECRET_KEY:-your-super-secret-key-change-in-production} SECRET_KEY: ${SECRET_KEY:-your-super-secret-key-change-in-production}
@@ -43,7 +43,7 @@ services:
ACCESS_TOKEN_EXPIRE_MINUTES: 30 ACCESS_TOKEN_EXPIRE_MINUTES: 30
# 硅基流动API配置 # 硅基流动API配置
SILICONFLOW_API_KEY: ${SILICONFLOW_API_KEY:-sk-pvvtosiglncktlucwarxilvsypqcttqizgpcfdvodgcuaezn} SILICONFLOW_API_KEY: ${SILICONFLOW_API_KEY:?请设置 SILICONFLOW_API_KEY 环境变量}
SILICONFLOW_BASE_URL: ${SILICONFLOW_BASE_URL:-https://api.siliconflow.cn/v1} SILICONFLOW_BASE_URL: ${SILICONFLOW_BASE_URL:-https://api.siliconflow.cn/v1}
SILICONFLOW_MODEL: ${SILICONFLOW_MODEL:-Qwen/Qwen3-30B-A3B-Thinking-2507} SILICONFLOW_MODEL: ${SILICONFLOW_MODEL:-Qwen/Qwen3-30B-A3B-Thinking-2507}