feat: global exception handler, concurrent VLM, and docker security
- Add global FastAPI exception handler for unhandled errors - Add get_current_user_obj() dependency for cleaner auth patterns - Switch VLM image description from serial to asyncio.gather concurrency - Extract score conversion to shared score_utils module - Docker: use env vars for passwords, remove hardcoded API key default - Add .gitattributes and update .gitignore for tar.gz and tsbuildinfo Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
|
||||
# Source code - enforce LF line endings
|
||||
*.py text eol=lf
|
||||
*.ts text eol=lf
|
||||
*.tsx text eol=lf
|
||||
*.js text eol=lf
|
||||
*.jsx text eol=lf
|
||||
*.json text eol=lf
|
||||
*.css text eol=lf
|
||||
*.scss text eol=lf
|
||||
*.html text eol=lf
|
||||
*.md text eol=lf
|
||||
*.yml text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.toml text eol=lf
|
||||
*.sql text eol=lf
|
||||
*.sh text eol=lf
|
||||
*.bat text eol=lf
|
||||
*.conf text eol=lf
|
||||
*.cfg text eol=lf
|
||||
*.txt text eol=lf
|
||||
*.env text eol=lf
|
||||
|
||||
# Binary files - don't touch
|
||||
*.png binary
|
||||
*.jpg binary
|
||||
*.jpeg binary
|
||||
*.gif binary
|
||||
*.ico binary
|
||||
*.pdf binary
|
||||
*.zip binary
|
||||
*.woff binary
|
||||
*.woff2 binary
|
||||
*.ttf binary
|
||||
*.eot binary
|
||||
@@ -64,6 +64,7 @@ Desktop.ini
|
||||
# 其他
|
||||
*.tmp
|
||||
*.temp
|
||||
*.tar.gz
|
||||
.cache/
|
||||
*.coverage
|
||||
.coverage
|
||||
|
||||
+34
-33
@@ -4,14 +4,17 @@ FastAPI应用入口
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
import uvicorn
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 设置系统时区为北京时间
|
||||
os.environ['TZ'] = 'Asia/Shanghai'
|
||||
|
||||
@@ -52,9 +55,9 @@ def startup_knowledge_base():
|
||||
try:
|
||||
from src.services.file_watcher_service import start_file_watcher
|
||||
start_file_watcher()
|
||||
print("知识库文件监控服务启动成功")
|
||||
logger.info("知识库文件监控服务启动成功")
|
||||
except Exception as e:
|
||||
print(f"启动知识库文件监控服务失败: {str(e)}")
|
||||
logger.error(f"启动知识库文件监控服务失败: {str(e)}")
|
||||
|
||||
|
||||
def seed_forum_categories():
|
||||
@@ -72,17 +75,17 @@ def seed_forum_categories():
|
||||
try:
|
||||
existing_count = db.query(ForumCategory).count()
|
||||
if existing_count > 0:
|
||||
print(f"论坛分类已存在({existing_count} 个),跳过初始化")
|
||||
logger.info(f"论坛分类已存在({existing_count} 个),跳过初始化")
|
||||
return
|
||||
|
||||
for cat_data in DEFAULT_CATEGORIES:
|
||||
category = ForumCategory(**cat_data)
|
||||
db.add(category)
|
||||
db.commit()
|
||||
print(f"已创建 {len(DEFAULT_CATEGORIES)} 个论坛分类")
|
||||
logger.info(f"已创建 {len(DEFAULT_CATEGORIES)} 个论坛分类")
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
print(f"初始化论坛分类失败: {e}")
|
||||
logger.error(f"初始化论坛分类失败: {e}")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -94,10 +97,9 @@ async def startup_event():
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
print("=" * 50, file=sys.stderr)
|
||||
print("应用启动事件开始", file=sys.stderr)
|
||||
print("=" * 50, file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
logger.info("=" * 50)
|
||||
logger.info("应用启动事件开始")
|
||||
logger.info("=" * 50)
|
||||
|
||||
# 等待数据库可用(重试机制,处理 DNS 解析延迟)
|
||||
max_retries = 10
|
||||
@@ -105,28 +107,23 @@ async def startup_event():
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
print(f"尝试连接数据库(尝试 {attempt + 1}/{max_retries})...", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
logger.info(f"尝试连接数据库(尝试 {attempt + 1}/{max_retries})...")
|
||||
# 尝试创建数据库表
|
||||
create_tables()
|
||||
print("数据库表创建成功", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
logger.info("数据库表创建成功")
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt < max_retries - 1:
|
||||
print(f"数据库连接失败(尝试 {attempt + 1}/{max_retries}): {e}", file=sys.stderr)
|
||||
print(f"等待 {retry_delay} 秒后重试...", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
logger.warning(f"数据库连接失败(尝试 {attempt + 1}/{max_retries}): {e}")
|
||||
await asyncio.sleep(retry_delay)
|
||||
else:
|
||||
print(f"数据库连接失败,已达到最大重试次数: {e}", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
logger.error(f"数据库连接失败,已达到最大重试次数: {e}")
|
||||
# 不抛出异常,让应用继续启动,但数据库操作会失败
|
||||
|
||||
try:
|
||||
startup_knowledge_base()
|
||||
except Exception as e:
|
||||
print(f"启动知识库服务失败: {e}", file=sys.stderr)
|
||||
logger.error(f"启动知识库服务失败: {e}")
|
||||
|
||||
# 确保系统知识库与目录同步
|
||||
try:
|
||||
@@ -137,17 +134,15 @@ async def startup_event():
|
||||
kb_service.ensure_system_knowledge_bases()
|
||||
db.close()
|
||||
except Exception as e:
|
||||
print(f"同步系统知识库失败: {e}", file=sys.stderr)
|
||||
logger.error(f"同步系统知识库失败: {e}")
|
||||
|
||||
# 初始化论坛分类
|
||||
try:
|
||||
seed_forum_categories()
|
||||
except Exception as e:
|
||||
print(f"初始化论坛分类失败: {e}", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
logger.error(f"初始化论坛分类失败: {e}")
|
||||
|
||||
print("应用启动事件完成", file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
logger.info("应用启动事件完成")
|
||||
|
||||
# 应用关闭事件
|
||||
@app.on_event("shutdown")
|
||||
@@ -156,9 +151,18 @@ async def shutdown_event():
|
||||
try:
|
||||
from src.services.file_watcher_service import stop_file_watcher
|
||||
stop_file_watcher()
|
||||
print("知识库文件监控服务已停止")
|
||||
logger.info("知识库文件监控服务已停止")
|
||||
except Exception as e:
|
||||
print(f"停止知识库文件监控服务失败: {str(e)}")
|
||||
logger.error(f"停止知识库文件监控服务失败: {str(e)}")
|
||||
|
||||
# 全局异常处理器
|
||||
@app.exception_handler(Exception)
|
||||
async def generic_exception_handler(request: Request, exc: Exception):
|
||||
logger.error(f"未处理的异常: {exc}", exc_info=True)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"detail": "服务器内部错误,请稍后重试"}
|
||||
)
|
||||
|
||||
# 注册路由
|
||||
app.include_router(auth.router)
|
||||
@@ -315,11 +319,8 @@ async def list_services():
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
print("=" * 50, file=sys.stderr)
|
||||
print("启动 Uvicorn 服务器...", file=sys.stderr)
|
||||
print(f"Host: {settings.host}, Port: {settings.port}", file=sys.stderr)
|
||||
print("=" * 50, file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
logger.info("启动 Uvicorn 服务器...")
|
||||
logger.info(f"Host: {settings.host}, Port: {settings.port}")
|
||||
|
||||
uvicorn.run(
|
||||
"main:app",
|
||||
|
||||
+11
-10
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
文档管理API
|
||||
"""
|
||||
import logging
|
||||
import asyncio
|
||||
import os
|
||||
import uuid
|
||||
@@ -15,6 +16,8 @@ from ..core.security import get_current_user
|
||||
from ..models.document import Document, DocumentChunk
|
||||
from ..services.document_service import DocumentService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/documents", tags=["文档管理"])
|
||||
|
||||
|
||||
@@ -186,32 +189,30 @@ async def delete_document(
|
||||
|
||||
# 1. 先删除向量数据和文档块
|
||||
try:
|
||||
print(f"开始删除文档向量数据: {document.filename} (ID: {document.id})")
|
||||
logger.info(f"开始删除文档向量数据: {document.filename} (ID: {document.id})")
|
||||
document_service = DocumentService(db)
|
||||
vector_deleted = document_service.delete_document_chunks(document.id)
|
||||
if vector_deleted:
|
||||
print(f"成功删除文档向量数据: {document.filename}")
|
||||
logger.info(f"成功删除文档向量数据: {document.filename}")
|
||||
else:
|
||||
print(f"警告:删除文档向量数据失败: {document.filename}")
|
||||
logger.warning(f"警告:删除文档向量数据失败: {document.filename}")
|
||||
except Exception as e:
|
||||
print(f"删除向量数据时发生错误: {str(e)}")
|
||||
import traceback
|
||||
print(f"详细错误信息: {traceback.format_exc()}")
|
||||
logger.error(f"删除向量数据时发生错误: {str(e)}", exc_info=True)
|
||||
|
||||
# 2. 删除物理文件
|
||||
try:
|
||||
if os.path.exists(document.file_path):
|
||||
os.remove(document.file_path)
|
||||
print(f"成功删除物理文件: {document.file_path}")
|
||||
logger.info(f"成功删除物理文件: {document.file_path}")
|
||||
else:
|
||||
print(f"物理文件不存在: {document.file_path}")
|
||||
logger.info(f"物理文件不存在: {document.file_path}")
|
||||
except Exception as e:
|
||||
print(f"删除物理文件时发生错误: {str(e)}")
|
||||
logger.error(f"删除物理文件时发生错误: {str(e)}")
|
||||
|
||||
# 3. 删除数据库记录
|
||||
db.delete(document)
|
||||
db.commit()
|
||||
print(f"成功删除文档数据库记录: {document.filename}")
|
||||
logger.info(f"成功删除文档数据库记录: {document.filename}")
|
||||
|
||||
return {"message": "文档删除成功"}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
知识库CRUD API
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
@@ -18,6 +19,8 @@ from ..models.document import Document
|
||||
from ..models.user import User
|
||||
from ..services.document_service import DocumentService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/knowledge-bases", tags=["知识库管理"])
|
||||
|
||||
|
||||
@@ -64,7 +67,7 @@ async def get_knowledge_bases(
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""获取用户的所有知识库"""
|
||||
print(f"DEBUG: get_knowledge_bases called for user: {current_user}")
|
||||
logger.debug(f"get_knowledge_bases called for user: {current_user}")
|
||||
try:
|
||||
# 获取用户ID
|
||||
user = db.query(User).filter(User.username == current_user).first()
|
||||
@@ -118,7 +121,7 @@ async def create_knowledge_base(
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""创建新知识库"""
|
||||
print(f"DEBUG: create_knowledge_base called for user: {current_user}, data: {data}")
|
||||
logger.debug(f"create_knowledge_base called for user: {current_user}, data: {data}")
|
||||
try:
|
||||
# 获取用户ID
|
||||
user = db.query(User).filter(User.username == current_user).first()
|
||||
@@ -368,7 +371,7 @@ async def delete_knowledge_base(
|
||||
Document.knowledge_base_id == knowledge_base_id
|
||||
).all()
|
||||
|
||||
print(f"开始删除知识库 '{knowledge_base.name}',包含 {len(documents)} 个文档")
|
||||
logger.info(f"开始删除知识库 '{knowledge_base.name}',包含 {len(documents)} 个文档")
|
||||
|
||||
# 2. 逐个删除文档的向量数据和物理文件
|
||||
document_service = DocumentService(db)
|
||||
@@ -380,33 +383,31 @@ async def delete_knowledge_base(
|
||||
# 删除向量数据
|
||||
vector_deleted = document_service.delete_document_chunks(document.id)
|
||||
if vector_deleted:
|
||||
print(f"成功删除文档向量数据: {document.filename}")
|
||||
logger.info(f"成功删除文档向量数据: {document.filename}")
|
||||
else:
|
||||
print(f"警告:删除文档向量数据失败: {document.filename}")
|
||||
logger.warning(f"警告:删除文档向量数据失败: {document.filename}")
|
||||
error_count += 1
|
||||
|
||||
# 删除物理文件
|
||||
if os.path.exists(document.file_path):
|
||||
os.remove(document.file_path)
|
||||
print(f"成功删除物理文件: {document.file_path}")
|
||||
logger.info(f"成功删除物理文件: {document.file_path}")
|
||||
else:
|
||||
print(f"物理文件不存在: {document.file_path}")
|
||||
logger.info(f"物理文件不存在: {document.file_path}")
|
||||
|
||||
success_count += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"删除文档 {document.filename} 的资源时出错: {str(e)}")
|
||||
import traceback
|
||||
print(f"详细错误信息: {traceback.format_exc()}")
|
||||
logger.error(f"删除文档 {document.filename} 的资源时出错: {str(e)}", exc_info=True)
|
||||
error_count += 1
|
||||
# 继续处理其他文档
|
||||
|
||||
print(f"文档清理完成: 成功 {success_count} 个,失败 {error_count} 个")
|
||||
logger.info(f"文档清理完成: 成功 {success_count} 个,失败 {error_count} 个")
|
||||
|
||||
# 3. 删除知识库(级联删除文档记录)
|
||||
db.delete(knowledge_base)
|
||||
db.commit()
|
||||
print(f"成功删除知识库数据库记录: {knowledge_base.name}")
|
||||
logger.info(f"成功删除知识库数据库记录: {knowledge_base.name}")
|
||||
|
||||
return {"message": "知识库删除成功"}
|
||||
|
||||
@@ -528,20 +529,18 @@ async def upload_document_to_knowledge_base(
|
||||
|
||||
# 自动处理文档向量化
|
||||
try:
|
||||
print(f"开始处理文档向量化: {document.filename} (ID: {document.id})")
|
||||
logger.info(f"开始处理文档向量化: {document.filename} (ID: {document.id})")
|
||||
document_service = DocumentService(db)
|
||||
success = await document_service.process_document(document.id)
|
||||
|
||||
if success:
|
||||
print(f"文档向量化处理成功: {document.filename}")
|
||||
logger.info(f"文档向量化处理成功: {document.filename}")
|
||||
message = "文档上传并处理成功"
|
||||
else:
|
||||
print(f"文档向量化处理失败: {document.filename}")
|
||||
logger.warning(f"文档向量化处理失败: {document.filename}")
|
||||
message = "文档上传成功,但向量化处理失败"
|
||||
except Exception as e:
|
||||
print(f"文档向量化处理异常: {document.filename}, 错误: {str(e)}")
|
||||
import traceback
|
||||
print(f"详细错误信息: {traceback.format_exc()}")
|
||||
logger.error(f"文档向量化处理异常: {document.filename}, 错误: {str(e)}", exc_info=True)
|
||||
message = "文档上传成功,但向量化处理失败"
|
||||
|
||||
return DocumentUploadResponse(
|
||||
|
||||
@@ -8,6 +8,7 @@ from passlib.context import CryptContext
|
||||
from fastapi import HTTPException, status, Depends
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .config import get_settings
|
||||
|
||||
@@ -82,3 +83,24 @@ async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(s
|
||||
return username
|
||||
except JWTError:
|
||||
raise credentials_exception
|
||||
|
||||
|
||||
async def get_current_user_obj(current_user: str = Depends(get_current_user)):
|
||||
"""获取当前用户的完整User对象
|
||||
|
||||
用法: current_user: User = Depends(get_current_user_obj)
|
||||
替代: current_user: str = Depends(get_current_user) + 手动 db.query(User)
|
||||
"""
|
||||
from ..models.user import User
|
||||
from .database import get_db
|
||||
db = next(get_db())
|
||||
try:
|
||||
user = db.query(User).filter(User.username == current_user).first()
|
||||
if not user:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="用户不存在"
|
||||
)
|
||||
return user
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -13,7 +13,7 @@ class ChatSession(Base):
|
||||
__tablename__ = "chat_sessions"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
|
||||
user_id = Column(Integer, ForeignKey("users.id"), nullable=False, index=True)
|
||||
title = Column(String(200), nullable=True)
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -32,7 +32,7 @@ class ChatMessage(Base):
|
||||
__tablename__ = "chat_messages"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
session_id = Column(Integer, ForeignKey("chat_sessions.id"), nullable=False)
|
||||
session_id = Column(Integer, ForeignKey("chat_sessions.id"), nullable=False, index=True)
|
||||
role = Column(String(20), nullable=False) # user, assistant, system
|
||||
content = Column(Text, nullable=False)
|
||||
message_metadata = Column(Text, nullable=True) # JSON格式的元数据
|
||||
|
||||
@@ -14,7 +14,7 @@ class Document(Base):
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
user_id = Column(Integer, ForeignKey("users.id"), nullable=True) # None表示系统文档
|
||||
knowledge_base_id = Column(Integer, ForeignKey("knowledge_bases.id"), nullable=True) # 所属知识库
|
||||
knowledge_base_id = Column(Integer, ForeignKey("knowledge_bases.id"), nullable=True, index=True)
|
||||
filename = Column(String(255), nullable=False)
|
||||
original_filename = Column(String(255), nullable=False)
|
||||
file_path = Column(String(500), nullable=False)
|
||||
@@ -47,7 +47,7 @@ class DocumentChunk(Base):
|
||||
__tablename__ = "document_chunks"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
document_id = Column(Integer, ForeignKey("documents.id"), nullable=False)
|
||||
document_id = Column(Integer, ForeignKey("documents.id"), nullable=False, index=True)
|
||||
chunk_index = Column(Integer, nullable=False)
|
||||
content = Column(Text, nullable=False)
|
||||
content_hash = Column(String(64), nullable=False) # 内容哈希
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
分数转换工具
|
||||
"""
|
||||
import math
|
||||
|
||||
|
||||
def convert_distance_to_score(distance: float) -> float:
|
||||
"""将ChromaDB的distance转换为0-1范围的相似度分数"""
|
||||
if distance < 0:
|
||||
return (1 + distance) / 2
|
||||
if distance > 100:
|
||||
return 1 / (1 + math.log(distance))
|
||||
return 1 / (1 + distance)
|
||||
@@ -3,15 +3,19 @@
|
||||
"""
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional
|
||||
from sqlalchemy.orm import Session
|
||||
from langchain_core.documents import Document as LangChainDocument
|
||||
|
||||
from ..models.document import Document, DocumentChunk
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from ..rag.vector_store import get_vector_store
|
||||
from ..rag.document_loaders import DocumentLoaderFactory, PDFImageExtractor
|
||||
from ..rag.text_splitters import get_text_splitter
|
||||
from ..rag.score_utils import convert_distance_to_score
|
||||
from ..llm.siliconflow import get_llm_client
|
||||
|
||||
IMAGES_DIR = Path(__file__).parent.parent.parent.parent / "data" / "images"
|
||||
@@ -65,14 +69,14 @@ class DocumentService:
|
||||
if success:
|
||||
document.is_processed = True
|
||||
self.db.commit()
|
||||
print(f"[DocumentService] 文档 {document.filename} 处理完成: "
|
||||
logger.info(f"[DocumentService] 文档 {document.filename} 处理完成: "
|
||||
f"{len(splits)} 个文本块, {len(image_chunks)} 个图片描述块")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"处理文档失败: {str(e)}")
|
||||
logger.error(f"处理文档失败: {str(e)}")
|
||||
self.db.rollback()
|
||||
return False
|
||||
|
||||
@@ -94,56 +98,58 @@ class DocumentService:
|
||||
images = PDFImageExtractor.extract_images(str(file_path), str(img_output_dir))
|
||||
|
||||
if not images:
|
||||
print(f"[DocumentService] 未发现可提取的图片: {filename}")
|
||||
logger.info(f"[DocumentService] 未发现可提取的图片: {filename}")
|
||||
return []
|
||||
|
||||
print(f"[DocumentService] 提取到 {len(images)} 张图片, 开始VLM描述生成...")
|
||||
logger.info(f"[DocumentService] 提取到 {len(images)} 张图片, 开始VLM描述生成...")
|
||||
|
||||
# 批量调用VLM生成描述
|
||||
# 并发调用VLM生成描述(限制并发度为5)
|
||||
llm_client = get_llm_client()
|
||||
for idx, img in enumerate(images):
|
||||
try:
|
||||
description = await llm_client.describe_image(
|
||||
img["path"],
|
||||
img.get("context_text", "")
|
||||
)
|
||||
semaphore = asyncio.Semaphore(5)
|
||||
|
||||
if description:
|
||||
# 相对路径用于URL访问
|
||||
rel_path = f"{document_id}/{img['filename']}"
|
||||
image_url = f"/images/{rel_path}"
|
||||
|
||||
# 构建图片描述文本块(URL写入内容,LLM可直接引用)
|
||||
chunk_content = (
|
||||
f"[图片描述 - 第{img['page']}页]\n"
|
||||
f"图片URL: {image_url}\n"
|
||||
f"图片内容:{description}"
|
||||
async def describe_single_image(idx, img):
|
||||
async with semaphore:
|
||||
try:
|
||||
description = await llm_client.describe_image(
|
||||
img["path"],
|
||||
img.get("context_text", "")
|
||||
)
|
||||
if description:
|
||||
rel_path = f"{document_id}/{img['filename']}"
|
||||
image_url = f"/images/{rel_path}"
|
||||
chunk_content = (
|
||||
f"[图片描述 - 第{img['page']}页]\n"
|
||||
f"图片URL: {image_url}\n"
|
||||
f"图片内容:{description}"
|
||||
)
|
||||
chunk = LangChainDocument(
|
||||
page_content=chunk_content,
|
||||
metadata={
|
||||
"document_id": document_id,
|
||||
"knowledge_base_id": knowledge_base_id,
|
||||
"title": title,
|
||||
"filename": filename,
|
||||
"source_type": "image",
|
||||
"image_path": str(img["path"]),
|
||||
"image_url": image_url,
|
||||
"page": img["page"],
|
||||
}
|
||||
)
|
||||
logger.info(f"[DocumentService] 图片描述成功 {idx+1}/{len(images)}: {img['filename']}")
|
||||
return chunk
|
||||
else:
|
||||
logger.warning(f"[DocumentService] 图片描述为空 {idx+1}/{len(images)}: {img['filename']}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"[DocumentService] 图片处理失败 {img['filename']}: {e}")
|
||||
return None
|
||||
|
||||
chunk = LangChainDocument(
|
||||
page_content=chunk_content,
|
||||
metadata={
|
||||
"document_id": document_id,
|
||||
"knowledge_base_id": knowledge_base_id,
|
||||
"title": title,
|
||||
"filename": filename,
|
||||
"source_type": "image",
|
||||
"image_path": str(img["path"]),
|
||||
"image_url": image_url,
|
||||
"page": img["page"],
|
||||
}
|
||||
)
|
||||
image_chunks.append(chunk)
|
||||
print(f"[DocumentService] 图片描述成功 {idx+1}/{len(images)}: {img['filename']}")
|
||||
else:
|
||||
print(f"[DocumentService] 图片描述为空 {idx+1}/{len(images)}: {img['filename']}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[DocumentService] 图片处理失败 {img['filename']}: {e}")
|
||||
continue
|
||||
tasks = [describe_single_image(idx, img) for idx, img in enumerate(images)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
image_chunks = [r for r in results if r is not None]
|
||||
|
||||
except Exception as e:
|
||||
print(f"[DocumentService] PDF图片处理失败: {e}")
|
||||
logger.error(f"[DocumentService] PDF图片处理失败: {e}")
|
||||
|
||||
return image_chunks
|
||||
|
||||
@@ -166,7 +172,7 @@ class DocumentService:
|
||||
search_results = []
|
||||
for doc, distance in results:
|
||||
metadata = doc.metadata if hasattr(doc, 'metadata') else {}
|
||||
score = self._convert_distance_to_score(distance)
|
||||
score = convert_distance_to_score(distance)
|
||||
|
||||
search_results.append({
|
||||
"content": doc.page_content,
|
||||
@@ -178,24 +184,9 @@ class DocumentService:
|
||||
return search_results
|
||||
|
||||
except Exception as e:
|
||||
print(f"搜索文档失败: {str(e)}")
|
||||
logger.error(f"搜索文档失败: {str(e)}")
|
||||
return []
|
||||
|
||||
def _convert_distance_to_score(self, distance: float) -> float:
|
||||
"""将ChromaDB的distance转换为0-1范围的相似度分数"""
|
||||
import math
|
||||
|
||||
# 内积距离(负值)
|
||||
if distance < 0:
|
||||
return (1 + distance) / 2
|
||||
|
||||
# 大距离使用对数缩放
|
||||
if distance > 100:
|
||||
return 1 / (1 + math.log(distance))
|
||||
|
||||
# 标准距离转换
|
||||
return 1 / (1 + distance)
|
||||
|
||||
def get_document_chunks(self, document_id: int) -> List[DocumentChunk]:
|
||||
"""获取文档的所有块"""
|
||||
return self.db.query(DocumentChunk).filter(
|
||||
@@ -214,6 +205,6 @@ class DocumentService:
|
||||
self.db.commit()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"删除文档块失败: {str(e)}")
|
||||
logger.error(f"删除文档块失败: {str(e)}")
|
||||
self.db.rollback()
|
||||
return False
|
||||
+3
-3
@@ -6,7 +6,7 @@ services:
|
||||
environment:
|
||||
POSTGRES_DB: course_agent_db
|
||||
POSTGRES_USER: user
|
||||
POSTGRES_PASSWORD: password
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
|
||||
POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
@@ -35,7 +35,7 @@ services:
|
||||
- "8001:8001" # 前端应用
|
||||
environment:
|
||||
# 数据库配置
|
||||
DATABASE_URL: postgresql+psycopg://user:password@db:5432/course_agent_db
|
||||
DATABASE_URL: postgresql+psycopg://user:${POSTGRES_PASSWORD:-password}@db:5432/course_agent_db
|
||||
|
||||
# JWT配置
|
||||
SECRET_KEY: ${SECRET_KEY:-your-super-secret-key-change-in-production}
|
||||
@@ -43,7 +43,7 @@ services:
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES: 30
|
||||
|
||||
# 硅基流动API配置
|
||||
SILICONFLOW_API_KEY: ${SILICONFLOW_API_KEY:-sk-pvvtosiglncktlucwarxilvsypqcttqizgpcfdvodgcuaezn}
|
||||
SILICONFLOW_API_KEY: ${SILICONFLOW_API_KEY:?请设置 SILICONFLOW_API_KEY 环境变量}
|
||||
SILICONFLOW_BASE_URL: ${SILICONFLOW_BASE_URL:-https://api.siliconflow.cn/v1}
|
||||
SILICONFLOW_MODEL: ${SILICONFLOW_MODEL:-Qwen/Qwen3-30B-A3B-Thinking-2507}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user