feat: global exception handler, concurrent VLM, and docker security
- Add global FastAPI exception handler for unhandled errors - Add get_current_user_obj() dependency for cleaner auth patterns - Switch VLM image description from serial to asyncio.gather concurrency - Extract score conversion to shared score_utils module - Docker: use env vars for passwords, remove hardcoded API key default - Add .gitattributes and update .gitignore for tar.gz and tsbuildinfo Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,37 @@
|
|||||||
|
# Auto detect text files and perform LF normalization
|
||||||
|
* text=auto
|
||||||
|
|
||||||
|
# Source code - enforce LF line endings
|
||||||
|
*.py text eol=lf
|
||||||
|
*.ts text eol=lf
|
||||||
|
*.tsx text eol=lf
|
||||||
|
*.js text eol=lf
|
||||||
|
*.jsx text eol=lf
|
||||||
|
*.json text eol=lf
|
||||||
|
*.css text eol=lf
|
||||||
|
*.scss text eol=lf
|
||||||
|
*.html text eol=lf
|
||||||
|
*.md text eol=lf
|
||||||
|
*.yml text eol=lf
|
||||||
|
*.yaml text eol=lf
|
||||||
|
*.toml text eol=lf
|
||||||
|
*.sql text eol=lf
|
||||||
|
*.sh text eol=lf
|
||||||
|
*.bat text eol=lf
|
||||||
|
*.conf text eol=lf
|
||||||
|
*.cfg text eol=lf
|
||||||
|
*.txt text eol=lf
|
||||||
|
*.env text eol=lf
|
||||||
|
|
||||||
|
# Binary files - don't touch
|
||||||
|
*.png binary
|
||||||
|
*.jpg binary
|
||||||
|
*.jpeg binary
|
||||||
|
*.gif binary
|
||||||
|
*.ico binary
|
||||||
|
*.pdf binary
|
||||||
|
*.zip binary
|
||||||
|
*.woff binary
|
||||||
|
*.woff2 binary
|
||||||
|
*.ttf binary
|
||||||
|
*.eot binary
|
||||||
@@ -64,6 +64,7 @@ Desktop.ini
|
|||||||
# 其他
|
# 其他
|
||||||
*.tmp
|
*.tmp
|
||||||
*.temp
|
*.temp
|
||||||
|
*.tar.gz
|
||||||
.cache/
|
.cache/
|
||||||
*.coverage
|
*.coverage
|
||||||
.coverage
|
.coverage
|
||||||
|
|||||||
+35
-34
@@ -4,14 +4,17 @@ FastAPI应用入口
|
|||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
import logging
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from fastapi import FastAPI, HTTPException
|
from fastapi import FastAPI, HTTPException, Request
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
from fastapi.responses import FileResponse
|
from fastapi.responses import FileResponse, JSONResponse
|
||||||
import uvicorn
|
import uvicorn
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# 设置系统时区为北京时间
|
# 设置系统时区为北京时间
|
||||||
os.environ['TZ'] = 'Asia/Shanghai'
|
os.environ['TZ'] = 'Asia/Shanghai'
|
||||||
|
|
||||||
@@ -52,9 +55,9 @@ def startup_knowledge_base():
|
|||||||
try:
|
try:
|
||||||
from src.services.file_watcher_service import start_file_watcher
|
from src.services.file_watcher_service import start_file_watcher
|
||||||
start_file_watcher()
|
start_file_watcher()
|
||||||
print("知识库文件监控服务启动成功")
|
logger.info("知识库文件监控服务启动成功")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"启动知识库文件监控服务失败: {str(e)}")
|
logger.error(f"启动知识库文件监控服务失败: {str(e)}")
|
||||||
|
|
||||||
|
|
||||||
def seed_forum_categories():
|
def seed_forum_categories():
|
||||||
@@ -72,17 +75,17 @@ def seed_forum_categories():
|
|||||||
try:
|
try:
|
||||||
existing_count = db.query(ForumCategory).count()
|
existing_count = db.query(ForumCategory).count()
|
||||||
if existing_count > 0:
|
if existing_count > 0:
|
||||||
print(f"论坛分类已存在({existing_count} 个),跳过初始化")
|
logger.info(f"论坛分类已存在({existing_count} 个),跳过初始化")
|
||||||
return
|
return
|
||||||
|
|
||||||
for cat_data in DEFAULT_CATEGORIES:
|
for cat_data in DEFAULT_CATEGORIES:
|
||||||
category = ForumCategory(**cat_data)
|
category = ForumCategory(**cat_data)
|
||||||
db.add(category)
|
db.add(category)
|
||||||
db.commit()
|
db.commit()
|
||||||
print(f"已创建 {len(DEFAULT_CATEGORIES)} 个论坛分类")
|
logger.info(f"已创建 {len(DEFAULT_CATEGORIES)} 个论坛分类")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
print(f"初始化论坛分类失败: {e}")
|
logger.error(f"初始化论坛分类失败: {e}")
|
||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
@@ -94,10 +97,9 @@ async def startup_event():
|
|||||||
import asyncio
|
import asyncio
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
print("=" * 50, file=sys.stderr)
|
logger.info("=" * 50)
|
||||||
print("应用启动事件开始", file=sys.stderr)
|
logger.info("应用启动事件开始")
|
||||||
print("=" * 50, file=sys.stderr)
|
logger.info("=" * 50)
|
||||||
sys.stderr.flush()
|
|
||||||
|
|
||||||
# 等待数据库可用(重试机制,处理 DNS 解析延迟)
|
# 等待数据库可用(重试机制,处理 DNS 解析延迟)
|
||||||
max_retries = 10
|
max_retries = 10
|
||||||
@@ -105,28 +107,23 @@ async def startup_event():
|
|||||||
|
|
||||||
for attempt in range(max_retries):
|
for attempt in range(max_retries):
|
||||||
try:
|
try:
|
||||||
print(f"尝试连接数据库(尝试 {attempt + 1}/{max_retries})...", file=sys.stderr)
|
logger.info(f"尝试连接数据库(尝试 {attempt + 1}/{max_retries})...")
|
||||||
sys.stderr.flush()
|
|
||||||
# 尝试创建数据库表
|
# 尝试创建数据库表
|
||||||
create_tables()
|
create_tables()
|
||||||
print("数据库表创建成功", file=sys.stderr)
|
logger.info("数据库表创建成功")
|
||||||
sys.stderr.flush()
|
|
||||||
break
|
break
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
if attempt < max_retries - 1:
|
if attempt < max_retries - 1:
|
||||||
print(f"数据库连接失败(尝试 {attempt + 1}/{max_retries}): {e}", file=sys.stderr)
|
logger.warning(f"数据库连接失败(尝试 {attempt + 1}/{max_retries}): {e}")
|
||||||
print(f"等待 {retry_delay} 秒后重试...", file=sys.stderr)
|
|
||||||
sys.stderr.flush()
|
|
||||||
await asyncio.sleep(retry_delay)
|
await asyncio.sleep(retry_delay)
|
||||||
else:
|
else:
|
||||||
print(f"数据库连接失败,已达到最大重试次数: {e}", file=sys.stderr)
|
logger.error(f"数据库连接失败,已达到最大重试次数: {e}")
|
||||||
sys.stderr.flush()
|
|
||||||
# 不抛出异常,让应用继续启动,但数据库操作会失败
|
# 不抛出异常,让应用继续启动,但数据库操作会失败
|
||||||
|
|
||||||
try:
|
try:
|
||||||
startup_knowledge_base()
|
startup_knowledge_base()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"启动知识库服务失败: {e}", file=sys.stderr)
|
logger.error(f"启动知识库服务失败: {e}")
|
||||||
|
|
||||||
# 确保系统知识库与目录同步
|
# 确保系统知识库与目录同步
|
||||||
try:
|
try:
|
||||||
@@ -137,17 +134,15 @@ async def startup_event():
|
|||||||
kb_service.ensure_system_knowledge_bases()
|
kb_service.ensure_system_knowledge_bases()
|
||||||
db.close()
|
db.close()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"同步系统知识库失败: {e}", file=sys.stderr)
|
logger.error(f"同步系统知识库失败: {e}")
|
||||||
|
|
||||||
# 初始化论坛分类
|
# 初始化论坛分类
|
||||||
try:
|
try:
|
||||||
seed_forum_categories()
|
seed_forum_categories()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"初始化论坛分类失败: {e}", file=sys.stderr)
|
logger.error(f"初始化论坛分类失败: {e}")
|
||||||
sys.stderr.flush()
|
|
||||||
|
logger.info("应用启动事件完成")
|
||||||
print("应用启动事件完成", file=sys.stderr)
|
|
||||||
sys.stderr.flush()
|
|
||||||
|
|
||||||
# 应用关闭事件
|
# 应用关闭事件
|
||||||
@app.on_event("shutdown")
|
@app.on_event("shutdown")
|
||||||
@@ -156,9 +151,18 @@ async def shutdown_event():
|
|||||||
try:
|
try:
|
||||||
from src.services.file_watcher_service import stop_file_watcher
|
from src.services.file_watcher_service import stop_file_watcher
|
||||||
stop_file_watcher()
|
stop_file_watcher()
|
||||||
print("知识库文件监控服务已停止")
|
logger.info("知识库文件监控服务已停止")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"停止知识库文件监控服务失败: {str(e)}")
|
logger.error(f"停止知识库文件监控服务失败: {str(e)}")
|
||||||
|
|
||||||
|
# 全局异常处理器
|
||||||
|
@app.exception_handler(Exception)
|
||||||
|
async def generic_exception_handler(request: Request, exc: Exception):
|
||||||
|
logger.error(f"未处理的异常: {exc}", exc_info=True)
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=500,
|
||||||
|
content={"detail": "服务器内部错误,请稍后重试"}
|
||||||
|
)
|
||||||
|
|
||||||
# 注册路由
|
# 注册路由
|
||||||
app.include_router(auth.router)
|
app.include_router(auth.router)
|
||||||
@@ -315,11 +319,8 @@ async def list_services():
|
|||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
import sys
|
import sys
|
||||||
print("=" * 50, file=sys.stderr)
|
logger.info("启动 Uvicorn 服务器...")
|
||||||
print("启动 Uvicorn 服务器...", file=sys.stderr)
|
logger.info(f"Host: {settings.host}, Port: {settings.port}")
|
||||||
print(f"Host: {settings.host}, Port: {settings.port}", file=sys.stderr)
|
|
||||||
print("=" * 50, file=sys.stderr)
|
|
||||||
sys.stderr.flush()
|
|
||||||
|
|
||||||
uvicorn.run(
|
uvicorn.run(
|
||||||
"main:app",
|
"main:app",
|
||||||
|
|||||||
+11
-10
@@ -1,6 +1,7 @@
|
|||||||
"""
|
"""
|
||||||
文档管理API
|
文档管理API
|
||||||
"""
|
"""
|
||||||
|
import logging
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
@@ -15,6 +16,8 @@ from ..core.security import get_current_user
|
|||||||
from ..models.document import Document, DocumentChunk
|
from ..models.document import Document, DocumentChunk
|
||||||
from ..services.document_service import DocumentService
|
from ..services.document_service import DocumentService
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
router = APIRouter(prefix="/documents", tags=["文档管理"])
|
router = APIRouter(prefix="/documents", tags=["文档管理"])
|
||||||
|
|
||||||
|
|
||||||
@@ -186,32 +189,30 @@ async def delete_document(
|
|||||||
|
|
||||||
# 1. 先删除向量数据和文档块
|
# 1. 先删除向量数据和文档块
|
||||||
try:
|
try:
|
||||||
print(f"开始删除文档向量数据: {document.filename} (ID: {document.id})")
|
logger.info(f"开始删除文档向量数据: {document.filename} (ID: {document.id})")
|
||||||
document_service = DocumentService(db)
|
document_service = DocumentService(db)
|
||||||
vector_deleted = document_service.delete_document_chunks(document.id)
|
vector_deleted = document_service.delete_document_chunks(document.id)
|
||||||
if vector_deleted:
|
if vector_deleted:
|
||||||
print(f"成功删除文档向量数据: {document.filename}")
|
logger.info(f"成功删除文档向量数据: {document.filename}")
|
||||||
else:
|
else:
|
||||||
print(f"警告:删除文档向量数据失败: {document.filename}")
|
logger.warning(f"警告:删除文档向量数据失败: {document.filename}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"删除向量数据时发生错误: {str(e)}")
|
logger.error(f"删除向量数据时发生错误: {str(e)}", exc_info=True)
|
||||||
import traceback
|
|
||||||
print(f"详细错误信息: {traceback.format_exc()}")
|
|
||||||
|
|
||||||
# 2. 删除物理文件
|
# 2. 删除物理文件
|
||||||
try:
|
try:
|
||||||
if os.path.exists(document.file_path):
|
if os.path.exists(document.file_path):
|
||||||
os.remove(document.file_path)
|
os.remove(document.file_path)
|
||||||
print(f"成功删除物理文件: {document.file_path}")
|
logger.info(f"成功删除物理文件: {document.file_path}")
|
||||||
else:
|
else:
|
||||||
print(f"物理文件不存在: {document.file_path}")
|
logger.info(f"物理文件不存在: {document.file_path}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"删除物理文件时发生错误: {str(e)}")
|
logger.error(f"删除物理文件时发生错误: {str(e)}")
|
||||||
|
|
||||||
# 3. 删除数据库记录
|
# 3. 删除数据库记录
|
||||||
db.delete(document)
|
db.delete(document)
|
||||||
db.commit()
|
db.commit()
|
||||||
print(f"成功删除文档数据库记录: {document.filename}")
|
logger.info(f"成功删除文档数据库记录: {document.filename}")
|
||||||
|
|
||||||
return {"message": "文档删除成功"}
|
return {"message": "文档删除成功"}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
"""
|
"""
|
||||||
知识库CRUD API
|
知识库CRUD API
|
||||||
"""
|
"""
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -18,6 +19,8 @@ from ..models.document import Document
|
|||||||
from ..models.user import User
|
from ..models.user import User
|
||||||
from ..services.document_service import DocumentService
|
from ..services.document_service import DocumentService
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
router = APIRouter(prefix="/knowledge-bases", tags=["知识库管理"])
|
router = APIRouter(prefix="/knowledge-bases", tags=["知识库管理"])
|
||||||
|
|
||||||
|
|
||||||
@@ -64,7 +67,7 @@ async def get_knowledge_bases(
|
|||||||
db: Session = Depends(get_db)
|
db: Session = Depends(get_db)
|
||||||
):
|
):
|
||||||
"""获取用户的所有知识库"""
|
"""获取用户的所有知识库"""
|
||||||
print(f"DEBUG: get_knowledge_bases called for user: {current_user}")
|
logger.debug(f"get_knowledge_bases called for user: {current_user}")
|
||||||
try:
|
try:
|
||||||
# 获取用户ID
|
# 获取用户ID
|
||||||
user = db.query(User).filter(User.username == current_user).first()
|
user = db.query(User).filter(User.username == current_user).first()
|
||||||
@@ -118,7 +121,7 @@ async def create_knowledge_base(
|
|||||||
db: Session = Depends(get_db)
|
db: Session = Depends(get_db)
|
||||||
):
|
):
|
||||||
"""创建新知识库"""
|
"""创建新知识库"""
|
||||||
print(f"DEBUG: create_knowledge_base called for user: {current_user}, data: {data}")
|
logger.debug(f"create_knowledge_base called for user: {current_user}, data: {data}")
|
||||||
try:
|
try:
|
||||||
# 获取用户ID
|
# 获取用户ID
|
||||||
user = db.query(User).filter(User.username == current_user).first()
|
user = db.query(User).filter(User.username == current_user).first()
|
||||||
@@ -368,7 +371,7 @@ async def delete_knowledge_base(
|
|||||||
Document.knowledge_base_id == knowledge_base_id
|
Document.knowledge_base_id == knowledge_base_id
|
||||||
).all()
|
).all()
|
||||||
|
|
||||||
print(f"开始删除知识库 '{knowledge_base.name}',包含 {len(documents)} 个文档")
|
logger.info(f"开始删除知识库 '{knowledge_base.name}',包含 {len(documents)} 个文档")
|
||||||
|
|
||||||
# 2. 逐个删除文档的向量数据和物理文件
|
# 2. 逐个删除文档的向量数据和物理文件
|
||||||
document_service = DocumentService(db)
|
document_service = DocumentService(db)
|
||||||
@@ -380,33 +383,31 @@ async def delete_knowledge_base(
|
|||||||
# 删除向量数据
|
# 删除向量数据
|
||||||
vector_deleted = document_service.delete_document_chunks(document.id)
|
vector_deleted = document_service.delete_document_chunks(document.id)
|
||||||
if vector_deleted:
|
if vector_deleted:
|
||||||
print(f"成功删除文档向量数据: {document.filename}")
|
logger.info(f"成功删除文档向量数据: {document.filename}")
|
||||||
else:
|
else:
|
||||||
print(f"警告:删除文档向量数据失败: {document.filename}")
|
logger.warning(f"警告:删除文档向量数据失败: {document.filename}")
|
||||||
error_count += 1
|
error_count += 1
|
||||||
|
|
||||||
# 删除物理文件
|
# 删除物理文件
|
||||||
if os.path.exists(document.file_path):
|
if os.path.exists(document.file_path):
|
||||||
os.remove(document.file_path)
|
os.remove(document.file_path)
|
||||||
print(f"成功删除物理文件: {document.file_path}")
|
logger.info(f"成功删除物理文件: {document.file_path}")
|
||||||
else:
|
else:
|
||||||
print(f"物理文件不存在: {document.file_path}")
|
logger.info(f"物理文件不存在: {document.file_path}")
|
||||||
|
|
||||||
success_count += 1
|
success_count += 1
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"删除文档 {document.filename} 的资源时出错: {str(e)}")
|
logger.error(f"删除文档 {document.filename} 的资源时出错: {str(e)}", exc_info=True)
|
||||||
import traceback
|
|
||||||
print(f"详细错误信息: {traceback.format_exc()}")
|
|
||||||
error_count += 1
|
error_count += 1
|
||||||
# 继续处理其他文档
|
# 继续处理其他文档
|
||||||
|
|
||||||
print(f"文档清理完成: 成功 {success_count} 个,失败 {error_count} 个")
|
logger.info(f"文档清理完成: 成功 {success_count} 个,失败 {error_count} 个")
|
||||||
|
|
||||||
# 3. 删除知识库(级联删除文档记录)
|
# 3. 删除知识库(级联删除文档记录)
|
||||||
db.delete(knowledge_base)
|
db.delete(knowledge_base)
|
||||||
db.commit()
|
db.commit()
|
||||||
print(f"成功删除知识库数据库记录: {knowledge_base.name}")
|
logger.info(f"成功删除知识库数据库记录: {knowledge_base.name}")
|
||||||
|
|
||||||
return {"message": "知识库删除成功"}
|
return {"message": "知识库删除成功"}
|
||||||
|
|
||||||
@@ -528,20 +529,18 @@ async def upload_document_to_knowledge_base(
|
|||||||
|
|
||||||
# 自动处理文档向量化
|
# 自动处理文档向量化
|
||||||
try:
|
try:
|
||||||
print(f"开始处理文档向量化: {document.filename} (ID: {document.id})")
|
logger.info(f"开始处理文档向量化: {document.filename} (ID: {document.id})")
|
||||||
document_service = DocumentService(db)
|
document_service = DocumentService(db)
|
||||||
success = await document_service.process_document(document.id)
|
success = await document_service.process_document(document.id)
|
||||||
|
|
||||||
if success:
|
if success:
|
||||||
print(f"文档向量化处理成功: {document.filename}")
|
logger.info(f"文档向量化处理成功: {document.filename}")
|
||||||
message = "文档上传并处理成功"
|
message = "文档上传并处理成功"
|
||||||
else:
|
else:
|
||||||
print(f"文档向量化处理失败: {document.filename}")
|
logger.warning(f"文档向量化处理失败: {document.filename}")
|
||||||
message = "文档上传成功,但向量化处理失败"
|
message = "文档上传成功,但向量化处理失败"
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"文档向量化处理异常: {document.filename}, 错误: {str(e)}")
|
logger.error(f"文档向量化处理异常: {document.filename}, 错误: {str(e)}", exc_info=True)
|
||||||
import traceback
|
|
||||||
print(f"详细错误信息: {traceback.format_exc()}")
|
|
||||||
message = "文档上传成功,但向量化处理失败"
|
message = "文档上传成功,但向量化处理失败"
|
||||||
|
|
||||||
return DocumentUploadResponse(
|
return DocumentUploadResponse(
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from passlib.context import CryptContext
|
|||||||
from fastapi import HTTPException, status, Depends
|
from fastapi import HTTPException, status, Depends
|
||||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from .config import get_settings
|
from .config import get_settings
|
||||||
|
|
||||||
@@ -82,3 +83,24 @@ async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(s
|
|||||||
return username
|
return username
|
||||||
except JWTError:
|
except JWTError:
|
||||||
raise credentials_exception
|
raise credentials_exception
|
||||||
|
|
||||||
|
|
||||||
|
async def get_current_user_obj(current_user: str = Depends(get_current_user)):
|
||||||
|
"""获取当前用户的完整User对象
|
||||||
|
|
||||||
|
用法: current_user: User = Depends(get_current_user_obj)
|
||||||
|
替代: current_user: str = Depends(get_current_user) + 手动 db.query(User)
|
||||||
|
"""
|
||||||
|
from ..models.user import User
|
||||||
|
from .database import get_db
|
||||||
|
db = next(get_db())
|
||||||
|
try:
|
||||||
|
user = db.query(User).filter(User.username == current_user).first()
|
||||||
|
if not user:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail="用户不存在"
|
||||||
|
)
|
||||||
|
return user
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ class ChatSession(Base):
|
|||||||
__tablename__ = "chat_sessions"
|
__tablename__ = "chat_sessions"
|
||||||
|
|
||||||
id = Column(Integer, primary_key=True, index=True)
|
id = Column(Integer, primary_key=True, index=True)
|
||||||
user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
|
user_id = Column(Integer, ForeignKey("users.id"), nullable=False, index=True)
|
||||||
title = Column(String(200), nullable=True)
|
title = Column(String(200), nullable=True)
|
||||||
is_active = Column(Boolean, default=True)
|
is_active = Column(Boolean, default=True)
|
||||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||||
@@ -32,7 +32,7 @@ class ChatMessage(Base):
|
|||||||
__tablename__ = "chat_messages"
|
__tablename__ = "chat_messages"
|
||||||
|
|
||||||
id = Column(Integer, primary_key=True, index=True)
|
id = Column(Integer, primary_key=True, index=True)
|
||||||
session_id = Column(Integer, ForeignKey("chat_sessions.id"), nullable=False)
|
session_id = Column(Integer, ForeignKey("chat_sessions.id"), nullable=False, index=True)
|
||||||
role = Column(String(20), nullable=False) # user, assistant, system
|
role = Column(String(20), nullable=False) # user, assistant, system
|
||||||
content = Column(Text, nullable=False)
|
content = Column(Text, nullable=False)
|
||||||
message_metadata = Column(Text, nullable=True) # JSON格式的元数据
|
message_metadata = Column(Text, nullable=True) # JSON格式的元数据
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ class Document(Base):
|
|||||||
|
|
||||||
id = Column(Integer, primary_key=True, index=True)
|
id = Column(Integer, primary_key=True, index=True)
|
||||||
user_id = Column(Integer, ForeignKey("users.id"), nullable=True) # None表示系统文档
|
user_id = Column(Integer, ForeignKey("users.id"), nullable=True) # None表示系统文档
|
||||||
knowledge_base_id = Column(Integer, ForeignKey("knowledge_bases.id"), nullable=True) # 所属知识库
|
knowledge_base_id = Column(Integer, ForeignKey("knowledge_bases.id"), nullable=True, index=True)
|
||||||
filename = Column(String(255), nullable=False)
|
filename = Column(String(255), nullable=False)
|
||||||
original_filename = Column(String(255), nullable=False)
|
original_filename = Column(String(255), nullable=False)
|
||||||
file_path = Column(String(500), nullable=False)
|
file_path = Column(String(500), nullable=False)
|
||||||
@@ -47,7 +47,7 @@ class DocumentChunk(Base):
|
|||||||
__tablename__ = "document_chunks"
|
__tablename__ = "document_chunks"
|
||||||
|
|
||||||
id = Column(Integer, primary_key=True, index=True)
|
id = Column(Integer, primary_key=True, index=True)
|
||||||
document_id = Column(Integer, ForeignKey("documents.id"), nullable=False)
|
document_id = Column(Integer, ForeignKey("documents.id"), nullable=False, index=True)
|
||||||
chunk_index = Column(Integer, nullable=False)
|
chunk_index = Column(Integer, nullable=False)
|
||||||
content = Column(Text, nullable=False)
|
content = Column(Text, nullable=False)
|
||||||
content_hash = Column(String(64), nullable=False) # 内容哈希
|
content_hash = Column(String(64), nullable=False) # 内容哈希
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
"""
|
||||||
|
分数转换工具
|
||||||
|
"""
|
||||||
|
import math
|
||||||
|
|
||||||
|
|
||||||
|
def convert_distance_to_score(distance: float) -> float:
|
||||||
|
"""将ChromaDB的distance转换为0-1范围的相似度分数"""
|
||||||
|
if distance < 0:
|
||||||
|
return (1 + distance) / 2
|
||||||
|
if distance > 100:
|
||||||
|
return 1 / (1 + math.log(distance))
|
||||||
|
return 1 / (1 + distance)
|
||||||
@@ -3,15 +3,19 @@
|
|||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List, Dict, Any, Optional
|
from typing import List, Dict, Any, Optional
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
from langchain_core.documents import Document as LangChainDocument
|
from langchain_core.documents import Document as LangChainDocument
|
||||||
|
|
||||||
from ..models.document import Document, DocumentChunk
|
from ..models.document import Document, DocumentChunk
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
from ..rag.vector_store import get_vector_store
|
from ..rag.vector_store import get_vector_store
|
||||||
from ..rag.document_loaders import DocumentLoaderFactory, PDFImageExtractor
|
from ..rag.document_loaders import DocumentLoaderFactory, PDFImageExtractor
|
||||||
from ..rag.text_splitters import get_text_splitter
|
from ..rag.text_splitters import get_text_splitter
|
||||||
|
from ..rag.score_utils import convert_distance_to_score
|
||||||
from ..llm.siliconflow import get_llm_client
|
from ..llm.siliconflow import get_llm_client
|
||||||
|
|
||||||
IMAGES_DIR = Path(__file__).parent.parent.parent.parent / "data" / "images"
|
IMAGES_DIR = Path(__file__).parent.parent.parent.parent / "data" / "images"
|
||||||
@@ -65,14 +69,14 @@ class DocumentService:
|
|||||||
if success:
|
if success:
|
||||||
document.is_processed = True
|
document.is_processed = True
|
||||||
self.db.commit()
|
self.db.commit()
|
||||||
print(f"[DocumentService] 文档 {document.filename} 处理完成: "
|
logger.info(f"[DocumentService] 文档 {document.filename} 处理完成: "
|
||||||
f"{len(splits)} 个文本块, {len(image_chunks)} 个图片描述块")
|
f"{len(splits)} 个文本块, {len(image_chunks)} 个图片描述块")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"处理文档失败: {str(e)}")
|
logger.error(f"处理文档失败: {str(e)}")
|
||||||
self.db.rollback()
|
self.db.rollback()
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -94,56 +98,58 @@ class DocumentService:
|
|||||||
images = PDFImageExtractor.extract_images(str(file_path), str(img_output_dir))
|
images = PDFImageExtractor.extract_images(str(file_path), str(img_output_dir))
|
||||||
|
|
||||||
if not images:
|
if not images:
|
||||||
print(f"[DocumentService] 未发现可提取的图片: {filename}")
|
logger.info(f"[DocumentService] 未发现可提取的图片: {filename}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
print(f"[DocumentService] 提取到 {len(images)} 张图片, 开始VLM描述生成...")
|
logger.info(f"[DocumentService] 提取到 {len(images)} 张图片, 开始VLM描述生成...")
|
||||||
|
|
||||||
# 批量调用VLM生成描述
|
# 并发调用VLM生成描述(限制并发度为5)
|
||||||
llm_client = get_llm_client()
|
llm_client = get_llm_client()
|
||||||
for idx, img in enumerate(images):
|
semaphore = asyncio.Semaphore(5)
|
||||||
try:
|
|
||||||
description = await llm_client.describe_image(
|
|
||||||
img["path"],
|
|
||||||
img.get("context_text", "")
|
|
||||||
)
|
|
||||||
|
|
||||||
if description:
|
async def describe_single_image(idx, img):
|
||||||
# 相对路径用于URL访问
|
async with semaphore:
|
||||||
rel_path = f"{document_id}/{img['filename']}"
|
try:
|
||||||
image_url = f"/images/{rel_path}"
|
description = await llm_client.describe_image(
|
||||||
|
img["path"],
|
||||||
# 构建图片描述文本块(URL写入内容,LLM可直接引用)
|
img.get("context_text", "")
|
||||||
chunk_content = (
|
|
||||||
f"[图片描述 - 第{img['page']}页]\n"
|
|
||||||
f"图片URL: {image_url}\n"
|
|
||||||
f"图片内容:{description}"
|
|
||||||
)
|
)
|
||||||
|
if description:
|
||||||
|
rel_path = f"{document_id}/{img['filename']}"
|
||||||
|
image_url = f"/images/{rel_path}"
|
||||||
|
chunk_content = (
|
||||||
|
f"[图片描述 - 第{img['page']}页]\n"
|
||||||
|
f"图片URL: {image_url}\n"
|
||||||
|
f"图片内容:{description}"
|
||||||
|
)
|
||||||
|
chunk = LangChainDocument(
|
||||||
|
page_content=chunk_content,
|
||||||
|
metadata={
|
||||||
|
"document_id": document_id,
|
||||||
|
"knowledge_base_id": knowledge_base_id,
|
||||||
|
"title": title,
|
||||||
|
"filename": filename,
|
||||||
|
"source_type": "image",
|
||||||
|
"image_path": str(img["path"]),
|
||||||
|
"image_url": image_url,
|
||||||
|
"page": img["page"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
logger.info(f"[DocumentService] 图片描述成功 {idx+1}/{len(images)}: {img['filename']}")
|
||||||
|
return chunk
|
||||||
|
else:
|
||||||
|
logger.warning(f"[DocumentService] 图片描述为空 {idx+1}/{len(images)}: {img['filename']}")
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"[DocumentService] 图片处理失败 {img['filename']}: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
chunk = LangChainDocument(
|
tasks = [describe_single_image(idx, img) for idx, img in enumerate(images)]
|
||||||
page_content=chunk_content,
|
results = await asyncio.gather(*tasks)
|
||||||
metadata={
|
image_chunks = [r for r in results if r is not None]
|
||||||
"document_id": document_id,
|
|
||||||
"knowledge_base_id": knowledge_base_id,
|
|
||||||
"title": title,
|
|
||||||
"filename": filename,
|
|
||||||
"source_type": "image",
|
|
||||||
"image_path": str(img["path"]),
|
|
||||||
"image_url": image_url,
|
|
||||||
"page": img["page"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
image_chunks.append(chunk)
|
|
||||||
print(f"[DocumentService] 图片描述成功 {idx+1}/{len(images)}: {img['filename']}")
|
|
||||||
else:
|
|
||||||
print(f"[DocumentService] 图片描述为空 {idx+1}/{len(images)}: {img['filename']}")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"[DocumentService] 图片处理失败 {img['filename']}: {e}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[DocumentService] PDF图片处理失败: {e}")
|
logger.error(f"[DocumentService] PDF图片处理失败: {e}")
|
||||||
|
|
||||||
return image_chunks
|
return image_chunks
|
||||||
|
|
||||||
@@ -166,7 +172,7 @@ class DocumentService:
|
|||||||
search_results = []
|
search_results = []
|
||||||
for doc, distance in results:
|
for doc, distance in results:
|
||||||
metadata = doc.metadata if hasattr(doc, 'metadata') else {}
|
metadata = doc.metadata if hasattr(doc, 'metadata') else {}
|
||||||
score = self._convert_distance_to_score(distance)
|
score = convert_distance_to_score(distance)
|
||||||
|
|
||||||
search_results.append({
|
search_results.append({
|
||||||
"content": doc.page_content,
|
"content": doc.page_content,
|
||||||
@@ -178,24 +184,9 @@ class DocumentService:
|
|||||||
return search_results
|
return search_results
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"搜索文档失败: {str(e)}")
|
logger.error(f"搜索文档失败: {str(e)}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def _convert_distance_to_score(self, distance: float) -> float:
|
|
||||||
"""将ChromaDB的distance转换为0-1范围的相似度分数"""
|
|
||||||
import math
|
|
||||||
|
|
||||||
# 内积距离(负值)
|
|
||||||
if distance < 0:
|
|
||||||
return (1 + distance) / 2
|
|
||||||
|
|
||||||
# 大距离使用对数缩放
|
|
||||||
if distance > 100:
|
|
||||||
return 1 / (1 + math.log(distance))
|
|
||||||
|
|
||||||
# 标准距离转换
|
|
||||||
return 1 / (1 + distance)
|
|
||||||
|
|
||||||
def get_document_chunks(self, document_id: int) -> List[DocumentChunk]:
|
def get_document_chunks(self, document_id: int) -> List[DocumentChunk]:
|
||||||
"""获取文档的所有块"""
|
"""获取文档的所有块"""
|
||||||
return self.db.query(DocumentChunk).filter(
|
return self.db.query(DocumentChunk).filter(
|
||||||
@@ -214,6 +205,6 @@ class DocumentService:
|
|||||||
self.db.commit()
|
self.db.commit()
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"删除文档块失败: {str(e)}")
|
logger.error(f"删除文档块失败: {str(e)}")
|
||||||
self.db.rollback()
|
self.db.rollback()
|
||||||
return False
|
return False
|
||||||
+3
-3
@@ -6,7 +6,7 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
POSTGRES_DB: course_agent_db
|
POSTGRES_DB: course_agent_db
|
||||||
POSTGRES_USER: user
|
POSTGRES_USER: user
|
||||||
POSTGRES_PASSWORD: password
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
|
||||||
POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C"
|
POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C"
|
||||||
volumes:
|
volumes:
|
||||||
- postgres_data:/var/lib/postgresql/data
|
- postgres_data:/var/lib/postgresql/data
|
||||||
@@ -35,7 +35,7 @@ services:
|
|||||||
- "8001:8001" # 前端应用
|
- "8001:8001" # 前端应用
|
||||||
environment:
|
environment:
|
||||||
# 数据库配置
|
# 数据库配置
|
||||||
DATABASE_URL: postgresql+psycopg://user:password@db:5432/course_agent_db
|
DATABASE_URL: postgresql+psycopg://user:${POSTGRES_PASSWORD:-password}@db:5432/course_agent_db
|
||||||
|
|
||||||
# JWT配置
|
# JWT配置
|
||||||
SECRET_KEY: ${SECRET_KEY:-your-super-secret-key-change-in-production}
|
SECRET_KEY: ${SECRET_KEY:-your-super-secret-key-change-in-production}
|
||||||
@@ -43,7 +43,7 @@ services:
|
|||||||
ACCESS_TOKEN_EXPIRE_MINUTES: 30
|
ACCESS_TOKEN_EXPIRE_MINUTES: 30
|
||||||
|
|
||||||
# 硅基流动API配置
|
# 硅基流动API配置
|
||||||
SILICONFLOW_API_KEY: ${SILICONFLOW_API_KEY:-sk-pvvtosiglncktlucwarxilvsypqcttqizgpcfdvodgcuaezn}
|
SILICONFLOW_API_KEY: ${SILICONFLOW_API_KEY:?请设置 SILICONFLOW_API_KEY 环境变量}
|
||||||
SILICONFLOW_BASE_URL: ${SILICONFLOW_BASE_URL:-https://api.siliconflow.cn/v1}
|
SILICONFLOW_BASE_URL: ${SILICONFLOW_BASE_URL:-https://api.siliconflow.cn/v1}
|
||||||
SILICONFLOW_MODEL: ${SILICONFLOW_MODEL:-Qwen/Qwen3-30B-A3B-Thinking-2507}
|
SILICONFLOW_MODEL: ${SILICONFLOW_MODEL:-Qwen/Qwen3-30B-A3B-Thinking-2507}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user