feat: multimodal RAG with PDF image extraction and display
Extract images from PDFs using pymupdf, generate descriptions via Qwen3-VL-8B, store in ChromaDB alongside text chunks, and render images in chat answers. Includes image proxy rewrite, force re-process endpoint, and VLM API timeout. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
+17
-11
@@ -227,10 +227,11 @@ async def delete_document(
|
||||
@router.post("/{document_id}/process")
|
||||
async def process_document(
|
||||
document_id: int,
|
||||
force: bool = False,
|
||||
current_user: str = Depends(get_current_user),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""处理文档（向量化）"""
|
||||
"""处理文档（向量化），force=true 强制重新处理"""
|
||||
try:
|
||||
# 获取用户ID
|
||||
from ..models.user import User
|
||||
@@ -240,26 +241,31 @@ async def process_document(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="用户不存在"
|
||||
)
|
||||
|
||||
|
||||
# 获取文档
|
||||
document = db.query(Document).filter(
|
||||
Document.id == document_id,
|
||||
Document.user_id == user.id
|
||||
).first()
|
||||
|
||||
|
||||
if not document:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="文档不存在"
|
||||
)
|
||||
|
||||
if document.is_processed:
|
||||
return {"message": "文档已经处理过了"}
|
||||
|
||||
# 处理文档
|
||||
|
||||
if document.is_processed and not force:
|
||||
return {"message": "文档已经处理过了，使用 force=true 强制重新处理"}
|
||||
|
||||
# 强制重新处理时，先删除已有的向量数据
|
||||
if force and document.is_processed:
|
||||
document_service = DocumentService(db)
|
||||
document_service.delete_document_chunks(document.id)
|
||||
|
||||
# 处理文档（直接 await 异步方法）
|
||||
document_service = DocumentService(db)
|
||||
success = await asyncio.to_thread(document_service.process_document, document.id)
|
||||
|
||||
success = await document_service.process_document(document.id)
|
||||
|
||||
if success:
|
||||
return {"message": "文档处理成功"}
|
||||
else:
|
||||
@@ -267,7 +273,7 @@ async def process_document(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="文档处理失败"
|
||||
)
|
||||
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""
|
||||
大模型API集成 — 支持 SiliconFlow 和 DeepSeek 官方
|
||||
大模型API集成 — 支持 SiliconFlow 和 DeepSeek 官方 + 视觉模型
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
from typing import List, Dict, Any, Optional, AsyncGenerator, Tuple
|
||||
from langchain_openai import ChatOpenAI
|
||||
from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, SystemMessage
|
||||
@@ -13,6 +14,17 @@ from ..core.config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
# 图片描述提示词
|
||||
IMAGE_DESCRIPTION_PROMPT = """你是一个国土空间规划专家。请详细描述这张PDF文档中的图片内容。
|
||||
|
||||
图片周围文字上下文（来自PDF页面）：{context_text}
|
||||
|
||||
要求：
|
||||
1. 说明图片类型（地图/规划图/图表/流程图/示意图/照片等）
|
||||
2. 描述图片中的关键信息、数据和空间关系
|
||||
3. 提取图中所有文字标注
|
||||
4. 描述控制在200-300字"""
|
||||
|
||||
# DeepSeek 官方模型 ID 前缀（用于自动路由）
|
||||
DEEPSEEK_OFFICIAL_MODELS = {
|
||||
"deepseek-chat",
|
||||
@@ -150,6 +162,61 @@ class SiliconFlowLLM:
|
||||
return messages
|
||||
|
||||
|
||||
async def describe_image(self, image_path: str, context_text: str = "") -> str:
|
||||
"""使用VLM模型描述图片内容
|
||||
|
||||
Args:
|
||||
image_path: 图片文件路径
|
||||
context_text: 图片周围的PDF文本上下文
|
||||
|
||||
Returns:
|
||||
图片的文字描述
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
# 读取图片并编码为base64
|
||||
with open(image_path, "rb") as f:
|
||||
image_data = base64.b64encode(f.read()).decode("utf-8")
|
||||
|
||||
# 检测图片格式
|
||||
ext = os.path.splitext(image_path)[1].lower()
|
||||
mime_map = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp"}
|
||||
mime_type = mime_map.get(ext, "image/png")
|
||||
|
||||
prompt = IMAGE_DESCRIPTION_PROMPT.format(context_text=context_text[:600])
|
||||
|
||||
vision_model = "Qwen/Qwen3-VL-8B-Instruct"
|
||||
api_key, base_url, _ = _resolve_provider(vision_model)
|
||||
|
||||
client = openai.AsyncOpenAI(api_key=api_key, base_url=base_url)
|
||||
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = await client.chat.completions.create(
|
||||
model=vision_model,
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{image_data}"}},
|
||||
],
|
||||
}],
|
||||
max_tokens=600,
|
||||
temperature=0.3,
|
||||
timeout=90.0,
|
||||
)
|
||||
return response.choices[0].message.content or ""
|
||||
|
||||
except Exception as e:
|
||||
print(f"[VLM] 描述失败 attempt={attempt+1}: {e}")
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
# 全局LLM实例（使用默认模型）
|
||||
llm_client = SiliconFlowLLM()
|
||||
|
||||
|
||||
@@ -198,7 +198,9 @@ class RAGChain:
|
||||
"filename": metadata.get("filename", "未知文件"),
|
||||
"page": metadata.get("chunk_index", 0),
|
||||
"score": metadata.get("score"),
|
||||
"preview": doc.page_content
|
||||
"preview": doc.page_content,
|
||||
"source_type": metadata.get("source_type", "rag"),
|
||||
"image_url": metadata.get("image_url"),
|
||||
})
|
||||
return sources
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""
|
||||
LangChain 1.0 文档加载器封装
|
||||
LangChain 1.0 文档加载器封装 + PDF图片提取
|
||||
"""
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Dict
|
||||
from pathlib import Path
|
||||
import fitz # pymupdf
|
||||
from langchain_community.document_loaders import (
|
||||
PyPDFLoader,
|
||||
Docx2txtLoader,
|
||||
@@ -11,6 +12,60 @@ from langchain_community.document_loaders import (
|
||||
)
|
||||
from langchain_core.documents import Document
|
||||
|
||||
|
||||
class PDFImageExtractor:
|
||||
"""使用pymupdf从PDF中提取内嵌图片"""
|
||||
|
||||
@staticmethod
|
||||
def extract_images(file_path: str, output_dir: str) -> List[dict]:
|
||||
"""提取PDF中所有图片，返回图片元数据列表"""
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
images = []
|
||||
doc = fitz.open(file_path)
|
||||
|
||||
for page_num in range(len(doc)):
|
||||
page = doc[page_num]
|
||||
# 获取页面文本作为图片上下文
|
||||
page_text = page.get_text("text")
|
||||
# 提取页面内嵌图片
|
||||
image_list = page.get_images(full=True)
|
||||
|
||||
for img_idx, img_info in enumerate(image_list):
|
||||
xref = img_info[0]
|
||||
try:
|
||||
base_image = doc.extract_image(xref)
|
||||
image_bytes = base_image["image"]
|
||||
ext = base_image["ext"]
|
||||
|
||||
# 过滤太小的图片（图标、装饰元素等）
|
||||
if len(image_bytes) < 2048:
|
||||
continue
|
||||
|
||||
# 图片周围文本（取该页文字前后各300字作为上下文）
|
||||
context_start = max(0, page_text.find(
|
||||
page_text[:len(page_text)//2]) if len(page_text) > 600
|
||||
else 0)
|
||||
context_text = page_text[context_start:context_start+600].strip()
|
||||
|
||||
filename = f"page{page_num+1}_img{img_idx+1}.{ext}"
|
||||
output_path = Path(output_dir) / filename
|
||||
output_path.write_bytes(image_bytes)
|
||||
|
||||
images.append({
|
||||
"path": str(output_path),
|
||||
"filename": filename,
|
||||
"page": page_num + 1,
|
||||
"context_text": context_text,
|
||||
"size": len(image_bytes),
|
||||
})
|
||||
except Exception as e:
|
||||
print(f"[ImageExtractor] 提取图片失败 page={page_num+1} img={img_idx}: {e}")
|
||||
continue
|
||||
|
||||
doc.close()
|
||||
return images
|
||||
|
||||
|
||||
class DocumentLoaderFactory:
|
||||
"""文档加载器工厂"""
|
||||
|
||||
|
||||
@@ -13,12 +13,16 @@ RAG_SYSTEM_PROMPT = """你是一个专业的国土空间规划知识问答助手
|
||||
要求：
|
||||
1. 综合评估上下文信息的相关性，优先采用与问题最相关的内容，不强行引用不相关的来源
|
||||
2. 回答中必须使用 [来源N] 格式（N为上下文中的来源编号）标注所引用的具体来源，例如"根据[来源3]，国土空间规划..."
|
||||
3. 回答要准确、专业、详细，结构清晰，逻辑性强
|
||||
4. 如果上下文中没有相关信息，请诚实说明
|
||||
5. 在回答末尾，列出所有实际引用的参考来源，格式为：
|
||||
3. 如果上下文中包含[图片描述]内容且与问题相关，**必须**在回答中展示该图片。上下文中已有"图片URL: /images/..."字段，请直接复制该URL，使用Markdown语法引用：
|
||||
|
||||
例如：上下文中某来源包含"图片URL: /images/1/page3_img1.png"，则插入：
|
||||
|
||||
4. 回答要准确、专业、详细，结构清晰，逻辑性强
|
||||
5. 如果上下文中没有相关信息，请诚实说明
|
||||
6. 在回答末尾，列出所有实际引用的参考来源，格式为：
|
||||
**参考来源：**
|
||||
- [来源N] 文档标题
|
||||
6. 回答长度控制在500-1000字之间
|
||||
7. 回答长度控制在500-1000字之间
|
||||
|
||||
请基于上述上下文信息回答用户的问题。"""
|
||||
|
||||
|
||||
@@ -65,6 +65,17 @@ class VectorStore:
|
||||
print(f"[DEBUG-VectorStore] 返回结果数量: {len(result)}")
|
||||
return result
|
||||
|
||||
def delete_by_document_id(self, document_id: int) -> bool:
|
||||
"""删除指定文档的所有向量数据"""
|
||||
try:
|
||||
self.vectorstore._collection.delete(
|
||||
where={"document_id": document_id}
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"删除向量数据失败: {str(e)}")
|
||||
return False
|
||||
|
||||
def max_marginal_relevance_search(
|
||||
self,
|
||||
query: str,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""
|
||||
文档处理服务（LangChain 1.0）
|
||||
文档处理服务（LangChain 1.0 + 多模态图片处理）
|
||||
"""
|
||||
import os
|
||||
import hashlib
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -10,24 +10,28 @@ from langchain_core.documents import Document as LangChainDocument
|
||||
|
||||
from ..models.document import Document, DocumentChunk
|
||||
from ..rag.vector_store import get_vector_store
|
||||
from ..rag.document_loaders import DocumentLoaderFactory
|
||||
from ..rag.document_loaders import DocumentLoaderFactory, PDFImageExtractor
|
||||
from ..rag.text_splitters import get_text_splitter
|
||||
from ..llm.siliconflow import get_llm_client
|
||||
|
||||
IMAGES_DIR = Path(__file__).parent.parent.parent.parent / "data" / "images"
|
||||
|
||||
|
||||
class DocumentService:
|
||||
"""文档处理服务"""
|
||||
|
||||
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
self.vector_store = get_vector_store()
|
||||
|
||||
|
||||
async def process_document(self, document_id: int) -> bool:
|
||||
"""处理文档（使用LangChain 1.0）"""
|
||||
"""处理文档（使用LangChain 1.0 + 多模态图片处理）"""
|
||||
try:
|
||||
document = self.db.query(Document).filter(Document.id == document_id).first()
|
||||
if not document:
|
||||
return False
|
||||
|
||||
# 1. 使用LangChain加载文档
|
||||
|
||||
# 1. 使用LangChain加载文档（文本）
|
||||
documents = DocumentLoaderFactory.load_document(
|
||||
file_path=document.file_path,
|
||||
file_type=document.file_type,
|
||||
@@ -38,25 +42,110 @@ class DocumentService:
|
||||
"filename": document.filename
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# 2. 使用中文优化的文本分割器
|
||||
text_splitter = get_text_splitter(chunk_size=1000, chunk_overlap=200)
|
||||
splits = text_splitter.split_documents(documents)
|
||||
|
||||
# 3. 添加到向量存储
|
||||
success = self.vector_store.add_documents(splits)
|
||||
|
||||
|
||||
# 3. PDF图片提取和描述（仅PDF文件）
|
||||
image_chunks = []
|
||||
if document.file_type == ".pdf":
|
||||
image_chunks = await self._process_pdf_images(
|
||||
file_path=document.file_path,
|
||||
document_id=document.id,
|
||||
knowledge_base_id=document.knowledge_base_id,
|
||||
title=document.title,
|
||||
filename=document.filename
|
||||
)
|
||||
|
||||
# 4. 将文本块和图片描述合并添加到向量存储
|
||||
all_splits = splits + image_chunks
|
||||
success = self.vector_store.add_documents(all_splits)
|
||||
|
||||
if success:
|
||||
document.is_processed = True
|
||||
self.db.commit()
|
||||
print(f"[DocumentService] 文档 {document.filename} 处理完成: "
|
||||
f"{len(splits)} 个文本块, {len(image_chunks)} 个图片描述块")
|
||||
return True
|
||||
|
||||
|
||||
return False
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(f"处理文档失败: {str(e)}")
|
||||
self.db.rollback()
|
||||
return False
|
||||
|
||||
async def _process_pdf_images(
|
||||
self,
|
||||
file_path: str,
|
||||
document_id: int,
|
||||
knowledge_base_id: int,
|
||||
title: str,
|
||||
filename: str
|
||||
) -> List[LangChainDocument]:
|
||||
"""提取PDF图片并用VLM生成描述"""
|
||||
image_chunks = []
|
||||
try:
|
||||
# 创建图片输出目录
|
||||
img_output_dir = IMAGES_DIR / str(document_id)
|
||||
|
||||
# 提取图片
|
||||
images = PDFImageExtractor.extract_images(str(file_path), str(img_output_dir))
|
||||
|
||||
if not images:
|
||||
print(f"[DocumentService] 未发现可提取的图片: {filename}")
|
||||
return []
|
||||
|
||||
print(f"[DocumentService] 提取到 {len(images)} 张图片, 开始VLM描述生成...")
|
||||
|
||||
# 批量调用VLM生成描述
|
||||
llm_client = get_llm_client()
|
||||
for idx, img in enumerate(images):
|
||||
try:
|
||||
description = await llm_client.describe_image(
|
||||
img["path"],
|
||||
img.get("context_text", "")
|
||||
)
|
||||
|
||||
if description:
|
||||
# 相对路径用于URL访问
|
||||
rel_path = f"{document_id}/{img['filename']}"
|
||||
image_url = f"/images/{rel_path}"
|
||||
|
||||
# 构建图片描述文本块（URL写入内容，LLM可直接引用）
|
||||
chunk_content = (
|
||||
f"[图片描述 - 第{img['page']}页]\n"
|
||||
f"图片URL: {image_url}\n"
|
||||
f"图片内容：{description}"
|
||||
)
|
||||
|
||||
chunk = LangChainDocument(
|
||||
page_content=chunk_content,
|
||||
metadata={
|
||||
"document_id": document_id,
|
||||
"knowledge_base_id": knowledge_base_id,
|
||||
"title": title,
|
||||
"filename": filename,
|
||||
"source_type": "image",
|
||||
"image_path": str(img["path"]),
|
||||
"image_url": image_url,
|
||||
"page": img["page"],
|
||||
}
|
||||
)
|
||||
image_chunks.append(chunk)
|
||||
print(f"[DocumentService] 图片描述成功 {idx+1}/{len(images)}: {img['filename']}")
|
||||
else:
|
||||
print(f"[DocumentService] 图片描述为空 {idx+1}/{len(images)}: {img['filename']}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[DocumentService] 图片处理失败 {img['filename']}: {e}")
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
print(f"[DocumentService] PDF图片处理失败: {e}")
|
||||
|
||||
return image_chunks
|
||||
|
||||
def search_documents(self, query: str, knowledge_base_ids: Optional[List[int]] = None, limit: int = 5) -> List[Dict[str, Any]]:
|
||||
"""搜索文档（保留原有接口兼容性）"""
|
||||
@@ -114,8 +203,11 @@ class DocumentService:
|
||||
).order_by(DocumentChunk.chunk_index).all()
|
||||
|
||||
def delete_document_chunks(self, document_id: int) -> bool:
|
||||
"""删除文档的所有块"""
|
||||
"""删除文档的所有块（数据库 + 向量存储）"""
|
||||
try:
|
||||
# 删除向量存储中的文档数据
|
||||
self.vector_store.delete_by_document_id(document_id)
|
||||
# 删除数据库中的chunk记录
|
||||
self.db.query(DocumentChunk).filter(
|
||||
DocumentChunk.document_id == document_id
|
||||
).delete()
|
||||
|
||||
Reference in New Issue
Block a user