feat: multimodal RAG with PDF image extraction and display

Extract images from PDFs using pymupdf, generate descriptions via
Qwen3-VL-8B, store in ChromaDB alongside text chunks, and render
images in chat answers. Includes image proxy rewrite, force re-process
endpoint, and VLM API timeout.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-05-28 12:14:40 +08:00
parent 3f12e96ea0
commit 4bb50ae9c1
15 changed files with 351 additions and 40 deletions
+68 -1
View File
@@ -1,7 +1,8 @@
"""
大模型API集成 — 支持 SiliconFlow 和 DeepSeek 官方
大模型API集成 — 支持 SiliconFlow 和 DeepSeek 官方 + 视觉模型
"""
import os
import base64
from typing import List, Dict, Any, Optional, AsyncGenerator, Tuple
from langchain_openai import ChatOpenAI
from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, SystemMessage
@@ -13,6 +14,17 @@ from ..core.config import get_settings
settings = get_settings()
# 图片描述提示词
IMAGE_DESCRIPTION_PROMPT = """你是一个国土空间规划专家。请详细描述这张PDF文档中的图片内容。
图片周围文字上下文(来自PDF页面):{context_text}
要求:
1. 说明图片类型(地图/规划图/图表/流程图/示意图/照片等)
2. 描述图片中的关键信息、数据和空间关系
3. 提取图中所有文字标注
4. 描述控制在200-300字"""
# DeepSeek 官方模型 ID 前缀(用于自动路由)
DEEPSEEK_OFFICIAL_MODELS = {
"deepseek-chat",
@@ -150,6 +162,61 @@ class SiliconFlowLLM:
return messages
async def describe_image(self, image_path: str, context_text: str = "") -> str:
"""使用VLM模型描述图片内容
Args:
image_path: 图片文件路径
context_text: 图片周围的PDF文本上下文
Returns:
图片的文字描述
"""
import asyncio
import time
# 读取图片并编码为base64
with open(image_path, "rb") as f:
image_data = base64.b64encode(f.read()).decode("utf-8")
# 检测图片格式
ext = os.path.splitext(image_path)[1].lower()
mime_map = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp"}
mime_type = mime_map.get(ext, "image/png")
prompt = IMAGE_DESCRIPTION_PROMPT.format(context_text=context_text[:600])
vision_model = "Qwen/Qwen3-VL-8B-Instruct"
api_key, base_url, _ = _resolve_provider(vision_model)
client = openai.AsyncOpenAI(api_key=api_key, base_url=base_url)
max_retries = 3
for attempt in range(max_retries):
try:
response = await client.chat.completions.create(
model=vision_model,
messages=[{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{image_data}"}},
],
}],
max_tokens=600,
temperature=0.3,
timeout=90.0,
)
return response.choices[0].message.content or ""
except Exception as e:
print(f"[VLM] 描述失败 attempt={attempt+1}: {e}")
if attempt < max_retries - 1:
await asyncio.sleep(2 ** attempt)
return ""
# 全局LLM实例(使用默认模型)
llm_client = SiliconFlowLLM()