feat: multimodal RAG with PDF image extraction and display
Extract images from PDFs using pymupdf, generate descriptions via Qwen3-VL-8B, store in ChromaDB alongside text chunks, and render images in chat answers. Includes image proxy rewrite, force re-process endpoint, and VLM API timeout. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
"""
|
||||
大模型API集成 — 支持 SiliconFlow 和 DeepSeek 官方
|
||||
大模型API集成 — 支持 SiliconFlow 和 DeepSeek 官方 + 视觉模型
|
||||
"""
|
||||
import os
|
||||
import base64
|
||||
from typing import List, Dict, Any, Optional, AsyncGenerator, Tuple
|
||||
from langchain_openai import ChatOpenAI
|
||||
from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, SystemMessage
|
||||
@@ -13,6 +14,17 @@ from ..core.config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
# 图片描述提示词
|
||||
IMAGE_DESCRIPTION_PROMPT = """你是一个国土空间规划专家。请详细描述这张PDF文档中的图片内容。
|
||||
|
||||
图片周围文字上下文(来自PDF页面):{context_text}
|
||||
|
||||
要求:
|
||||
1. 说明图片类型(地图/规划图/图表/流程图/示意图/照片等)
|
||||
2. 描述图片中的关键信息、数据和空间关系
|
||||
3. 提取图中所有文字标注
|
||||
4. 描述控制在200-300字"""
|
||||
|
||||
# DeepSeek 官方模型 ID 前缀(用于自动路由)
|
||||
DEEPSEEK_OFFICIAL_MODELS = {
|
||||
"deepseek-chat",
|
||||
@@ -150,6 +162,61 @@ class SiliconFlowLLM:
|
||||
return messages
|
||||
|
||||
|
||||
async def describe_image(self, image_path: str, context_text: str = "") -> str:
|
||||
"""使用VLM模型描述图片内容
|
||||
|
||||
Args:
|
||||
image_path: 图片文件路径
|
||||
context_text: 图片周围的PDF文本上下文
|
||||
|
||||
Returns:
|
||||
图片的文字描述
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
# 读取图片并编码为base64
|
||||
with open(image_path, "rb") as f:
|
||||
image_data = base64.b64encode(f.read()).decode("utf-8")
|
||||
|
||||
# 检测图片格式
|
||||
ext = os.path.splitext(image_path)[1].lower()
|
||||
mime_map = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp"}
|
||||
mime_type = mime_map.get(ext, "image/png")
|
||||
|
||||
prompt = IMAGE_DESCRIPTION_PROMPT.format(context_text=context_text[:600])
|
||||
|
||||
vision_model = "Qwen/Qwen3-VL-8B-Instruct"
|
||||
api_key, base_url, _ = _resolve_provider(vision_model)
|
||||
|
||||
client = openai.AsyncOpenAI(api_key=api_key, base_url=base_url)
|
||||
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = await client.chat.completions.create(
|
||||
model=vision_model,
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{image_data}"}},
|
||||
],
|
||||
}],
|
||||
max_tokens=600,
|
||||
temperature=0.3,
|
||||
timeout=90.0,
|
||||
)
|
||||
return response.choices[0].message.content or ""
|
||||
|
||||
except Exception as e:
|
||||
print(f"[VLM] 描述失败 attempt={attempt+1}: {e}")
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
# 全局LLM实例(使用默认模型)
|
||||
llm_client = SiliconFlowLLM()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user