feat: multimodal RAG with PDF image extraction and display

Extract images from PDFs using pymupdf, generate descriptions via
Qwen3-VL-8B, store in ChromaDB alongside text chunks, and render
images in chat answers. Includes image proxy rewrite, force re-process
endpoint, and VLM API timeout.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-05-28 12:14:40 +08:00
parent 3f12e96ea0
commit 4bb50ae9c1
15 changed files with 351 additions and 40 deletions
+4
View File
@@ -32,6 +32,10 @@ const nextConfig = {
source: '/generated_images/:path*',
destination: `${backendUrl}/generated_images/:path*`,
},
{
source: '/images/:path*',
destination: `${backendUrl}/images/:path*`,
},
];
},
// 禁用静态生成,避免 SSR 时使用浏览器 API 的错误
+16
View File
@@ -252,6 +252,22 @@ export default function MessageItem({ message, selectedModel }: MessageItemProps
td: ({ children }) => (
<td className="border border-border px-2 py-1.5">{children}</td>
),
img: ({ src, alt }: any) => (
<a href={src} target="_blank" rel="noopener noreferrer" className="block my-3 group">
<img
src={src}
alt={alt || "图片来源"}
loading="lazy"
className="max-w-full max-h-80 rounded-lg border border-border/50 cursor-pointer
hover:border-primary/40 transition-colors object-contain bg-muted/20"
/>
{alt && (
<span className="block text-[11px] text-muted-foreground/70 mt-1 text-center">
{alt}
</span>
)}
</a>
),
}}
>
{preprocessCitations(message.content, String(message.id))}
+30 -3
View File
@@ -1,7 +1,7 @@
"use client";
import { useState, useMemo } from "react";
import { Database, Globe, ChevronDown, ChevronRight, Star } from "lucide-react";
import { Database, Globe, ChevronDown, ChevronRight, Star, Image } from "lucide-react";
import { Button } from "@/components/ui/button";
import { cn } from "@/lib/utils";
@@ -13,7 +13,8 @@ interface SourceReference {
score?: number;
preview: string;
url?: string;
source_type?: "web" | "rag";
source_type?: "web" | "rag" | "image";
image_url?: string;
}
interface SourceReferencesProps {
@@ -188,7 +189,8 @@ export default function SourceReferences({
const showExpandButton = sources.length > maxSources;
const visibleSources = expanded ? sources : displaySources;
const ragSources = visibleSources.filter((s) => s.source_type !== "web");
const ragSources = visibleSources.filter((s) => s.source_type !== "web" && s.source_type !== "image");
const imageSources = visibleSources.filter((s) => s.source_type === "image");
const webSources = visibleSources.filter((s) => s.source_type === "web");
const sourceSection = (
@@ -218,6 +220,31 @@ export default function SourceReferences({
</div>
)}
{imageSources.length > 0 && (
<div className="space-y-1">
<div className="flex items-center gap-1.5 text-[11px] font-medium text-muted-foreground px-0.5">
<Image className="h-3 w-3 text-purple-500" />
<span>
({imageSources.length}
{expanded && showExpandButton
? `/${sources.filter((s) => s.source_type === "image").length}`
: ""}
)
</span>
</div>
<div className="divide-y divide-border/30">
{imageSources.map((source, i) => (
<SourceRow
key={i}
source={source}
answerContent={answerContent}
messageId={messageId}
/>
))}
</div>
</div>
)}
{webSources.length > 0 && (
<div className="space-y-1">
<div className="flex items-center gap-1.5 text-[11px] font-medium text-muted-foreground px-0.5">
+2 -1
View File
@@ -101,7 +101,8 @@ export interface SourceInfo {
score?: number;
preview: string;
url?: string;
source_type?: "web" | "rag";
source_type?: "web" | "rag" | "image";
image_url?: string;
}
export interface ChatResponse {