Initial: integrated 2025 LawGraph (graphrag_pipeline) + 2026 kg_project
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,75 @@
|
||||
# 测试文件说明
|
||||
|
||||
本目录包含所有测试脚本。
|
||||
|
||||
## 测试文件列表
|
||||
|
||||
### 1. `test_basic.py`
|
||||
基础功能测试，包括：
|
||||
- 模块导入测试
|
||||
- 本体模型测试
|
||||
- Prompt模板测试
|
||||
|
||||
运行方式：
|
||||
```bash
|
||||
uv run python tests/test_basic.py
|
||||
```
|
||||
|
||||
### 2. `test_siliconflow.py`
|
||||
测试硅基流动API配置，验证API密钥和连接是否正常。
|
||||
|
||||
运行方式：
|
||||
```bash
|
||||
uv run python tests/test_siliconflow.py
|
||||
```
|
||||
|
||||
### 3. `test_documents.py`
|
||||
测试文档解析功能，包括：
|
||||
- 文档解析（.docx文件）
|
||||
- TextUnit切分
|
||||
- 不调用LLM，仅测试解析逻辑
|
||||
|
||||
运行方式：
|
||||
```bash
|
||||
uv run python tests/test_documents.py
|
||||
```
|
||||
|
||||
### 4. `test_model.py`
|
||||
测试硅基流动平台上的多个模型，找出可用模型。
|
||||
|
||||
运行方式：
|
||||
```bash
|
||||
uv run python tests/test_model.py
|
||||
```
|
||||
|
||||
### 5. `test_my_models.py`
|
||||
测试项目配置使用的两个指定模型：
|
||||
- `deepseek-ai/DeepSeek-R1-Distill-Qwen-7B`
|
||||
- `Qwen/Qwen2.5-7B-Instruct`
|
||||
|
||||
运行方式：
|
||||
```bash
|
||||
uv run python tests/test_my_models.py
|
||||
```
|
||||
|
||||
### 6. `test_quick.py`
|
||||
快速测试完整流程，处理少量TextUnit（1个文档的前2个），验证：
|
||||
- 文档解析
|
||||
- 实体识别（NER）
|
||||
- 关系抽取（RE）
|
||||
- 知识图谱构建
|
||||
- 社区检测
|
||||
|
||||
运行方式：
|
||||
```bash
|
||||
uv run python tests/test_quick.py
|
||||
```
|
||||
|
||||
## 注意事项
|
||||
|
||||
- 测试脚本需要使用配置的API密钥（`.env`文件）
|
||||
- 部分测试会调用LLM API，可能需要一些时间
|
||||
- 确保已安装所有依赖包：`uv sync`
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
基础功能测试
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
def test_imports():
|
||||
"""测试模块导入"""
|
||||
try:
|
||||
from src.preprocessing.text_processor import TextProcessor
|
||||
from src.preprocessing.document_parser import DocumentParser
|
||||
from src.ontology.schema import OntologySchema
|
||||
from src.prompts.ner_prompts import NERPromptBuilder
|
||||
from src.extraction.ner import NERExtractor
|
||||
from src.kg_builder.graph import KnowledgeGraph
|
||||
print("✅ 所有模块导入成功")
|
||||
return True
|
||||
except ImportError as e:
|
||||
print(f"❌ 模块导入失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def test_ontology():
|
||||
"""测试本体模型"""
|
||||
try:
|
||||
from src.ontology.schema import OntologySchema
|
||||
|
||||
schema = OntologySchema()
|
||||
summary = schema.get_schema_summary()
|
||||
|
||||
assert summary["entity_types"] == 9
|
||||
assert summary["relation_types"] == 8
|
||||
print("✅ 本体模型测试通过")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ 本体模型测试失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def test_prompt_template():
|
||||
"""测试Prompt模板"""
|
||||
try:
|
||||
from src.prompts.template import PromptTemplate
|
||||
|
||||
template = PromptTemplate(
|
||||
task_description={"system": "test", "user": "test"},
|
||||
candidate_targets={"entity": "definition"},
|
||||
task_examples=[{"input": "test", "output": "test"}],
|
||||
task_emphasis="test",
|
||||
)
|
||||
|
||||
assert template.task_description is not None
|
||||
print("✅ Prompt模板测试通过")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ Prompt模板测试失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("开始运行基础测试...")
|
||||
test_imports()
|
||||
test_ontology()
|
||||
test_prompt_template()
|
||||
print("测试完成！")
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
测试文档解析（仅测试3个文件，不调用LLM）
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
# 修复：移除conda环境的路径，确保使用虚拟环境的包
|
||||
sys.path = [p for p in sys.path if 'anaconda3' not in p.lower() and 'conda' not in p.lower()]
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.preprocessing.document_parser import DocumentParser
|
||||
from src.utils.config import load_config
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def main():
|
||||
"""测试文档解析"""
|
||||
logger.info("=" * 60)
|
||||
logger.info("测试文档解析（3个文件）")
|
||||
logger.info("=" * 60)
|
||||
|
||||
# 配置
|
||||
config = load_config()
|
||||
data_dir = Path("../data/1法律")
|
||||
output_dir = Path("./output")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 解析文档
|
||||
logger.info(f"解析目录: {data_dir}")
|
||||
parser = DocumentParser()
|
||||
docs = parser.parse_directory(str(data_dir))
|
||||
|
||||
# 只处理前3个
|
||||
docs = docs[:3]
|
||||
logger.info(f"处理 {len(docs)} 个文档")
|
||||
|
||||
# 切分为TextUnit
|
||||
logger.info("切分文档为TextUnit...")
|
||||
all_textunits = []
|
||||
for i, doc in enumerate(docs, 1):
|
||||
logger.info(f"处理文档 {i}/{len(docs)}: {Path(doc.get('file_path', '')).name}")
|
||||
textunits = parser.split_into_textunits(
|
||||
doc,
|
||||
max_length=config.MAX_TEXTUNIT_LENGTH
|
||||
)
|
||||
for tu in textunits:
|
||||
tu["doc_id"] = doc.get("file_path", "")
|
||||
tu["id"] = f"{doc.get('file_path', '')}_{tu.get('paragraph_index', 0)}"
|
||||
all_textunits.extend(textunits)
|
||||
logger.info(f" - 生成 {len(textunits)} 个TextUnit")
|
||||
|
||||
logger.info(f"共生成 {len(all_textunits)} 个TextUnit")
|
||||
|
||||
# 保存结果
|
||||
output_file = output_dir / "test_textunits.json"
|
||||
result = {
|
||||
"num_docs": len(docs),
|
||||
"num_textunits": len(all_textunits),
|
||||
"docs": [
|
||||
{
|
||||
"file_path": doc.get("file_path", ""),
|
||||
"title": doc.get("title", ""),
|
||||
"text_length": len(doc.get("text", "")),
|
||||
}
|
||||
for doc in docs
|
||||
],
|
||||
"sample_textunits": [
|
||||
{
|
||||
"id": tu.get("id", ""),
|
||||
"text": tu.get("text", "")[:200] + "..." if len(tu.get("text", "")) > 200 else tu.get("text", ""),
|
||||
"paragraph_index": tu.get("paragraph_index", 0),
|
||||
}
|
||||
for tu in all_textunits[:10] # 只保存前10个作为示例
|
||||
]
|
||||
}
|
||||
|
||||
with open(output_file, "w", encoding="utf-8") as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("测试完成！")
|
||||
logger.info(f"结果已保存到: {output_file}")
|
||||
logger.info(f"统计信息:")
|
||||
logger.info(f" - 文档数: {result['num_docs']}")
|
||||
logger.info(f" - TextUnit数: {result['num_textunits']}")
|
||||
logger.info("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
测试硅基流动API并找到可用的模型
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
sys.path = [p for p in sys.path if 'anaconda3' not in p.lower() and 'conda' not in p.lower()]
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.utils.config import load_config
|
||||
from src.utils.llm_client import LLMClient, LLMProvider
|
||||
|
||||
# 常见的硅基流动模型名称
|
||||
TEST_MODELS = [
|
||||
"deepseek-ai/DeepSeek-V2.5",
|
||||
"deepseek-ai/DeepSeek-V2",
|
||||
"Qwen/Qwen2.5-72B-Instruct",
|
||||
"Qwen/Qwen2.5-72B-Chat",
|
||||
"meta-llama/Llama-3.1-70B-Instruct",
|
||||
"01-ai/Yi-1.5-34B-Chat",
|
||||
"mistralai/Mistral-7B-Instruct-v0.2",
|
||||
]
|
||||
|
||||
def test_model(provider, model_name):
|
||||
"""测试单个模型"""
|
||||
try:
|
||||
config = load_config()
|
||||
client = LLMClient(provider=provider, model=model_name, config=config)
|
||||
messages = [{"role": "user", "content": "你好"}]
|
||||
response = client.chat(messages)
|
||||
print(f"✅ 模型 {model_name} 可用")
|
||||
print(f" 响应: {response[:50]}...")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ 模型 {model_name} 不可用: {str(e)[:100]}")
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("=" * 60)
|
||||
print("测试硅基流动模型")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
config = load_config()
|
||||
if not config.SILICONFLOW_API_KEY:
|
||||
print("❌ SILICONFLOW_API_KEY 未配置")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"API密钥: {config.SILICONFLOW_API_KEY[:20]}...")
|
||||
print(f"API Base: {config.SILICONFLOW_API_BASE}")
|
||||
print()
|
||||
|
||||
# 测试所有模型
|
||||
available_models = []
|
||||
for model in TEST_MODELS:
|
||||
if test_model(LLMProvider.SILICONFLOW, model):
|
||||
available_models.append(model)
|
||||
print()
|
||||
|
||||
print("=" * 60)
|
||||
if available_models:
|
||||
print(f"✅ 找到 {len(available_models)} 个可用模型:")
|
||||
for model in available_models:
|
||||
print(f" - {model}")
|
||||
print(f"\n建议使用的模型: {available_models[0]}")
|
||||
else:
|
||||
print("❌ 没有找到可用的模型，请检查API密钥或模型名称")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
测试用户指定的两个模型
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path = [p for p in sys.path if 'anaconda3' not in p.lower() and 'conda' not in p.lower()]
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.utils.llm_client import LLMClient, LLMProvider
|
||||
from src.utils.config import load_config
|
||||
|
||||
def test_model(provider, model_name):
|
||||
"""测试单个模型"""
|
||||
try:
|
||||
config = load_config()
|
||||
client = LLMClient(provider=provider, model=model_name, config=config)
|
||||
messages = [{"role": "user", "content": "你好"}]
|
||||
response = client.chat(messages)
|
||||
print(f"✅ 模型 {model_name} 可用")
|
||||
print(f" 响应: {response[:50]}...")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ 模型 {model_name} 不可用: {str(e)[:100]}")
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("=" * 60)
|
||||
print("测试指定的两个模型")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
config = load_config()
|
||||
if not config.SILICONFLOW_API_KEY:
|
||||
print("❌ SILICONFLOW_API_KEY 未配置")
|
||||
sys.exit(1)
|
||||
|
||||
# 测试用户指定的两个模型
|
||||
models_to_test = [
|
||||
"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B",
|
||||
"Qwen/Qwen2.5-7B-Instruct",
|
||||
]
|
||||
|
||||
available = []
|
||||
for model in models_to_test:
|
||||
if test_model(LLMProvider.SILICONFLOW, model):
|
||||
available.append(model)
|
||||
print()
|
||||
|
||||
print("=" * 60)
|
||||
if len(available) == len(models_to_test):
|
||||
print(f"✅ 所有模型都可用！")
|
||||
elif available:
|
||||
print(f"⚠️ 部分模型可用: {len(available)}/{len(models_to_test)}")
|
||||
else:
|
||||
print("❌ 没有可用模型")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
快速测试：只处理1个文档的前2个TextUnit
|
||||
"""
|
||||
|
||||
import sys
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
sys.path = [p for p in sys.path if 'anaconda3' not in p.lower() and 'conda' not in p.lower()]
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.preprocessing.document_parser import DocumentParser
|
||||
from src.kg_builder.indexer import GraphIndexer
|
||||
from src.utils.config import load_config
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def main():
|
||||
logger.info("=" * 60)
|
||||
logger.info("快速测试：处理1个文档的前2个TextUnit")
|
||||
logger.info("=" * 60)
|
||||
|
||||
config = load_config()
|
||||
|
||||
# 解析1个文档
|
||||
parser = DocumentParser()
|
||||
doc_path = "../data/1法律/10-中华人民共和国乡村振兴促进法.docx"
|
||||
doc = parser.parse_docx(doc_path)
|
||||
|
||||
# 切分为TextUnit，只取前2个
|
||||
textunits = parser.split_into_textunits(doc, max_length=config.MAX_TEXTUNIT_LENGTH)[:2]
|
||||
for tu in textunits:
|
||||
tu["doc_id"] = doc.get("file_path", "")
|
||||
tu["id"] = f"{doc.get('file_path', '')}_{tu.get('paragraph_index', 0)}"
|
||||
|
||||
logger.info(f"准备处理 {len(textunits)} 个TextUnit")
|
||||
|
||||
# 构建知识图谱（只处理这2个）
|
||||
logger.info("开始构建知识图谱...")
|
||||
indexer = GraphIndexer(config=config)
|
||||
kg_graph = indexer.index(textunits, use_verification=True)
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("快速测试完成！")
|
||||
logger.info(f"节点数: {kg_graph.number_of_nodes()}")
|
||||
logger.info(f"边数: {kg_graph.number_of_edges()}")
|
||||
logger.info("=" * 60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
测试硅基流动API配置
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.utils.llm_client import LLMClient, LLMProvider
|
||||
from src.utils.config import load_config
|
||||
|
||||
|
||||
def test_siliconflow():
|
||||
"""测试硅基流动API"""
|
||||
print("=" * 60)
|
||||
print("测试硅基流动API配置")
|
||||
print("=" * 60)
|
||||
|
||||
# 加载配置
|
||||
config = load_config()
|
||||
|
||||
# 检查API密钥
|
||||
if not config.SILICONFLOW_API_KEY:
|
||||
print("❌ 错误: SILICONFLOW_API_KEY 未配置")
|
||||
print("请在 .env 文件中设置 SILICONFLOW_API_KEY")
|
||||
return False
|
||||
|
||||
print(f"✅ API密钥已配置: {config.SILICONFLOW_API_KEY[:20]}...")
|
||||
print(f"✅ API Base URL: {config.SILICONFLOW_API_BASE}")
|
||||
|
||||
# 创建客户端（使用一个常见的模型，用户需要根据实际可用模型调整）
|
||||
print("\n尝试创建客户端...")
|
||||
try:
|
||||
# 注意：这里使用的模型名称需要根据硅基流动平台实际可用模型调整
|
||||
# 常用的模型包括：Qwen/Qwen2.5-72B-Instruct, meta-llama/Llama-3.1-70B-Instruct 等
|
||||
client = LLMClient(
|
||||
provider=LLMProvider.SILICONFLOW,
|
||||
model="Qwen/Qwen2.5-72B-Instruct", # 请根据实际情况修改模型名称
|
||||
config=config
|
||||
)
|
||||
print("✅ 客户端创建成功")
|
||||
except Exception as e:
|
||||
print(f"❌ 客户端创建失败: {e}")
|
||||
return False
|
||||
|
||||
# 测试简单对话
|
||||
print("\n测试API调用...")
|
||||
messages = [
|
||||
{"role": "user", "content": "请用一句话介绍知识图谱。"}
|
||||
]
|
||||
|
||||
try:
|
||||
response = client.chat(messages)
|
||||
print(f"✅ API调用成功")
|
||||
print(f"\n响应内容：\n{response}")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ API调用失败: {e}")
|
||||
print("\n提示：")
|
||||
print("1. 请检查API密钥是否正确")
|
||||
print("2. 请检查网络连接")
|
||||
print("3. 请确认模型名称是否在硅基流动平台可用")
|
||||
print("4. 可以访问 https://siliconflow.cn/ 查看可用模型列表")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = test_siliconflow()
|
||||
if success:
|
||||
print("\n" + "=" * 60)
|
||||
print("✅ 硅基流动API配置测试通过！")
|
||||
print("=" * 60)
|
||||
else:
|
||||
print("\n" + "=" * 60)
|
||||
print("❌ 硅基流动API配置测试失败，请检查配置")
|
||||
print("=" * 60)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user