Initial: integrated 2025 LawGraph (graphrag_pipeline) + 2026 kg_project
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
知识图谱构建脚本
|
||||
示例用法：python scripts/build_kg.py --data-dir ../data/1法律 --output ./output/kg.json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 添加src到路径
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.preprocessing.document_parser import DocumentParser
|
||||
from src.kg_builder.indexer import GraphIndexer
|
||||
from src.kg_builder.graph import KnowledgeGraph
|
||||
from src.utils.config import Config, load_config
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="构建法规知识图谱")
|
||||
parser.add_argument(
|
||||
"--data-dir",
|
||||
type=str,
|
||||
required=True,
|
||||
help="法规数据目录路径"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
default="output/kg.json",
|
||||
help="输出文件路径"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-docs",
|
||||
type=int,
|
||||
default=None,
|
||||
help="最大处理文档数（用于测试）"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-verification",
|
||||
action="store_true",
|
||||
help="不使用二次对话验证（加快速度）"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 加载配置
|
||||
config = load_config()
|
||||
|
||||
# 确保输出目录存在
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 解析文档
|
||||
logger.info(f"解析目录: {args.data_dir}")
|
||||
parser_obj = DocumentParser()
|
||||
docs = parser_obj.parse_directory(args.data_dir)
|
||||
|
||||
if args.max_docs:
|
||||
docs = docs[:args.max_docs]
|
||||
|
||||
logger.info(f"找到 {len(docs)} 个文档")
|
||||
|
||||
# 切分为TextUnit
|
||||
logger.info("切分文档为TextUnit...")
|
||||
all_textunits = []
|
||||
for doc in docs:
|
||||
textunits = parser_obj.split_into_textunits(
|
||||
doc,
|
||||
max_length=config.MAX_TEXTUNIT_LENGTH
|
||||
)
|
||||
for tu in textunits:
|
||||
tu["doc_id"] = doc.get("file_path", "")
|
||||
tu["id"] = f"{doc.get('file_path', '')}_{tu.get('paragraph_index', 0)}"
|
||||
all_textunits.extend(textunits)
|
||||
|
||||
logger.info(f"共生成 {len(all_textunits)} 个TextUnit")
|
||||
|
||||
# 构建知识图谱
|
||||
logger.info("开始构建知识图谱...")
|
||||
indexer = GraphIndexer(config=config)
|
||||
kg_graph = indexer.index(
|
||||
all_textunits,
|
||||
use_verification=not args.no_verification
|
||||
)
|
||||
|
||||
# 保存图谱
|
||||
kg_obj = KnowledgeGraph()
|
||||
kg_obj.graph = kg_graph
|
||||
kg_obj.save(str(output_path), format="json")
|
||||
|
||||
# 保存统计信息
|
||||
stats = {
|
||||
"num_docs": len(docs),
|
||||
"num_textunits": len(all_textunits),
|
||||
"num_nodes": kg_graph.number_of_nodes(),
|
||||
"num_edges": kg_graph.number_of_edges(),
|
||||
"metadata": kg_graph.graph.get("metadata", {})
|
||||
}
|
||||
|
||||
stats_path = output_path.parent / f"{output_path.stem}_stats.json"
|
||||
with open(stats_path, "w", encoding="utf-8") as f:
|
||||
json.dump(stats, f, ensure_ascii=False, indent=2)
|
||||
|
||||
logger.info(f"知识图谱构建完成！")
|
||||
logger.info(f" - 节点数: {stats['num_nodes']}")
|
||||
logger.info(f" - 边数: {stats['num_edges']}")
|
||||
logger.info(f" - 结果已保存到: {output_path}")
|
||||
logger.info(f" - 统计信息已保存到: {stats_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
完整流程脚本：从文档到知识图谱构建和查询
|
||||
示例用法：python scripts/complete_pipeline.py --data-dir ../data/1法律 --output-dir ./output
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 修复：移除conda环境的路径，确保使用虚拟环境的包
|
||||
sys.path = [p for p in sys.path if 'anaconda3' not in p.lower() and 'conda' not in p.lower()]
|
||||
# 添加src到路径
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.preprocessing.document_parser import DocumentParser
|
||||
from src.kg_builder.indexer import GraphIndexer
|
||||
from src.kg_builder.graph import KnowledgeGraph
|
||||
from src.utils.config import Config, load_config
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="完整的知识图谱构建流程")
|
||||
parser.add_argument(
|
||||
"--data-dir",
|
||||
type=str,
|
||||
required=True,
|
||||
help="法规数据目录路径"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=str,
|
||||
default="output",
|
||||
help="输出目录"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-docs",
|
||||
type=int,
|
||||
default=None,
|
||||
help="最大处理文档数（用于测试）"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-index",
|
||||
action="store_true",
|
||||
help="跳过索引构建（使用已有索引）"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 加载配置
|
||||
config = load_config()
|
||||
|
||||
# 创建输出目录
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
kg_file = output_dir / "knowledge_graph.json"
|
||||
|
||||
# 步骤1: 构建知识图谱索引
|
||||
if not args.skip_index:
|
||||
logger.info("=" * 60)
|
||||
logger.info("步骤1: 解析文档并构建知识图谱")
|
||||
logger.info("=" * 60)
|
||||
|
||||
# 解析文档
|
||||
logger.info(f"解析目录: {args.data_dir}")
|
||||
parser_obj = DocumentParser()
|
||||
docs = parser_obj.parse_directory(args.data_dir)
|
||||
|
||||
if args.max_docs:
|
||||
docs = docs[:args.max_docs]
|
||||
|
||||
logger.info(f"找到 {len(docs)} 个文档")
|
||||
|
||||
# 切分为TextUnit
|
||||
logger.info("切分文档为TextUnit...")
|
||||
all_textunits = []
|
||||
for doc in docs:
|
||||
textunits = parser_obj.split_into_textunits(
|
||||
doc,
|
||||
max_length=config.MAX_TEXTUNIT_LENGTH
|
||||
)
|
||||
for tu in textunits:
|
||||
tu["doc_id"] = doc.get("file_path", "")
|
||||
tu["id"] = f"{doc.get('file_path', '')}_{tu.get('paragraph_index', 0)}"
|
||||
all_textunits.extend(textunits)
|
||||
|
||||
logger.info(f"共生成 {len(all_textunits)} 个TextUnit")
|
||||
|
||||
# 构建知识图谱
|
||||
logger.info("开始构建知识图谱...")
|
||||
indexer = GraphIndexer(config=config)
|
||||
kg_graph = indexer.index(all_textunits, use_verification=True)
|
||||
|
||||
# 保存图谱
|
||||
kg_obj = KnowledgeGraph()
|
||||
kg_obj.graph = kg_graph
|
||||
kg_obj.save(str(kg_file), format="json")
|
||||
|
||||
logger.info(f"知识图谱已保存到: {kg_file}")
|
||||
else:
|
||||
logger.info(f"跳过索引构建，从文件加载: {kg_file}")
|
||||
kg_obj = KnowledgeGraph()
|
||||
kg_obj.load(str(kg_file), format="json")
|
||||
|
||||
# 步骤2: 展示图谱统计信息
|
||||
logger.info("=" * 60)
|
||||
logger.info("步骤2: 知识图谱统计信息")
|
||||
logger.info("=" * 60)
|
||||
|
||||
if kg_obj.graph:
|
||||
metadata = kg_obj.graph.graph.get("metadata", {})
|
||||
logger.info(f"节点数: {kg_obj.graph.number_of_nodes()}")
|
||||
logger.info(f"边数: {kg_obj.graph.number_of_edges()}")
|
||||
logger.info(f"三元组数: {metadata.get('num_triplets', 0)}")
|
||||
logger.info(f"社区数: {metadata.get('num_communities', 0)}")
|
||||
|
||||
# 展示一些社区摘要
|
||||
summaries = metadata.get("summaries", {})
|
||||
if summaries:
|
||||
logger.info("\n前5个社区摘要：")
|
||||
for i, (comm_id, comm_data) in enumerate(list(summaries.items())[:5], 1):
|
||||
logger.info(f"{i}. {comm_id}: {comm_data.get('summary', '无摘要')[:100]}...")
|
||||
|
||||
logger.info("\n知识图谱构建流程完成！")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
知识抽取脚本
|
||||
示例用法：python scripts/extract.py --input data/sample.txt --output output/entities.json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 添加src到路径
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.preprocessing.document_parser import DocumentParser
|
||||
from src.extraction.ner import NERExtractor
|
||||
from src.utils.config import Config, load_config
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="法规知识抽取工具")
|
||||
parser.add_argument(
|
||||
"--input",
|
||||
type=str,
|
||||
required=True,
|
||||
help="输入文件路径（.docx或.txt）"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
default="output/entities.json",
|
||||
help="输出文件路径"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use-verification",
|
||||
action="store_true",
|
||||
default=True,
|
||||
help="使用二次对话验证"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 加载配置
|
||||
config = load_config()
|
||||
|
||||
# 确保输出目录存在
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 解析文档
|
||||
input_path = Path(args.input)
|
||||
if input_path.suffix == ".docx":
|
||||
parser = DocumentParser()
|
||||
doc_data = parser.parse_docx(str(input_path))
|
||||
text = doc_data["text"]
|
||||
else:
|
||||
# 假设是纯文本文件
|
||||
with open(input_path, "r", encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
|
||||
# 执行NER
|
||||
logger.info("开始实体识别...")
|
||||
ner_extractor = NERExtractor(config=config)
|
||||
entities = ner_extractor.extract(text, use_verification=args.use_verification)
|
||||
|
||||
# 保存结果
|
||||
result = {
|
||||
"input_file": str(input_path),
|
||||
"entities": entities,
|
||||
"statistics": {
|
||||
entity_type: len(entity_list)
|
||||
for entity_type, entity_list in entities.items()
|
||||
}
|
||||
}
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
|
||||
logger.info(f"实体识别完成，结果已保存到: {output_path}")
|
||||
logger.info(f"统计信息: {result['statistics']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
知识图谱查询脚本
|
||||
示例用法：python scripts/query_kg.py --kg output/kg.json --query "城市更新"
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 添加src到路径
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.kg_builder.graph import KnowledgeGraph
|
||||
from src.query.global_search import GlobalSearcher
|
||||
from src.query.local_search import LocalSearcher
|
||||
from src.utils.config import load_config
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="查询知识图谱")
|
||||
parser.add_argument(
|
||||
"--kg",
|
||||
type=str,
|
||||
required=True,
|
||||
help="知识图谱文件路径（JSON格式）"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--query",
|
||||
type=str,
|
||||
required=True,
|
||||
help="查询文本"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
type=str,
|
||||
choices=["global", "local", "drift"],
|
||||
default="global",
|
||||
help="查询模式"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
default=None,
|
||||
help="输出结果文件路径"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 加载知识图谱
|
||||
logger.info(f"加载知识图谱: {args.kg}")
|
||||
kg = KnowledgeGraph()
|
||||
kg.load(str(args.kg), format="json")
|
||||
|
||||
# 获取社区信息（从元数据）
|
||||
communities = {}
|
||||
if hasattr(kg, 'graph') and kg.graph:
|
||||
metadata = kg.graph.graph.get("metadata", {})
|
||||
communities = metadata.get("summaries", {})
|
||||
|
||||
# 执行查询
|
||||
logger.info(f"执行{args.mode}查询: {args.query}")
|
||||
|
||||
kg_graph = kg.graph if hasattr(kg, 'graph') and kg.graph else None
|
||||
|
||||
if args.mode == "global":
|
||||
searcher = GlobalSearcher(kg_graph, communities)
|
||||
result = searcher.search(args.query)
|
||||
elif args.mode == "local":
|
||||
searcher = LocalSearcher(kg_graph)
|
||||
# 尝试从查询中提取实体
|
||||
entity = args.query # 简化处理，可以将查询作为实体
|
||||
result = searcher.search(entity, depth=2)
|
||||
else:
|
||||
# DRIFT search
|
||||
from src.query.drift_search import DriftSearcher
|
||||
searcher = DriftSearcher(kg_graph, communities)
|
||||
result = searcher.search(args.query)
|
||||
|
||||
# 输出结果
|
||||
print("\n查询结果：")
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
logger.info(f"结果已保存到: {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user