Initial: integrated 2025 LawGraph (graphrag_pipeline) + 2026 kg_project
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
config/api_keys.yaml
|
||||
output/
|
||||
logs/
|
||||
*.log
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
@@ -0,0 +1,122 @@
|
||||
llm:
|
||||
model: "deepseek-chat"
|
||||
temperature: 0.0
|
||||
max_tokens: 8192
|
||||
max_retries: 3
|
||||
retry_delay: 2
|
||||
request_timeout: 120
|
||||
batch_timeout: 600
|
||||
|
||||
batch_processing:
|
||||
batch_size: 10
|
||||
concurrent_requests: 5
|
||||
test_batch_size: 5
|
||||
incremental_save: true
|
||||
|
||||
deduplication:
|
||||
similarity_threshold: 0.85
|
||||
enable_fuzzy_matching: true
|
||||
text_normalization: true
|
||||
|
||||
validation:
|
||||
enable_entity_validation: true
|
||||
enable_relationship_validation: true
|
||||
min_confidence_score: 0.7
|
||||
strict_mode: false
|
||||
filter_generic_concepts: true
|
||||
|
||||
output:
|
||||
nodes_file: "output/nodes_llm_v2.csv"
|
||||
relationships_file: "output/rels_llm_v2.csv"
|
||||
report_file: "output/extraction_report_v2.md"
|
||||
encoding: "utf-8-sig"
|
||||
|
||||
progress:
|
||||
progress_file: "output/progress_v2.json"
|
||||
checkpoint_interval: 10
|
||||
|
||||
logging:
|
||||
log_file: "logs/deep_extraction_v2.log"
|
||||
level: "INFO"
|
||||
format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
|
||||
# ============================================================
|
||||
# 第二轮本体：城市更新法规政策工具箱 (21实体 + 34关系)
|
||||
# ============================================================
|
||||
|
||||
entity_type_prompts:
|
||||
Agency: "主体机构：发布主体、实施主体、监管主体、责任主体，包括政府部门（全国人大常委会、国务院、自然资源部、住建部、生态环境部、发改委、地方政府）、实施机构、平台公司、社区组织、第三方机构等。属性：level（国家/省级/市级/县级/乡镇街道/社区）、agency_type（立法机关/行政机关/主管部门/实施主体/监管主体/责任主体/市场主体/社区组织/第三方机构）"
|
||||
LegalObject: "法规事项：法规政策所规制的事项，回答'管什么事项'。如规划编制、土地利用、建设管理、历史文化保护、生态保护、财政支持、金融支持、产权登记、公众参与、行政审批、监督管理、城市更新、公共服务、基础设施、安全韧性等。属性：category"
|
||||
SpatialObject: "空间对象：城市更新中的具体空间对象，回答'作用于什么空间'。如老旧小区、城中村、低效用地、工业遗存、历史文化街区、历史建筑、公共空间、公共服务设施、市政基础设施、存量建筑、建设用地、城市更新单元、生态保护红线、永久基本农田、城镇开发边界等。属性：spatial_type"
|
||||
RenewalScene: "更新场景：城市更新应用场景，回答'用于什么更新场景'。如老旧小区改造、城中村改造、低效用地再开发、工业遗存更新、历史文化保护更新、完整社区建设、公共空间提升、基础设施补短板、城市安全韧性提升、城市生态修复等。属性：scene_type、derivation_method（原文提取/语义归纳/人工标注/模型推理）"
|
||||
PolicyTool: "具体政策工具：城市更新中用于实现治理目标的具体政策、制度、措施或治理手段。属性：tool_type（命令型/激励型/劝诫型/能力型/系统变革型）、mandatory_level（强制/引导/鼓励/支持/禁止/限制）、source_text"
|
||||
ToolCategory: "政策工具类别：归类政策工具的分类节点。属性：classification_system（一般政策工具分类/城市更新工具分类）、urban_renewal_category（规划引导/土地配置/开发激励/财政金融/权益协调/实施管理/监督反馈）"
|
||||
Procedure: "程序：审批、备案、评估、论证、公示、听证、入库、验收、监督、征收、补偿等行政或治理程序。属性：procedure_type、time_limit"
|
||||
Obligation: "权利义务：法规规定的权利、义务、禁止或责任。属性：obligation_type（权利/义务/禁止/职责/鼓励/支持）、modality（应当/可以/不得/禁止/鼓励/支持/负责/依法）"
|
||||
Condition: "适用条件：政策工具或程序适用时的前置条件。如'经批准后''符合XX条件'。属性：condition_type（规划条件/审批条件/权属条件/安全条件/环保条件/公众参与条件/程序条件/资金条件）"
|
||||
Constraint: "约束条件：对行为设置的边界性、底线性或限制性要求。如'不得突破''严守''控制线'。属性：constraint_type（用途管制/空间边界/强制标准/历史保护/生态保护/权益保护/安全底线/公共利益）"
|
||||
Penalty: "法律责任：违反法规后的处罚。属性：penalty_type（行政处罚/刑事处罚/民事责任/行政处分/信用惩戒/整改责任）、amount_or_measure"
|
||||
TimePoint: "时间节点：生效、修订、废止、截止、过渡期等时间信息。属性：time_type（发布日期/生效日期/修订日期/废止日期/截止日期/过渡期）"
|
||||
Region: "适用区域：全国、省、市、县、特定区域等。属性：level（全国/区域/省/市/县/乡镇街道/社区/特定区域）"
|
||||
|
||||
relationship_type_prompts:
|
||||
# 文档层次
|
||||
has_chapter: "文档包含章"
|
||||
has_section: "章包含节"
|
||||
has_article: "文件/章/节包含条文"
|
||||
has_clause: "条文包含款"
|
||||
article_in_document: "条文或款属于某一法规文件"
|
||||
# 法条依据
|
||||
cites: "引用其他法规：如'根据《城乡规划法》第X条'、'依据《土地管理法》规定'、'按照国务院有关规定'"
|
||||
implements: "实施上位法：如'为实施《城乡规划法》制定本办法'、'根据《土地管理法》授权'"
|
||||
amends: "修正/修订：如'对《XX法》作如下修改'、'将第X条修改为'"
|
||||
replaces: "替代/废止：如'本法自施行之日起，XX法同时废止'"
|
||||
supplements: "补充规定：如'在XX基础上补充规定'"
|
||||
basis_for_planning: "法规或条文作为规划编制、审批、实施或政策工具设立的依据"
|
||||
# 主体关系
|
||||
issued_by: "法规文件由某主体发布或制定"
|
||||
implemented_by: "条文、政策工具、程序或义务由某主体负责实施"
|
||||
assigned_to: "监管职责、管理职责、实施任务分配给某主体"
|
||||
# 语义规制
|
||||
regulates: "条文对某法规事项进行规制（准入、禁止、限制、鼓励、保护等）"
|
||||
defines_object: "条文对法规事项、空间对象或更新场景进行定义或界定"
|
||||
applies_to: "条文、程序或政策工具适用于某法规事项、空间对象、更新场景或区域"
|
||||
sets_obligation: "条文设定权利、义务、禁止、职责或支持性要求"
|
||||
sets_condition: "条文、政策工具或程序设置适用条件（前置条件）"
|
||||
sets_constraint: "条文、政策工具或程序设置边界性、底线性或限制性要求"
|
||||
prescribes_penalty: "条文规定了违法行为的法律后果"
|
||||
# 政策工具
|
||||
extracts_tool: "从条文中识别出具体政策工具"
|
||||
belongs_to_category: "具体政策工具属于某类政策工具类别"
|
||||
applies_to_scene: "政策工具适用于某类城市更新场景"
|
||||
targets_object: "政策工具作用于某一法规事项或空间对象"
|
||||
coordinates_with: "两个政策工具之间存在并列协同或组合使用关系"
|
||||
supports_tool: "一个政策工具对另一个形成前置支撑或条件支撑"
|
||||
# 程序
|
||||
requires_procedure: "条文、政策工具或法规事项要求履行某类程序"
|
||||
procedure_for: "某一程序对应某类法规事项、空间对象、更新场景或政策工具"
|
||||
precedes_procedure: "一个程序在时序上先于另一个程序"
|
||||
# 辅助
|
||||
governs_region: "法规政策或条文适用于某一区域"
|
||||
effective_timeline: "法规文件关联时间节点（发布/生效/修订/废止）"
|
||||
standard_for: "标准、规范或条文适用于某一法规事项、空间对象、程序或政策工具"
|
||||
article_effective_timeline: "条文关联时间节点（生效/截止/过渡期）"
|
||||
|
||||
extraction_tips:
|
||||
citation_patterns: "注意识别法条引用模式：'根据《XX法》'、'依照第X条规定'、'按照国务院XX规定'、'参照XX标准执行'"
|
||||
article_classification: "法条类型判断：管理性（应当/必须）、禁止性（不得/禁止）、授权性（可以/有权）、程序性（申请/审批/备案）、处罚性（罚款/责令/没收）"
|
||||
cross_reference: "特别注意跨文档引用：当条文引用其他法规时，创建cites关系并记录引用上下文"
|
||||
entity_boundary_rules: |
|
||||
实体边界规则（严格遵守）：
|
||||
- LegalObject：法规事项，回答"管什么事项"，如规划编制、土地利用、财政支持
|
||||
- SpatialObject：空间对象，回答"作用于什么空间"，如老旧小区、城中村、低效用地
|
||||
- RenewalScene：更新场景，回答"用于什么更新场景"，如老旧小区改造、城中村改造
|
||||
- Procedure：程序，强调流程步骤（审批/备案/评估/公示/验收/入库）
|
||||
- PolicyTool：政策工具，强调治理手段、政策措施、制度安排或支持机制
|
||||
- Obligation：权利义务，强调主体应当/可以/不得做什么
|
||||
- Condition：适用条件，强调前置条件（经批准/符合/满足/具备）
|
||||
- Constraint：约束条件，强调底线限制（不得突破/严守/控制线/强制性标准）
|
||||
do_not_extract: |
|
||||
- 不要新增本体之外的实体类型或关系类型
|
||||
- 文本中没有依据时不要强行抽取
|
||||
- conflicts_with和has_gap不纳入本阶段抽取
|
||||
@@ -0,0 +1,86 @@
|
||||
project_name: "urban_planning_law_kg"
|
||||
project_description: "城市规划法律法规知识图谱构建"
|
||||
|
||||
data:
|
||||
source_dir: "E:\\Project\\SI\\2026_KG_PlanningLaw\\data\\城市规划法律法规\\城市更新法规数据库"
|
||||
total_documents: 296
|
||||
categories:
|
||||
- {id: 0, name: "0知识图谱目录", type: "Index"}
|
||||
- {id: 1, name: "1法律", type: "Law"}
|
||||
- {id: 2, name: "2行政法规", type: "AdministrativeRegulation"}
|
||||
- {id: 3, name: "3部门规章", type: "DepartmentalRule"}
|
||||
- {id: 4, name: "4党中央国务院文件", type: "PolicyDocument"}
|
||||
- {id: 5, name: "5国家主管部门文件", type: "PolicyDocument"}
|
||||
- {id: 6, name: "6主要技术标准", type: "PolicyDocument"}
|
||||
min_text_length: 50
|
||||
|
||||
api:
|
||||
config_file: "config/api_keys.yaml"
|
||||
model: "deepseek-chat"
|
||||
max_tokens: 8192
|
||||
temperature: 0.0
|
||||
|
||||
processing:
|
||||
concurrent_requests: 5
|
||||
max_retries: 3
|
||||
retry_delay: 2
|
||||
batch_size: 10
|
||||
|
||||
paths:
|
||||
config_dir: "config"
|
||||
data_dir: "data"
|
||||
output_dir: "output"
|
||||
logs_dir: "logs"
|
||||
ontology_dir: "ontology"
|
||||
scripts_dir: "scripts"
|
||||
src_dir: "src"
|
||||
|
||||
ontology:
|
||||
ontology_file: "ontology/legal_ontology.json"
|
||||
entity_ontology_file: "config/legal_ontology.yaml"
|
||||
|
||||
neo4j:
|
||||
uri: "bolt://localhost:7687"
|
||||
username: "neo4j"
|
||||
password: "password"
|
||||
database: "neo4j"
|
||||
|
||||
extraction:
|
||||
entity_types:
|
||||
- Law
|
||||
- AdministrativeRegulation
|
||||
- DepartmentalRule
|
||||
- PolicyDocument
|
||||
- Chapter
|
||||
- Section
|
||||
- Article
|
||||
- GovernmentBody
|
||||
- LegalSubject
|
||||
- SpatialConcept
|
||||
- AdministrativeProcedure
|
||||
- Obligation
|
||||
- Penalty
|
||||
- TimePoint
|
||||
- Region
|
||||
relation_types:
|
||||
- cites
|
||||
- implements
|
||||
- amends
|
||||
- replaces
|
||||
- supplements
|
||||
- issued_by
|
||||
- implemented_by
|
||||
- regulates
|
||||
- defines_spatial
|
||||
- requires_procedure
|
||||
- sets_obligation
|
||||
- prescribes_penalty
|
||||
- applies_to
|
||||
- governs_region
|
||||
confidence_threshold:
|
||||
entity: 0.8
|
||||
relation: 0.7
|
||||
|
||||
logging:
|
||||
level: "INFO"
|
||||
format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
@@ -0,0 +1,246 @@
|
||||
entity_types:
|
||||
Law:
|
||||
prefix: "LAW"
|
||||
description: "全国人大及其常委会制定的法律"
|
||||
required_attributes: [name]
|
||||
optional_attributes:
|
||||
- full_title
|
||||
- promulgation_date
|
||||
- effective_date
|
||||
- status
|
||||
- amendment_dates
|
||||
|
||||
AdministrativeRegulation:
|
||||
prefix: "REG"
|
||||
description: "国务院制定的行政法规"
|
||||
required_attributes: [name]
|
||||
optional_attributes:
|
||||
- full_title
|
||||
- promulgation_date
|
||||
- effective_date
|
||||
- status
|
||||
- order_number
|
||||
|
||||
DepartmentalRule:
|
||||
prefix: "RULE"
|
||||
description: "各部委制定的部门规章"
|
||||
required_attributes: [name]
|
||||
optional_attributes:
|
||||
- full_title
|
||||
- promulgation_date
|
||||
- effective_date
|
||||
- status
|
||||
- order_number
|
||||
|
||||
PolicyDocument:
|
||||
prefix: "DOC"
|
||||
description: "党中央/国务院/主管部门文件、技术标准"
|
||||
required_attributes: [name]
|
||||
optional_attributes:
|
||||
- full_title
|
||||
- issue_date
|
||||
- document_number
|
||||
- document_type
|
||||
- status
|
||||
|
||||
Chapter:
|
||||
prefix: "CH"
|
||||
description: "法律文件的章"
|
||||
required_attributes: [name, number]
|
||||
optional_attributes:
|
||||
- title
|
||||
|
||||
Section:
|
||||
prefix: "SEC"
|
||||
description: "法律文件的节"
|
||||
required_attributes: [name, number]
|
||||
optional_attributes:
|
||||
- title
|
||||
|
||||
Article:
|
||||
prefix: "ART"
|
||||
description: "法律文件的具体条文"
|
||||
required_attributes: [number, text]
|
||||
optional_attributes:
|
||||
- article_type
|
||||
|
||||
GovernmentBody:
|
||||
prefix: "GOV"
|
||||
description: "立法/行政/主管部门"
|
||||
required_attributes: [name]
|
||||
optional_attributes:
|
||||
- full_name
|
||||
- level
|
||||
- type
|
||||
|
||||
LegalSubject:
|
||||
prefix: "SUBJ"
|
||||
description: "法律规制的事项或主体"
|
||||
required_attributes: [name]
|
||||
optional_attributes:
|
||||
- category
|
||||
- description
|
||||
|
||||
SpatialConcept:
|
||||
prefix: "SPAT"
|
||||
description: "国土空间规划中的空间概念"
|
||||
required_attributes: [name]
|
||||
optional_attributes:
|
||||
- type
|
||||
- definition
|
||||
|
||||
AdministrativeProcedure:
|
||||
prefix: "PROC"
|
||||
description: "规划/用地/建设的行政程序"
|
||||
required_attributes: [name]
|
||||
optional_attributes:
|
||||
- type
|
||||
- time_limit
|
||||
- applicable_subjects
|
||||
|
||||
Obligation:
|
||||
prefix: "OBL"
|
||||
description: "法律条文规定的权利、义务或禁止"
|
||||
required_attributes: [text]
|
||||
optional_attributes:
|
||||
- type
|
||||
|
||||
Penalty:
|
||||
prefix: "PEN"
|
||||
description: "违反法律规定的处罚"
|
||||
required_attributes: [type, description]
|
||||
optional_attributes:
|
||||
- applicable_violation
|
||||
- legal_basis
|
||||
|
||||
TimePoint:
|
||||
prefix: "TIME"
|
||||
description: "法规中提到的具体时间"
|
||||
required_attributes: [description]
|
||||
optional_attributes:
|
||||
- year
|
||||
- time_type
|
||||
|
||||
Region:
|
||||
prefix: "REGION"
|
||||
description: "法规适用的区域"
|
||||
required_attributes: [name]
|
||||
optional_attributes:
|
||||
- level
|
||||
- feature
|
||||
|
||||
relationship_types:
|
||||
has_chapter:
|
||||
description: "包含章"
|
||||
source_types: ["Law", "AdministrativeRegulation", "DepartmentalRule", "PolicyDocument"]
|
||||
target_types: ["Chapter"]
|
||||
|
||||
has_section:
|
||||
description: "包含节"
|
||||
source_types: ["Chapter"]
|
||||
target_types: ["Section"]
|
||||
|
||||
contains_article:
|
||||
description: "包含条"
|
||||
source_types: ["Chapter", "Section"]
|
||||
target_types: ["Article"]
|
||||
|
||||
article_in_document:
|
||||
description: "条属于文件"
|
||||
source_types: ["Article"]
|
||||
target_types: ["Law", "AdministrativeRegulation", "DepartmentalRule", "PolicyDocument"]
|
||||
|
||||
cites:
|
||||
description: "引用其他法规"
|
||||
source_types: ["Article", "PolicyDocument"]
|
||||
target_types: ["Law", "AdministrativeRegulation", "DepartmentalRule"]
|
||||
|
||||
implements:
|
||||
description: "实施/落实上位法"
|
||||
source_types: ["PolicyDocument", "DepartmentalRule"]
|
||||
target_types: ["Law", "AdministrativeRegulation"]
|
||||
|
||||
amends:
|
||||
description: "修正/修订"
|
||||
source_types: ["Law", "AdministrativeRegulation", "DepartmentalRule"]
|
||||
target_types: ["Law", "AdministrativeRegulation", "DepartmentalRule"]
|
||||
|
||||
replaces:
|
||||
description: "替代/废止"
|
||||
source_types: ["Law", "AdministrativeRegulation", "DepartmentalRule"]
|
||||
target_types: ["Law", "AdministrativeRegulation", "DepartmentalRule"]
|
||||
|
||||
supplements:
|
||||
description: "补充"
|
||||
source_types: ["PolicyDocument"]
|
||||
target_types: ["Law", "AdministrativeRegulation", "DepartmentalRule"]
|
||||
|
||||
issued_by:
|
||||
description: "发布机关"
|
||||
source_types: ["Law", "AdministrativeRegulation", "DepartmentalRule", "PolicyDocument"]
|
||||
target_types: ["GovernmentBody"]
|
||||
|
||||
implemented_by:
|
||||
description: "实施机关"
|
||||
source_types: ["Article", "AdministrativeProcedure"]
|
||||
target_types: ["GovernmentBody"]
|
||||
|
||||
regulates:
|
||||
description: "规制事项"
|
||||
source_types: ["Article", "Obligation"]
|
||||
target_types: ["LegalSubject"]
|
||||
|
||||
defines_spatial:
|
||||
description: "界定空间概念"
|
||||
source_types: ["Article"]
|
||||
target_types: ["SpatialConcept"]
|
||||
|
||||
applies_to:
|
||||
description: "适用于"
|
||||
source_types: ["Article", "AdministrativeProcedure"]
|
||||
target_types: ["LegalSubject"]
|
||||
|
||||
sets_obligation:
|
||||
description: "定义务"
|
||||
source_types: ["Article"]
|
||||
target_types: ["Obligation"]
|
||||
|
||||
prescribes_penalty:
|
||||
description: "规定处罚"
|
||||
source_types: ["Article", "Obligation"]
|
||||
target_types: ["Penalty"]
|
||||
|
||||
requires_procedure:
|
||||
description: "要求程序"
|
||||
source_types: ["Article"]
|
||||
target_types: ["AdministrativeProcedure"]
|
||||
|
||||
procedure_for:
|
||||
description: "程序对应事项"
|
||||
source_types: ["AdministrativeProcedure"]
|
||||
target_types: ["LegalSubject"]
|
||||
|
||||
governs_region:
|
||||
description: "管辖区域"
|
||||
source_types: ["Law", "AdministrativeRegulation", "DepartmentalRule"]
|
||||
target_types: ["Region"]
|
||||
|
||||
effective_timeline:
|
||||
description: "生效时间线"
|
||||
source_types: ["Law", "AdministrativeRegulation", "DepartmentalRule"]
|
||||
target_types: ["TimePoint"]
|
||||
|
||||
related_to:
|
||||
description: "相关"
|
||||
source_types: ["Any"]
|
||||
target_types: ["Any"]
|
||||
|
||||
standard_for:
|
||||
description: "标准适用于"
|
||||
source_types: ["PolicyDocument"]
|
||||
target_types: ["LegalSubject"]
|
||||
|
||||
category_index:
|
||||
description: "分类索引"
|
||||
source_types: ["Law", "AdministrativeRegulation", "DepartmentalRule", "PolicyDocument"]
|
||||
target_types: ["LegalSubject"]
|
||||
@@ -0,0 +1,139 @@
|
||||
"""
|
||||
Neo4j数据导入脚本 - 从CSV导入法规知识图谱数据
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
from neo4j import GraphDatabase
|
||||
import json
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Neo4jImporter:
|
||||
"""Neo4j数据导入器"""
|
||||
|
||||
def __init__(self, uri: str, username: str, password: str, database: str = 'neo4j'):
|
||||
self.driver = GraphDatabase.driver(uri, auth=(username, password))
|
||||
self.database = database
|
||||
logger.info(f"连接Neo4j: {uri}")
|
||||
|
||||
def close(self):
|
||||
self.driver.close()
|
||||
|
||||
def _run_query(self, query, parameters=None):
|
||||
with self.driver.session(database=self.database) as session:
|
||||
result = session.run(query, parameters or {})
|
||||
return [record.data() for record in result]
|
||||
|
||||
def create_constraints(self):
|
||||
"""创建约束和索引"""
|
||||
schema_file = Path(__file__).parent / 'schema.cypher'
|
||||
if schema_file.exists():
|
||||
with open(schema_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
for line in content.split('\n'):
|
||||
line = line.strip()
|
||||
if line.startswith('CREATE ') and not line.startswith('//'):
|
||||
try:
|
||||
self._run_query(line)
|
||||
logger.info(f"执行: {line[:60]}...")
|
||||
except Exception as e:
|
||||
if 'already exists' not in str(e):
|
||||
logger.warning(f"约束跳过: {e}")
|
||||
|
||||
def import_nodes(self, nodes_csv: str):
|
||||
"""导入节点"""
|
||||
nodes_df = pd.read_csv(nodes_csv, encoding='utf-8-sig')
|
||||
logger.info(f"导入 {len(nodes_df)} 个节点")
|
||||
|
||||
# 按类型分批导入
|
||||
for node_type in nodes_df['type'].unique():
|
||||
type_df = nodes_df[nodes_df['type'] == node_type]
|
||||
label = node_type
|
||||
|
||||
for _, row in type_df.iterrows():
|
||||
props = {}
|
||||
try:
|
||||
props = json.loads(row.get('properties', '{}'))
|
||||
except:
|
||||
pass
|
||||
props['id'] = row['id']
|
||||
props['name'] = row['label']
|
||||
|
||||
query = f"MERGE (n:{label} {{id: $id}}) SET n += $props"
|
||||
self._run_query(query, {'id': row['id'], 'props': props})
|
||||
|
||||
logger.info(f" {label}: {len(type_df)} 个")
|
||||
|
||||
def import_relationships(self, rels_csv: str):
|
||||
"""导入关系"""
|
||||
rels_df = pd.read_csv(rels_csv, encoding='utf-8-sig')
|
||||
logger.info(f"导入 {len(rels_df)} 个关系")
|
||||
|
||||
imported = 0
|
||||
for _, row in rels_df.iterrows():
|
||||
rel_type = row['type']
|
||||
props = {}
|
||||
try:
|
||||
props = json.loads(row.get('properties', '{}'))
|
||||
except:
|
||||
pass
|
||||
|
||||
query = f"""
|
||||
MATCH (a {{id: $source}})
|
||||
MATCH (b {{id: $target}})
|
||||
MERGE (a)-[r:{rel_type}]->(b)
|
||||
SET r += $props
|
||||
"""
|
||||
try:
|
||||
self._run_query(query, {
|
||||
'source': row['source'],
|
||||
'target': row['target'],
|
||||
'props': props
|
||||
})
|
||||
imported += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"关系导入失败: {row['source']} -> {row['target']} ({rel_type}): {e}")
|
||||
|
||||
logger.info(f"成功导入 {imported}/{len(rels_df)} 个关系")
|
||||
|
||||
def import_all(self, nodes_csv: str, rels_csv: str):
|
||||
"""完整导入流程"""
|
||||
logger.info("开始导入...")
|
||||
self.create_constraints()
|
||||
self.import_nodes(nodes_csv)
|
||||
self.import_relationships(rels_csv)
|
||||
logger.info("导入完成！")
|
||||
|
||||
|
||||
def main():
|
||||
import yaml
|
||||
|
||||
config_file = Path(__file__).parent.parent / 'config' / 'legal_config.yaml'
|
||||
with open(config_file, 'r', encoding='utf-8') as f:
|
||||
config = yaml.safe_load(f)
|
||||
|
||||
neo4j_config = config['neo4j']
|
||||
importer = Neo4jImporter(
|
||||
neo4j_config['uri'],
|
||||
neo4j_config['username'],
|
||||
neo4j_config['password'],
|
||||
neo4j_config['database']
|
||||
)
|
||||
|
||||
output_dir = Path(__file__).parent.parent / 'output'
|
||||
nodes_csv = str(output_dir / 'nodes_merged.csv')
|
||||
rels_csv = str(output_dir / 'rels_merged.csv')
|
||||
|
||||
try:
|
||||
importer.import_all(nodes_csv, rels_csv)
|
||||
finally:
|
||||
importer.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,62 @@
|
||||
// Neo4j数据模型 - 城市规划法律法规知识图谱
|
||||
|
||||
// ============================================================
|
||||
// 约束和索引
|
||||
// ============================================================
|
||||
|
||||
CREATE CONSTRAINT law_id IF NOT EXISTS FOR (l:Law) REQUIRE l.law_id IS UNIQUE;
|
||||
CREATE CONSTRAINT reg_id IF NOT EXISTS FOR (r:AdministrativeRegulation) REQUIRE r.reg_id IS UNIQUE;
|
||||
CREATE CONSTRAINT rule_id IF NOT EXISTS FOR (r:DepartmentalRule) REQUIRE r.rule_id IS UNIQUE;
|
||||
CREATE CONSTRAINT doc_id IF NOT EXISTS FOR (d:PolicyDocument) REQUIRE d.doc_id IS UNIQUE;
|
||||
CREATE CONSTRAINT chapter_id IF NOT EXISTS FOR (c:Chapter) REQUIRE c.chapter_id IS UNIQUE;
|
||||
CREATE CONSTRAINT section_id IF NOT EXISTS FOR (s:Section) REQUIRE s.section_id IS UNIQUE;
|
||||
CREATE CONSTRAINT article_id IF NOT EXISTS FOR (a:Article) REQUIRE a.article_id IS UNIQUE;
|
||||
CREATE CONSTRAINT body_id IF NOT EXISTS FOR (g:GovernmentBody) REQUIRE g.body_id IS UNIQUE;
|
||||
CREATE CONSTRAINT subject_id IF NOT EXISTS FOR (s:LegalSubject) REQUIRE s.subject_id IS UNIQUE;
|
||||
CREATE CONSTRAINT concept_id IF NOT EXISTS FOR (s:SpatialConcept) REQUIRE s.concept_id IS UNIQUE;
|
||||
CREATE CONSTRAINT proc_id IF NOT EXISTS FOR (p:AdministrativeProcedure) REQUIRE p.proc_id IS UNIQUE;
|
||||
|
||||
// 索引
|
||||
CREATE INDEX law_name IF NOT EXISTS FOR (l:Law) ON (l.name);
|
||||
CREATE INDEX law_status IF NOT EXISTS FOR (l:Law) ON (l.status);
|
||||
CREATE INDEX article_type IF NOT EXISTS FOR (a:Article) ON (a.article_type);
|
||||
CREATE INDEX body_name IF NOT EXISTS FOR (g:GovernmentBody) ON (g.name);
|
||||
CREATE INDEX subject_category IF NOT EXISTS FOR (s:LegalSubject) ON (s.category);
|
||||
|
||||
// 全文搜索
|
||||
CREATE FULLTEXT INDEX law_search IF NOT EXISTS FOR (l:Law) ON EACH [l.name, l.full_title];
|
||||
CREATE FULLTEXT INDEX article_search IF NOT EXISTS FOR (a:Article) ON EACH [a.text];
|
||||
|
||||
// ============================================================
|
||||
// 导入命令示例
|
||||
// ============================================================
|
||||
|
||||
// LOAD CSV WITH HEADERS FROM 'file:///nodes.csv' AS row
|
||||
// WITH row WHERE row.type = 'Law'
|
||||
// CREATE (:Law {law_id: row.id, name: row.label, full_title: apoc.text.replace(row.label, '"', '')});
|
||||
//
|
||||
// LOAD CSV WITH HEADERS FROM 'file:///rels.csv' AS row
|
||||
// MATCH (a {id: row.source})
|
||||
// MATCH (b {id: row.target})
|
||||
// CALL apoc.create.relationship(a, row.type, {}, b) YIELD rel
|
||||
// RETURN rel;
|
||||
|
||||
// ============================================================
|
||||
// 常用查询
|
||||
// ============================================================
|
||||
|
||||
// 查询某法律的所有条文
|
||||
// MATCH (l:Law {name: '城乡规划法'})-[:HAS_CHAPTER]->(ch)-[:CONTAINS_ARTICLE]->(art:Article)
|
||||
// RETURN ch.title, art.number, art.text;
|
||||
|
||||
// 查询引用某法律的法规
|
||||
// MATCH (d)-[r:CITES]->(l:Law {name: '城乡规划法'})
|
||||
// RETURN d.name, type(r), r.context;
|
||||
|
||||
// 查询涉及某主题的所有条文
|
||||
// MATCH (art:Article)-[:REGULATES]->(s:LegalSubject {name: '建设用地'})
|
||||
// RETURN art.text;
|
||||
|
||||
// 查询某机关发布的所有文件
|
||||
// MATCH (d)-[:ISSUED_BY]->(g:GovernmentBody {name: '自然资源部'})
|
||||
// RETURN d.name, labels(d);
|
||||
@@ -0,0 +1,213 @@
|
||||
{
|
||||
"ontology": {
|
||||
"name": "Urban Planning Law Knowledge Graph Ontology",
|
||||
"version": "1.0",
|
||||
"description": "城市规划法律法规知识图谱本体模型",
|
||||
"created_date": "2026-05-13",
|
||||
"concepts": [
|
||||
{
|
||||
"id": "Law",
|
||||
"name": "法律",
|
||||
"description": "全国人大及其常委会制定的法律",
|
||||
"properties": [
|
||||
{"name": "law_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "name", "type": "string", "required": true},
|
||||
{"name": "full_title", "type": "string"},
|
||||
{"name": "promulgation_date", "type": "string"},
|
||||
{"name": "effective_date", "type": "string"},
|
||||
{"name": "amendment_dates", "type": "array<string>"},
|
||||
{"name": "status", "type": "string", "enum": ["现行有效", "修订", "废止", "草案"]},
|
||||
{"name": "legal_hierarchy", "type": "string", "default": "法律"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "AdministrativeRegulation",
|
||||
"name": "行政法规",
|
||||
"description": "国务院制定的行政法规",
|
||||
"properties": [
|
||||
{"name": "reg_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "name", "type": "string", "required": true},
|
||||
{"name": "full_title", "type": "string"},
|
||||
{"name": "promulgation_date", "type": "string"},
|
||||
{"name": "effective_date", "type": "string"},
|
||||
{"name": "status", "type": "string"},
|
||||
{"name": "order_number", "type": "string"},
|
||||
{"name": "legal_hierarchy", "type": "string", "default": "行政法规"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "DepartmentalRule",
|
||||
"name": "部门规章",
|
||||
"description": "各部委制定的部门规章",
|
||||
"properties": [
|
||||
{"name": "rule_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "name", "type": "string", "required": true},
|
||||
{"name": "full_title", "type": "string"},
|
||||
{"name": "promulgation_date", "type": "string"},
|
||||
{"name": "effective_date", "type": "string"},
|
||||
{"name": "status", "type": "string"},
|
||||
{"name": "order_number", "type": "string"},
|
||||
{"name": "legal_hierarchy", "type": "string", "default": "部门规章"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "PolicyDocument",
|
||||
"name": "政策文件",
|
||||
"description": "党中央/国务院/主管部门文件、技术标准",
|
||||
"properties": [
|
||||
{"name": "doc_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "name", "type": "string", "required": true},
|
||||
{"name": "full_title", "type": "string"},
|
||||
{"name": "issue_date", "type": "string"},
|
||||
{"name": "document_number", "type": "string"},
|
||||
{"name": "document_type", "type": "string", "enum": ["意见", "通知", "纲要", "指南", "标准", "规范", "办法", "规定", "决定", "批复", "函", "其他"]},
|
||||
{"name": "status", "type": "string"},
|
||||
{"name": "legal_hierarchy", "type": "string", "default": "政策文件"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "Chapter",
|
||||
"name": "章",
|
||||
"description": "法律文件的章",
|
||||
"properties": [
|
||||
{"name": "chapter_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "number", "type": "string"},
|
||||
{"name": "title", "type": "string"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "Section",
|
||||
"name": "节",
|
||||
"description": "法律文件的节",
|
||||
"properties": [
|
||||
{"name": "section_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "number", "type": "string"},
|
||||
{"name": "title", "type": "string"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "Article",
|
||||
"name": "条",
|
||||
"description": "法律文件的具体条文",
|
||||
"properties": [
|
||||
{"name": "article_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "number", "type": "string"},
|
||||
{"name": "text", "type": "string"},
|
||||
{"name": "article_type", "type": "string", "enum": ["管理性", "禁止性", "授权性", "程序性", "处罚性", "定义性"]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "GovernmentBody",
|
||||
"name": "政府机构",
|
||||
"description": "立法/行政/主管部门",
|
||||
"properties": [
|
||||
{"name": "body_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "name", "type": "string", "required": true},
|
||||
{"name": "full_name", "type": "string"},
|
||||
{"name": "level", "type": "string", "enum": ["国家", "省级", "市级", "县级"]},
|
||||
{"name": "type", "type": "string", "enum": ["立法机关", "行政机关", "主管部门", "地方机关"]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "LegalSubject",
|
||||
"name": "法定事项",
|
||||
"description": "法律规制的事项或主体",
|
||||
"properties": [
|
||||
{"name": "subject_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "name", "type": "string", "required": true},
|
||||
{"name": "category", "type": "string", "enum": ["土地", "规划", "建设", "环境", "文物", "交通", "防灾", "生态", "基础设施", "更新", "保护", "测绘", "登记", "处罚", "住房", "市政", "园林", "水资源", "矿业"]},
|
||||
{"name": "description", "type": "string"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "SpatialConcept",
|
||||
"name": "空间概念",
|
||||
"description": "国土空间规划中的空间概念",
|
||||
"properties": [
|
||||
{"name": "concept_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "name", "type": "string", "required": true},
|
||||
{"name": "type", "type": "string", "enum": ["国土空间", "生态红线", "永久基本农田", "城镇开发边界", "规划区", "建设用地", "农用地", "未利用地", "历史文化保护区", "控制线", "城市更新单元"]},
|
||||
{"name": "definition", "type": "string"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "AdministrativeProcedure",
|
||||
"name": "行政程序",
|
||||
"description": "规划/用地/建设的行政程序",
|
||||
"properties": [
|
||||
{"name": "proc_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "name", "type": "string", "required": true},
|
||||
{"name": "type", "type": "string", "enum": ["审批", "核准", "备案", "登记", "许可", "验收", "监督", "听证", "征收", "拆迁", "出让", "转让"]},
|
||||
{"name": "time_limit", "type": "string"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "Obligation",
|
||||
"name": "权利义务",
|
||||
"description": "法律条文规定的权利、义务或禁止",
|
||||
"properties": [
|
||||
{"name": "oblig_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "text", "type": "string"},
|
||||
{"name": "type", "type": "string", "enum": ["权利", "义务", "禁止", "限制", "鼓励"]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "Penalty",
|
||||
"name": "法律责任",
|
||||
"description": "违反法律规定的处罚",
|
||||
"properties": [
|
||||
{"name": "penalty_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "type", "type": "string", "enum": ["行政处罚", "刑事处罚", "民事责任", "行政处分"]},
|
||||
{"name": "description", "type": "string"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "TimePoint",
|
||||
"name": "时间节点",
|
||||
"description": "法规中提到的具体时间",
|
||||
"properties": [
|
||||
{"name": "time_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "year", "type": "integer"},
|
||||
{"name": "description", "type": "string"},
|
||||
{"name": "time_type", "type": "string", "enum": ["生效日期", "废止日期", "修订日期", "过渡期", "截止日期"]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "Region",
|
||||
"name": "区域",
|
||||
"description": "法规适用的区域",
|
||||
"properties": [
|
||||
{"name": "region_id", "type": "string", "required": true, "unique": true},
|
||||
{"name": "name", "type": "string", "required": true},
|
||||
{"name": "level", "type": "string", "enum": ["全国", "区域", "省", "市", "县", "特定区域"]}
|
||||
]
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{"name": "has_chapter", "description": "包含章", "from": "Law/AdministrativeRegulation/DepartmentalRule/PolicyDocument", "to": "Chapter", "properties": [{"name": "order_number", "type": "integer"}]},
|
||||
{"name": "has_section", "description": "包含节", "from": "Chapter", "to": "Section", "properties": [{"name": "order_number", "type": "integer"}]},
|
||||
{"name": "contains_article", "description": "包含条", "from": "Chapter/Section", "to": "Article", "properties": [{"name": "order_number", "type": "integer"}]},
|
||||
{"name": "article_in_document", "description": "条属于文件", "from": "Article", "to": "Law/AdministrativeRegulation/DepartmentalRule/PolicyDocument", "properties": []},
|
||||
{"name": "cites", "description": "引用", "from": "Article/PolicyDocument", "to": "Law/AdministrativeRegulation/DepartmentalRule", "properties": [{"name": "context", "type": "string"}]},
|
||||
{"name": "implements", "description": "实施/落实", "from": "PolicyDocument/DepartmentalRule", "to": "Law/AdministrativeRegulation", "properties": [{"name": "scope", "type": "string"}]},
|
||||
{"name": "amends", "description": "修正/修订", "from": "Law/Regulation/Rule", "to": "Law/Regulation/Rule", "properties": [{"name": "amendment_date", "type": "string"}]},
|
||||
{"name": "replaces", "description": "替代/废止", "from": "Law/Regulation/Rule", "to": "Law/Regulation/Rule", "properties": [{"name": "replacement_date", "type": "string"}]},
|
||||
{"name": "supplements", "description": "补充", "from": "PolicyDocument", "to": "Law/Regulation/Rule", "properties": [{"name": "scope", "type": "string"}]},
|
||||
{"name": "issued_by", "description": "发布机关", "from": "Law/Regulation/Rule/PolicyDocument", "to": "GovernmentBody", "properties": [{"name": "role", "type": "string"}]},
|
||||
{"name": "implemented_by", "description": "实施机关", "from": "Article/AdministrativeProcedure", "to": "GovernmentBody", "properties": [{"name": "role", "type": "string"}]},
|
||||
{"name": "regulates", "description": "规制", "from": "Article/Obligation", "to": "LegalSubject", "properties": [{"name": "regulation_type", "type": "string"}]},
|
||||
{"name": "defines_spatial", "description": "界定空间", "from": "Article", "to": "SpatialConcept", "properties": [{"name": "scope", "type": "string"}]},
|
||||
{"name": "applies_to", "description": "适用于", "from": "Article/AdministrativeProcedure", "to": "LegalSubject", "properties": [{"name": "context", "type": "string"}]},
|
||||
{"name": "sets_obligation", "description": "定义务", "from": "Article", "to": "Obligation", "properties": [{"name": "obligation_type", "type": "string"}]},
|
||||
{"name": "prescribes_penalty", "description": "规定处罚", "from": "Article/Obligation", "to": "Penalty", "properties": [{"name": "conditions", "type": "string"}]},
|
||||
{"name": "requires_procedure", "description": "要求程序", "from": "Article", "to": "AdministrativeProcedure", "properties": [{"name": "mandatory", "type": "string"}]},
|
||||
{"name": "procedure_for", "description": "程序对应事项", "from": "AdministrativeProcedure", "to": "LegalSubject", "properties": [{"name": "purpose", "type": "string"}]},
|
||||
{"name": "procedure_step", "description": "程序步骤", "from": "AdministrativeProcedure", "to": "AdministrativeProcedure", "properties": [{"name": "step_order", "type": "integer"}]},
|
||||
{"name": "governs_region", "description": "管辖区域", "from": "Law/Regulation/Rule", "to": "Region", "properties": [{"name": "scope", "type": "string"}]},
|
||||
{"name": "effective_timeline", "description": "生效时间线", "from": "Law/Regulation/Rule", "to": "TimePoint", "properties": [{"name": "timeline_type", "type": "string"}]},
|
||||
{"name": "related_to", "description": "相关", "from": "Any", "to": "Any", "properties": [{"name": "description", "type": "string"}]},
|
||||
{"name": "standard_for", "description": "标准适用于", "from": "PolicyDocument", "to": "LegalSubject", "properties": [{"name": "standard_type", "type": "string"}]},
|
||||
{"name": "category_index", "description": "分类索引", "from": "Law/Regulation/Rule/PolicyDocument", "to": "LegalSubject", "properties": [{"name": "source", "type": "string"}]}
|
||||
]
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,29 @@
|
||||
# 城市规划法律法规知识图谱构建 - 依赖包
|
||||
|
||||
# LLM相关
|
||||
langchain-deepseek>=0.1.0
|
||||
langchain-core>=0.1.0
|
||||
|
||||
# 文档读取
|
||||
python-docx>=1.1.0
|
||||
|
||||
# Neo4j
|
||||
neo4j>=5.15.0
|
||||
|
||||
# 数据处理
|
||||
pandas>=2.0.0
|
||||
numpy>=1.24.0
|
||||
|
||||
# 文本相似度
|
||||
python-Levenshtein>=0.23.0
|
||||
|
||||
# 可视化
|
||||
networkx>=3.1
|
||||
matplotlib>=3.7.0
|
||||
|
||||
# 配置
|
||||
pyyaml>=6.0
|
||||
python-dotenv>=1.0.0
|
||||
|
||||
# 工具
|
||||
tqdm>=4.65.0
|
||||
@@ -0,0 +1,221 @@
|
||||
"""
|
||||
Word文档读取器 - 读取城市更新法规数据库中所有.docx文件
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any
|
||||
|
||||
try:
|
||||
from docx import Document
|
||||
except ImportError:
|
||||
print("错误: 请先安装 python-docx")
|
||||
print("运行: pip install python-docx")
|
||||
raise
|
||||
|
||||
|
||||
class DocxReader:
|
||||
"""Word文档读取器"""
|
||||
|
||||
# 目录类别映射
|
||||
CATEGORY_MAP = {
|
||||
'0知识图谱目录': {'id': 0, 'type': 'Index', 'name': '知识图谱目录'},
|
||||
'1法律': {'id': 1, 'type': 'Law', 'name': '法律'},
|
||||
'2行政法规': {'id': 2, 'type': 'AdministrativeRegulation', 'name': '行政法规'},
|
||||
'3部门规章': {'id': 3, 'type': 'DepartmentalRule', 'name': '部门规章'},
|
||||
'4党中央国务院文件': {'id': 4, 'type': 'PolicyDocument', 'name': '党中央国务院文件'},
|
||||
'5国家主管部门文件': {'id': 5, 'type': 'PolicyDocument', 'name': '国家主管部门文件'},
|
||||
'6主要技术标准': {'id': 6, 'type': 'PolicyDocument', 'name': '主要技术标准'},
|
||||
}
|
||||
|
||||
def __init__(self, source_dir: str):
|
||||
self.source_dir = Path(source_dir)
|
||||
self.logger = self._setup_logger()
|
||||
self.documents = []
|
||||
|
||||
def _setup_logger(self):
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
return logging.getLogger(__name__)
|
||||
|
||||
def _extract_text_from_docx(self, filepath: Path) -> str:
|
||||
"""从Word文档提取纯文本"""
|
||||
doc = Document(str(filepath))
|
||||
paragraphs = []
|
||||
for para in doc.paragraphs:
|
||||
text = para.text.strip()
|
||||
if text:
|
||||
paragraphs.append(text)
|
||||
|
||||
# 也提取表格中的文本
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
row_text = []
|
||||
for cell in row.cells:
|
||||
cell_text = cell.text.strip()
|
||||
if cell_text:
|
||||
row_text.append(cell_text)
|
||||
if row_text:
|
||||
paragraphs.append(' | '.join(row_text))
|
||||
|
||||
return '\n'.join(paragraphs)
|
||||
|
||||
def _parse_filename_info(self, filename: str) -> Dict[str, Any]:
|
||||
"""从文件名解析基本信息"""
|
||||
info = {
|
||||
'original_filename': filename,
|
||||
'document_number': None,
|
||||
'document_name': None,
|
||||
'is_draft': False,
|
||||
}
|
||||
|
||||
# 去掉扩展名
|
||||
name_without_ext = Path(filename).stem
|
||||
|
||||
# 检测是否为草案/征求意见稿
|
||||
if '草案' in name_without_ext or '征求意见稿' in name_without_ext:
|
||||
info['is_draft'] = True
|
||||
|
||||
# 尝试提取编号前缀 (如 "3-中华人民共和国城乡规划法")
|
||||
match = re.match(r'^(\d+)[-—]\s*(.+)$', name_without_ext)
|
||||
if match:
|
||||
info['file_number'] = int(match.group(1))
|
||||
name_part = match.group(2)
|
||||
else:
|
||||
name_part = name_without_ext
|
||||
|
||||
# 提取书名号中的名称
|
||||
title_match = re.search(r'《(.+?)》', name_part)
|
||||
if title_match:
|
||||
info['document_name'] = title_match.group(1)
|
||||
else:
|
||||
# 去掉常见前缀
|
||||
cleaned = re.sub(r'^(中华人民共和国|国务院|国土资源部|建设部|住房城乡建设部|自然资源部)\s*', '', name_part)
|
||||
info['document_name'] = cleaned if cleaned else name_part
|
||||
|
||||
return info
|
||||
|
||||
def read_all_documents(self) -> List[Dict[str, Any]]:
|
||||
"""读取所有Word文档"""
|
||||
self.logger.info(f"开始读取文档目录: {self.source_dir}")
|
||||
self.documents = []
|
||||
errors = []
|
||||
|
||||
for subdir_name, category_info in self.CATEGORY_MAP.items():
|
||||
subdir = self.source_dir / subdir_name
|
||||
if not subdir.exists():
|
||||
self.logger.warning(f"子目录不存在: {subdir}")
|
||||
continue
|
||||
|
||||
docx_files = sorted(subdir.glob('*.docx'))
|
||||
self.logger.info(f" {subdir_name}: 发现 {len(docx_files)} 个文件")
|
||||
|
||||
for docx_file in docx_files:
|
||||
try:
|
||||
text = self._extract_text_from_docx(docx_file)
|
||||
filename_info = self._parse_filename_info(docx_file.name)
|
||||
|
||||
doc_record = {
|
||||
'doc_id': f"{category_info['type'][:3].upper()}-{len(self.documents):04d}",
|
||||
'filename': docx_file.name,
|
||||
'category_id': category_info['id'],
|
||||
'category_name': category_info['name'],
|
||||
'document_type': category_info['type'],
|
||||
'subdirectory': subdir_name,
|
||||
'raw_text': text,
|
||||
'text_length': len(text),
|
||||
'conversion_timestamp': datetime.now().isoformat(),
|
||||
**filename_info,
|
||||
}
|
||||
|
||||
self.documents.append(doc_record)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"读取失败: {docx_file.name} - {str(e)}"
|
||||
self.logger.error(error_msg)
|
||||
errors.append({
|
||||
'filename': docx_file.name,
|
||||
'subdirectory': subdir_name,
|
||||
'error': str(e)
|
||||
})
|
||||
|
||||
self.logger.info(f"读取完成: {len(self.documents)} 个文档, {len(errors)} 个错误")
|
||||
return self.documents
|
||||
|
||||
def save_to_json(self, output_path: str):
|
||||
"""保存到JSON文件"""
|
||||
output = Path(output_path)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
result = {
|
||||
'metadata': {
|
||||
'total_documents': len(self.documents),
|
||||
'source_dir': str(self.source_dir),
|
||||
'created_at': datetime.now().isoformat(),
|
||||
'category_distribution': {},
|
||||
},
|
||||
'documents': self.documents,
|
||||
}
|
||||
|
||||
# 统计分类分布
|
||||
for doc in self.documents:
|
||||
cat = doc['category_name']
|
||||
result['metadata']['category_distribution'][cat] = \
|
||||
result['metadata']['category_distribution'].get(cat, 0) + 1
|
||||
|
||||
with open(output, 'w', encoding='utf-8') as f:
|
||||
json.dump(result, f, indent=2, ensure_ascii=False)
|
||||
|
||||
self.logger.info(f"已保存到: {output}")
|
||||
return result
|
||||
|
||||
def print_summary(self):
|
||||
"""打印摘要"""
|
||||
print("\n" + "=" * 60)
|
||||
print("文档读取摘要")
|
||||
print("=" * 60)
|
||||
print(f"总文档数: {len(self.documents)}")
|
||||
|
||||
# 按类别统计
|
||||
category_counts = {}
|
||||
total_chars = 0
|
||||
for doc in self.documents:
|
||||
cat = doc['category_name']
|
||||
category_counts[cat] = category_counts.get(cat, 0) + 1
|
||||
total_chars += doc['text_length']
|
||||
|
||||
print("\n类别分布:")
|
||||
for cat, count in sorted(category_counts.items()):
|
||||
print(f" {cat}: {count} 个")
|
||||
|
||||
print(f"\n总字符数: {total_chars:,}")
|
||||
print(f"平均字符数: {total_chars // max(len(self.documents), 1):,}")
|
||||
|
||||
# 文本长度分布
|
||||
lengths = [doc['text_length'] for doc in self.documents]
|
||||
if lengths:
|
||||
print(f"\n文本长度:")
|
||||
print(f" 最短: {min(lengths):,} 字符")
|
||||
print(f" 最长: {max(lengths):,} 字符")
|
||||
print(f" 中位数: {sorted(lengths)[len(lengths)//2]:,} 字符")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
def main():
|
||||
source_dir = r"E:\Project\SI\2026_KG_PlanningLaw\data\城市规划法律法规\城市更新法规数据库"
|
||||
output_path = r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output\all_documents.json"
|
||||
|
||||
reader = DocxReader(source_dir)
|
||||
reader.read_all_documents()
|
||||
reader.save_to_json(output_path)
|
||||
reader.print_summary()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,314 @@
|
||||
"""
|
||||
从结构化法规数据生成知识图谱CSV文件（nodes.csv + rels.csv）
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any, Tuple
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 节点类型到ID前缀的映射
|
||||
TYPE_PREFIX = {
|
||||
'Law': 'LAW',
|
||||
'AdministrativeRegulation': 'REG',
|
||||
'DepartmentalRule': 'RULE',
|
||||
'PolicyDocument': 'DOC',
|
||||
'Index': 'DOC',
|
||||
'Chapter': 'CH',
|
||||
'Section': 'SEC',
|
||||
'Article': 'ART',
|
||||
'GovernmentBody': 'GOV',
|
||||
}
|
||||
|
||||
|
||||
def generate_id(prefix: str, text: str) -> str:
|
||||
"""根据文本哈希生成ID"""
|
||||
return f"{prefix}-{abs(hash(text)) % 100000:05d}"
|
||||
|
||||
|
||||
def build_document_nodes(doc: Dict[str, Any], doc_index: int) -> Tuple[List[Dict], List[Dict]]:
|
||||
"""从单个文档构建节点和关系"""
|
||||
nodes = []
|
||||
rels = []
|
||||
rel_id = 0
|
||||
|
||||
# 文档类型映射
|
||||
doc_type = doc['document_type']
|
||||
if doc_type == 'Index':
|
||||
doc_type = 'PolicyDocument'
|
||||
|
||||
# 1. 创建文档节点
|
||||
doc_id = generate_id(TYPE_PREFIX.get(doc_type, 'DOC'), doc['title'])
|
||||
|
||||
doc_node = {
|
||||
'id': doc_id,
|
||||
'label': doc['title'],
|
||||
'type': doc_type,
|
||||
'properties': json.dumps({
|
||||
'full_title': doc.get('document_name', ''),
|
||||
'promulgation_date': doc.get('promulgation_date'),
|
||||
'effective_date': doc.get('effective_date'),
|
||||
'document_number': doc.get('document_number'),
|
||||
'is_draft': doc.get('is_draft', False),
|
||||
'category': doc.get('category_name', ''),
|
||||
'filename': doc.get('filename', ''),
|
||||
'text_length': doc.get('text_length', 0),
|
||||
'article_count': doc.get('article_count', 0),
|
||||
}, ensure_ascii=False)
|
||||
}
|
||||
nodes.append(doc_node)
|
||||
|
||||
# 2. 创建发布机关节点和关系
|
||||
if doc.get('issuing_body'):
|
||||
body_name = doc['issuing_body']
|
||||
body_id = generate_id('GOV', body_name)
|
||||
|
||||
# 检查是否已存在
|
||||
body_node = {
|
||||
'id': body_id,
|
||||
'label': body_name,
|
||||
'type': 'GovernmentBody',
|
||||
'properties': json.dumps({}, ensure_ascii=False)
|
||||
}
|
||||
nodes.append(body_node)
|
||||
|
||||
rel_id += 1
|
||||
rels.append({
|
||||
'source': doc_id,
|
||||
'target': body_id,
|
||||
'type': 'ISSUED_BY',
|
||||
'properties': json.dumps({}, ensure_ascii=False)
|
||||
})
|
||||
|
||||
# 3. 创建章节条文节点和层次关系
|
||||
for ch_idx, chapter in enumerate(doc.get('chapters', [])):
|
||||
ch_title = chapter.get('title', f'第{ch_idx+1}章')
|
||||
ch_id = generate_id('CH', f"{doc_id}_{ch_title}")
|
||||
|
||||
ch_node = {
|
||||
'id': ch_id,
|
||||
'label': ch_title,
|
||||
'type': 'Chapter',
|
||||
'properties': json.dumps({
|
||||
'name': chapter.get('name', ''),
|
||||
'number': str(ch_idx + 1),
|
||||
}, ensure_ascii=False)
|
||||
}
|
||||
nodes.append(ch_node)
|
||||
|
||||
# 文档→章
|
||||
rel_id += 1
|
||||
rels.append({
|
||||
'source': doc_id,
|
||||
'target': ch_id,
|
||||
'type': 'HAS_CHAPTER',
|
||||
'properties': json.dumps({'order_number': ch_idx + 1}, ensure_ascii=False)
|
||||
})
|
||||
|
||||
# 章下的条文
|
||||
for art_idx, article in enumerate(chapter.get('articles', [])):
|
||||
art_number = article.get('number', str(art_idx + 1))
|
||||
art_text = article.get('text', '')
|
||||
art_id = generate_id('ART', f"{doc_id}_{art_number}")
|
||||
|
||||
art_node = {
|
||||
'id': art_id,
|
||||
'label': f"第{art_number}条",
|
||||
'type': 'Article',
|
||||
'properties': json.dumps({
|
||||
'number': art_number,
|
||||
'text': art_text[:500],
|
||||
'article_type': article.get('article_type', ''),
|
||||
}, ensure_ascii=False)
|
||||
}
|
||||
nodes.append(art_node)
|
||||
|
||||
# 章→条
|
||||
rel_id += 1
|
||||
rels.append({
|
||||
'source': ch_id,
|
||||
'target': art_id,
|
||||
'type': 'CONTAINS_ARTICLE',
|
||||
'properties': json.dumps({'order_number': art_idx + 1}, ensure_ascii=False)
|
||||
})
|
||||
|
||||
# 条→文档
|
||||
rel_id += 1
|
||||
rels.append({
|
||||
'source': art_id,
|
||||
'target': doc_id,
|
||||
'type': 'ARTICLE_IN_DOCUMENT',
|
||||
'properties': json.dumps({}, ensure_ascii=False)
|
||||
})
|
||||
|
||||
# 节
|
||||
for sec_idx, section in enumerate(chapter.get('sections', [])):
|
||||
sec_title = section.get('title', f'第{sec_idx+1}节')
|
||||
sec_id = generate_id('SEC', f"{doc_id}_{ch_title}_{sec_title}")
|
||||
|
||||
sec_node = {
|
||||
'id': sec_id,
|
||||
'label': sec_title,
|
||||
'type': 'Section',
|
||||
'properties': json.dumps({
|
||||
'name': section.get('name', ''),
|
||||
'number': str(sec_idx + 1),
|
||||
}, ensure_ascii=False)
|
||||
}
|
||||
nodes.append(sec_node)
|
||||
|
||||
# 章→节
|
||||
rel_id += 1
|
||||
rels.append({
|
||||
'source': ch_id,
|
||||
'target': sec_id,
|
||||
'type': 'HAS_SECTION',
|
||||
'properties': json.dumps({'order_number': sec_idx + 1}, ensure_ascii=False)
|
||||
})
|
||||
|
||||
# 节下的条文
|
||||
for art_idx, article in enumerate(section.get('articles', [])):
|
||||
art_number = article.get('number', str(art_idx + 1))
|
||||
art_text = article.get('text', '')
|
||||
art_id = generate_id('ART', f"{doc_id}_{art_number}")
|
||||
|
||||
art_node = {
|
||||
'id': art_id,
|
||||
'label': f"第{art_number}条",
|
||||
'type': 'Article',
|
||||
'properties': json.dumps({
|
||||
'number': art_number,
|
||||
'text': art_text[:500],
|
||||
'article_type': article.get('article_type', ''),
|
||||
}, ensure_ascii=False)
|
||||
}
|
||||
nodes.append(art_node)
|
||||
|
||||
# 节→条
|
||||
rel_id += 1
|
||||
rels.append({
|
||||
'source': sec_id,
|
||||
'target': art_id,
|
||||
'type': 'CONTAINS_ARTICLE',
|
||||
'properties': json.dumps({'order_number': art_idx + 1}, ensure_ascii=False)
|
||||
})
|
||||
|
||||
# 条→文档
|
||||
rel_id += 1
|
||||
rels.append({
|
||||
'source': art_id,
|
||||
'target': doc_id,
|
||||
'type': 'ARTICLE_IN_DOCUMENT',
|
||||
'properties': json.dumps({}, ensure_ascii=False)
|
||||
})
|
||||
|
||||
return nodes, rels
|
||||
|
||||
|
||||
def deduplicate_nodes(nodes: List[Dict]) -> List[Dict]:
|
||||
"""去重节点（基于id）"""
|
||||
seen = {}
|
||||
for node in nodes:
|
||||
if node['id'] not in seen:
|
||||
seen[node['id']] = node
|
||||
else:
|
||||
# 合并properties
|
||||
existing = seen[node['id']]
|
||||
if existing['properties'] == '{}' and node['properties'] != '{}':
|
||||
existing['properties'] = node['properties']
|
||||
return list(seen.values())
|
||||
|
||||
|
||||
def generate_report(nodes_df: pd.DataFrame, rels_df: pd.DataFrame, output_dir: Path):
|
||||
"""生成统计报告"""
|
||||
lines = ["# 结构化CSV提取报告\n"]
|
||||
lines.append(f"生成时间: {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
|
||||
|
||||
# 节点统计
|
||||
lines.append("## 节点统计\n\n")
|
||||
lines.append(f"**节点总数**: {len(nodes_df)}\n\n")
|
||||
|
||||
node_type_counts = nodes_df['type'].value_counts().sort_index()
|
||||
lines.append("| 节点类型 | 数量 | 占比 |\n")
|
||||
lines.append("|---------|------|------|\n")
|
||||
for nt, count in node_type_counts.items():
|
||||
lines.append(f"| {nt} | {count} | {count/len(nodes_df)*100:.1f}% |\n")
|
||||
|
||||
# 关系统计
|
||||
lines.append("\n## 关系统计\n\n")
|
||||
lines.append(f"**关系总数**: {len(rels_df)}\n\n")
|
||||
|
||||
rel_type_counts = rels_df['type'].value_counts().sort_index()
|
||||
lines.append("| 关系类型 | 数量 | 占比 |\n")
|
||||
lines.append("|---------|------|------|\n")
|
||||
for rt, count in rel_type_counts.items():
|
||||
lines.append(f"| {rt} | {count} | {count/len(rels_df)*100:.1f}% |\n")
|
||||
|
||||
report_path = output_dir / 'extraction_report_structured.md'
|
||||
with open(report_path, 'w', encoding='utf-8') as f:
|
||||
f.writelines(lines)
|
||||
|
||||
print(f"\n报告已保存: {report_path}")
|
||||
|
||||
|
||||
def main():
|
||||
input_path = Path(r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output\structured_documents.json")
|
||||
output_dir = Path(r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"读取结构化文档: {input_path}")
|
||||
with open(input_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
documents = data['documents']
|
||||
logger.info(f"共 {len(documents)} 个文档")
|
||||
|
||||
all_nodes = []
|
||||
all_rels = []
|
||||
|
||||
for i, doc in enumerate(documents):
|
||||
nodes, rels = build_document_nodes(doc, i)
|
||||
all_nodes.extend(nodes)
|
||||
all_rels.extend(rels)
|
||||
|
||||
if (i + 1) % 50 == 0:
|
||||
logger.info(f" 已处理 {i + 1}/{len(documents)}")
|
||||
|
||||
# 去重
|
||||
all_nodes = deduplicate_nodes(all_nodes)
|
||||
|
||||
# 保存
|
||||
nodes_df = pd.DataFrame(all_nodes)
|
||||
rels_df = pd.DataFrame(all_rels)
|
||||
|
||||
nodes_path = output_dir / 'nodes_structured.csv'
|
||||
rels_path = output_dir / 'rels_structured.csv'
|
||||
|
||||
nodes_df.to_csv(nodes_path, index=False, encoding='utf-8-sig')
|
||||
rels_df.to_csv(rels_path, index=False, encoding='utf-8-sig')
|
||||
|
||||
logger.info(f"节点已保存: {nodes_path} ({len(nodes_df)} 行)")
|
||||
logger.info(f"关系已保存: {rels_path} ({len(rels_df)} 行)")
|
||||
|
||||
# 统计
|
||||
print(f"\n{'='*60}")
|
||||
print("结构化CSV提取完成")
|
||||
print(f"{'='*60}")
|
||||
print(f"节点总数: {len(nodes_df)}")
|
||||
for nt, count in nodes_df['type'].value_counts().sort_index().items():
|
||||
print(f" - {nt}: {count}")
|
||||
print(f"关系总数: {len(rels_df)}")
|
||||
for rt, count in rels_df['type'].value_counts().sort_index().items():
|
||||
print(f" - {rt}: {count}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
generate_report(nodes_df, rels_df, output_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,354 @@
|
||||
"""
|
||||
法规元数据解析器 - 从文档文本中解析章节结构、发布机关、日期等
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
|
||||
class LegalMetadataExtractor:
|
||||
"""法规元数据解析器"""
|
||||
|
||||
def __init__(self):
|
||||
self.logger = self._setup_logger()
|
||||
|
||||
def _setup_logger(self):
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
return logging.getLogger(__name__)
|
||||
|
||||
def _extract_document_title(self, text: str, filename: str) -> str:
|
||||
"""提取文档标题"""
|
||||
# 尝试从书名号中提取
|
||||
match = re.search(r'《(.+?)》', text[:2000])
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
# 尝试从前几行中提取
|
||||
lines = text[:500].split('\n')
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
# 过滤掉编号前缀
|
||||
cleaned = re.sub(r'^\d+[-—]\s*', '', line)
|
||||
if len(cleaned) > 4 and len(cleaned) < 100:
|
||||
return cleaned
|
||||
|
||||
# 从文件名提取
|
||||
name = Path(filename).stem
|
||||
name = re.sub(r'^\d+[-—]\s*', '', name)
|
||||
return name
|
||||
|
||||
def _extract_issuing_body(self, text: str) -> Optional[str]:
|
||||
"""提取发布机关"""
|
||||
# 常见模式：在文末签名块
|
||||
patterns = [
|
||||
r'(?:发布|公布|印发)[^\n]*?[机关部院会委局厅处]\s*[：:]\s*(.+?)(?:\n|$)',
|
||||
r'^[\s]*(.{4,20}(?:部|委员会|院|局|厅|处|办公室|小组))\s*$',
|
||||
]
|
||||
|
||||
# 从后往前搜索（发布机关通常在文末）
|
||||
text_end = text[-3000:] if len(text) > 3000 else text
|
||||
lines = text_end.split('\n')
|
||||
|
||||
for line in reversed(lines):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
# 匹配机关名称
|
||||
if re.match(r'^.{2,15}(?:部|委员会|院|局|厅|处|办公室|小组|政府|大会|人大常委会)$', line):
|
||||
return line
|
||||
|
||||
# 从标题区域搜索
|
||||
text_start = text[:1000]
|
||||
body_patterns = [
|
||||
r'((?:全国人民代表大会(?:常务委员会)?|国务院|.{2,10}部|.{2,10}委员会|.{2,10}局|.{2,10}厅))\s*(?:令|公告|通知|制定)',
|
||||
r'(.{2,10}(?:部|委员会|局|厅))\s*(?:令|公告|通知|印发)',
|
||||
]
|
||||
for pattern in body_patterns:
|
||||
match = re.search(pattern, text_start)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
return None
|
||||
|
||||
def _extract_dates(self, text: str) -> Dict[str, Optional[str]]:
|
||||
"""提取日期信息"""
|
||||
dates = {'promulgation_date': None, 'effective_date': None}
|
||||
|
||||
# 搜索发布日期
|
||||
date_patterns = [
|
||||
r'(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日\s*(?:起?\s*)?(?:施行|实施|生效|执行)',
|
||||
r'自\s*(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日\s*起?\s*(?:施行|实施|生效|执行)',
|
||||
]
|
||||
for pattern in date_patterns:
|
||||
match = re.search(pattern, text)
|
||||
if match:
|
||||
dates['effective_date'] = f"{match.group(1)}-{match.group(2).zfill(2)}-{match.group(3).zfill(2)}"
|
||||
break
|
||||
|
||||
# 搜索公布日期（通常在文末）
|
||||
pub_patterns = [
|
||||
r'(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日\s*(?:公布|发布|印发)',
|
||||
r'(?:公布|发布|印发)\s*[：:]*\s*(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日',
|
||||
]
|
||||
text_end = text[-2000:] if len(text) > 2000 else text
|
||||
for pattern in pub_patterns:
|
||||
match = re.search(pattern, text_end)
|
||||
if match:
|
||||
dates['promulgation_date'] = f"{match.group(1)}-{match.group(2).zfill(2)}-{match.group(3).zfill(2)}"
|
||||
break
|
||||
|
||||
# 如果没有明确日期，尝试从文末找任意日期
|
||||
if not dates['promulgation_date']:
|
||||
match = re.findall(r'(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日', text_end)
|
||||
if match:
|
||||
last_date = match[-1]
|
||||
dates['promulgation_date'] = f"{last_date[0]}-{last_date[1].zfill(2)}-{last_date[2].zfill(2)}"
|
||||
|
||||
return dates
|
||||
|
||||
def _extract_document_number(self, text: str) -> Optional[str]:
|
||||
"""提取文号"""
|
||||
patterns = [
|
||||
r'[（(]\s*(\d{4})\s*[）)]\s*[^\s]*?\s*第?\s*(\d+)\s*号',
|
||||
r'第?\s*(\d+)\s*号\s*[令公告通知]',
|
||||
r'([A-Za-z一-鿿]+[〔(]\d{4}[）)][^\s]*?号)',
|
||||
r'([国发|国办发|建发|自然资发|国土资发|建城|建规|住建|建住房|建村|办发|发改委][〔(]\d{4}[）)][^\s]*?号)',
|
||||
]
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, text[:2000])
|
||||
if match:
|
||||
return match.group(0).strip()
|
||||
return None
|
||||
|
||||
def _parse_chapter_structure(self, text: str) -> List[Dict[str, Any]]:
|
||||
"""解析章节结构"""
|
||||
chapters = []
|
||||
current_chapter = None
|
||||
current_section = None
|
||||
current_articles = []
|
||||
|
||||
# 按行处理
|
||||
lines = text.split('\n')
|
||||
|
||||
def flush_articles():
|
||||
nonlocal current_articles
|
||||
result = current_articles
|
||||
current_articles = []
|
||||
return result
|
||||
|
||||
for line in lines:
|
||||
line_stripped = line.strip()
|
||||
if not line_stripped:
|
||||
continue
|
||||
|
||||
# 检测章标题
|
||||
ch_match = re.match(r'^第[一二三四五六七八九十百]+[章节部分]\s*(.*)$', line_stripped)
|
||||
if ch_match:
|
||||
# 先保存之前的章/节的文章
|
||||
if current_section:
|
||||
current_section['articles'] = flush_articles()
|
||||
elif current_chapter:
|
||||
current_chapter['articles'].extend(flush_articles())
|
||||
|
||||
current_chapter = {
|
||||
'title': line_stripped,
|
||||
'name': ch_match.group(1).strip() if ch_match.group(1) else line_stripped,
|
||||
'sections': [],
|
||||
'articles': [],
|
||||
}
|
||||
chapters.append(current_chapter)
|
||||
current_section = None
|
||||
continue
|
||||
|
||||
# 检测节标题
|
||||
sec_match = re.match(r'^第[一二三四五六七八九十百]+节\s*(.*)$', line_stripped)
|
||||
if sec_match and current_chapter:
|
||||
if current_section:
|
||||
current_section['articles'] = flush_articles()
|
||||
|
||||
current_section = {
|
||||
'title': line_stripped,
|
||||
'name': sec_match.group(1).strip() if sec_match.group(1) else line_stripped,
|
||||
'articles': [],
|
||||
}
|
||||
current_chapter['sections'].append(current_section)
|
||||
continue
|
||||
|
||||
# 检测条文
|
||||
art_match = re.match(r'^第[一二三四五六七八九十百零千]+条\s*(.*)$', line_stripped)
|
||||
if art_match:
|
||||
article = {
|
||||
'number': line_stripped.split('条')[0].replace('第', ''),
|
||||
'text': art_match.group(1).strip(),
|
||||
'full_text': line_stripped,
|
||||
}
|
||||
current_articles.append(article)
|
||||
continue
|
||||
|
||||
# 续接上一条
|
||||
if current_articles and line_stripped:
|
||||
current_articles[-1]['text'] += ' ' + line_stripped
|
||||
current_articles[-1]['full_text'] += ' ' + line_stripped
|
||||
|
||||
# 刷新最后的文章
|
||||
if current_section:
|
||||
current_section['articles'].extend(flush_articles())
|
||||
elif current_chapter:
|
||||
current_chapter['articles'].extend(flush_articles())
|
||||
|
||||
return chapters
|
||||
|
||||
def _classify_article_type(self, text: str) -> str:
|
||||
"""分类条文类型"""
|
||||
if re.search(r'不得|禁止|严禁', text):
|
||||
return '禁止性'
|
||||
elif re.search(r'可以|有权|依法享有', text):
|
||||
return '授权性'
|
||||
elif re.search(r'应当|必须|须|应当依法', text):
|
||||
return '管理性'
|
||||
elif re.search(r'申请|审批|备案|登记|许可|核准', text):
|
||||
return '程序性'
|
||||
elif re.search(r'罚款|责令|没收|吊销|刑事|处分', text):
|
||||
return '处罚性'
|
||||
else:
|
||||
return '定义性'
|
||||
|
||||
def process_document(self, doc: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""处理单个文档"""
|
||||
text = doc.get('raw_text', '')
|
||||
filename = doc.get('filename', '')
|
||||
|
||||
# 提取元数据
|
||||
title = self._extract_document_title(text, filename)
|
||||
issuing_body = self._extract_issuing_body(text)
|
||||
dates = self._extract_dates(text)
|
||||
doc_number = self._extract_document_number(text)
|
||||
|
||||
# 解析章节结构
|
||||
chapters = self._parse_chapter_structure(text)
|
||||
|
||||
# 统计条文数
|
||||
total_articles = 0
|
||||
for ch in chapters:
|
||||
total_articles += len(ch.get('articles', []))
|
||||
for sec in ch.get('sections', []):
|
||||
total_articles += len(sec.get('articles', []))
|
||||
|
||||
# 分类条文
|
||||
for ch in chapters:
|
||||
for art in ch.get('articles', []):
|
||||
art['article_type'] = self._classify_article_type(art['text'])
|
||||
for sec in ch.get('sections', []):
|
||||
for art in sec.get('articles', []):
|
||||
art['article_type'] = self._classify_article_type(art['text'])
|
||||
|
||||
result = {
|
||||
'doc_id': doc['doc_id'],
|
||||
'document_type': doc['document_type'],
|
||||
'category_name': doc['category_name'],
|
||||
'title': title,
|
||||
'document_name': doc.get('document_name', title),
|
||||
'issuing_body': issuing_body,
|
||||
'promulgation_date': dates['promulgation_date'],
|
||||
'effective_date': dates['effective_date'],
|
||||
'document_number': doc_number,
|
||||
'is_draft': doc.get('is_draft', False),
|
||||
'text_length': doc['text_length'],
|
||||
'filename': filename,
|
||||
'chapter_count': len(chapters),
|
||||
'article_count': total_articles,
|
||||
'chapters': chapters,
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
def process_all(self, input_path: str, output_path: str) -> List[Dict[str, Any]]:
|
||||
"""处理所有文档"""
|
||||
self.logger.info(f"读取文档数据: {input_path}")
|
||||
|
||||
with open(input_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
documents = data['documents']
|
||||
self.logger.info(f"共 {len(documents)} 个文档待处理")
|
||||
|
||||
results = []
|
||||
for i, doc in enumerate(documents):
|
||||
if doc['text_length'] == 0:
|
||||
self.logger.warning(f"跳过空文档: {doc['filename']}")
|
||||
continue
|
||||
|
||||
try:
|
||||
result = self.process_document(doc)
|
||||
results.append(result)
|
||||
|
||||
if (i + 1) % 50 == 0:
|
||||
self.logger.info(f" 已处理 {i + 1}/{len(documents)}")
|
||||
except Exception as e:
|
||||
self.logger.error(f"处理失败: {doc['filename']} - {str(e)}")
|
||||
|
||||
# 保存
|
||||
output = Path(output_path)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
save_data = {
|
||||
'metadata': {
|
||||
'total_documents': len(results),
|
||||
'processed_at': datetime.now().isoformat(),
|
||||
},
|
||||
'documents': results,
|
||||
}
|
||||
|
||||
with open(output, 'w', encoding='utf-8') as f:
|
||||
json.dump(save_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
self.logger.info(f"已保存到: {output}")
|
||||
self._print_summary(results)
|
||||
|
||||
return results
|
||||
|
||||
def _print_summary(self, results: List[Dict]):
|
||||
"""打印摘要"""
|
||||
total_articles = sum(r['article_count'] for r in results)
|
||||
total_chapters = sum(r['chapter_count'] for r in results)
|
||||
docs_with_body = sum(1 for r in results if r['issuing_body'])
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print("法规元数据解析摘要")
|
||||
print(f"{'='*60}")
|
||||
print(f"处理文档数: {len(results)}")
|
||||
print(f"解析出章数: {total_chapters}")
|
||||
print(f"解析出条文数: {total_articles}")
|
||||
print(f"提取发布机关: {docs_with_body}/{len(results)} ({docs_with_body/max(len(results),1)*100:.1f}%)")
|
||||
|
||||
# 按类型统计
|
||||
type_counts = {}
|
||||
for r in results:
|
||||
t = r['document_type']
|
||||
type_counts[t] = type_counts.get(t, 0) + 1
|
||||
print("\n按类型统计:")
|
||||
for t, c in sorted(type_counts.items()):
|
||||
print(f" {t}: {c} 个")
|
||||
|
||||
print(f"{'='*60}")
|
||||
|
||||
|
||||
def main():
|
||||
input_path = r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output\all_documents.json"
|
||||
output_path = r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output\structured_documents.json"
|
||||
|
||||
extractor = LegalMetadataExtractor()
|
||||
extractor.process_all(input_path, output_path)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
法规知识图谱可视化工具
|
||||
使用networkx和matplotlib绘制知识图谱
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
import networkx as nx
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import json
|
||||
|
||||
|
||||
def setup_chinese_font():
|
||||
"""设置中文字体"""
|
||||
for font in ['Microsoft YaHei', 'SimHei', 'SimSun', 'KaiTi']:
|
||||
try:
|
||||
plt.rcParams['font.sans-serif'] = [font]
|
||||
plt.rcParams['axes.unicode_minus'] = False
|
||||
return
|
||||
except:
|
||||
continue
|
||||
|
||||
|
||||
# 节点类型颜色
|
||||
NODE_COLORS = {
|
||||
'Law': '#E74C3C',
|
||||
'AdministrativeRegulation': '#E67E22',
|
||||
'DepartmentalRule': '#F1C40F',
|
||||
'PolicyDocument': '#2ECC71',
|
||||
'Chapter': '#3498DB',
|
||||
'Section': '#9B59B6',
|
||||
'Article': '#1ABC9C',
|
||||
'GovernmentBody': '#8E44AD',
|
||||
'LegalSubject': '#E91E63',
|
||||
'SpatialConcept': '#00BCD4',
|
||||
'AdministrativeProcedure': '#FF9800',
|
||||
'Obligation': '#F44336',
|
||||
'Penalty': '#795548',
|
||||
'TimePoint': '#607D8B',
|
||||
'Region': '#4CAF50',
|
||||
}
|
||||
|
||||
# 节点类型大小
|
||||
NODE_SIZES = {
|
||||
'Law': 600,
|
||||
'AdministrativeRegulation': 500,
|
||||
'DepartmentalRule': 400,
|
||||
'PolicyDocument': 350,
|
||||
'Chapter': 250,
|
||||
'Section': 200,
|
||||
'Article': 100,
|
||||
'GovernmentBody': 400,
|
||||
'LegalSubject': 300,
|
||||
'SpatialConcept': 250,
|
||||
'AdministrativeProcedure': 200,
|
||||
'Obligation': 150,
|
||||
'Penalty': 150,
|
||||
'TimePoint': 100,
|
||||
'Region': 150,
|
||||
}
|
||||
|
||||
|
||||
def load_data(nodes_csv, rels_csv):
|
||||
"""加载数据"""
|
||||
nodes_df = pd.read_csv(nodes_csv, encoding='utf-8-sig')
|
||||
rels_df = pd.read_csv(rels_csv, encoding='utf-8-sig')
|
||||
print(f"加载节点: {len(nodes_df)}, 关系: {len(rels_df)}")
|
||||
return nodes_df, rels_df
|
||||
|
||||
|
||||
def build_graph(nodes_df, rels_df):
|
||||
"""构建NetworkX图"""
|
||||
G = nx.DiGraph()
|
||||
|
||||
for _, row in nodes_df.iterrows():
|
||||
node_id = row['id']
|
||||
label = str(row['label'])
|
||||
node_type = row['type']
|
||||
|
||||
display_label = label[:12] + '...' if len(label) > 12 else label
|
||||
|
||||
G.add_node(node_id,
|
||||
label=display_label,
|
||||
full_label=label,
|
||||
node_type=node_type,
|
||||
color=NODE_COLORS.get(node_type, '#CCCCCC'),
|
||||
size=NODE_SIZES.get(node_type, 150))
|
||||
|
||||
for _, row in rels_df.iterrows():
|
||||
source = row['source']
|
||||
target = row['target']
|
||||
rel_type = row['type']
|
||||
if source in G.nodes() and target in G.nodes():
|
||||
G.add_edge(source, target, rel_type=rel_type)
|
||||
|
||||
print(f"图: {G.number_of_nodes()} 节点, {G.number_of_edges()} 边")
|
||||
return G
|
||||
|
||||
|
||||
def draw_graph(G, output_path, title="法规知识图谱", max_nodes=300):
|
||||
"""绘制图谱"""
|
||||
if G.number_of_nodes() > max_nodes:
|
||||
degrees = dict(G.degree())
|
||||
top_nodes = sorted(degrees, key=degrees.get, reverse=True)[:max_nodes]
|
||||
G = G.subgraph(top_nodes).copy()
|
||||
|
||||
plt.figure(figsize=(24, 18))
|
||||
pos = nx.spring_layout(G, k=2.5, iterations=50, seed=42)
|
||||
|
||||
# 边
|
||||
nx.draw_networkx_edges(G, pos, alpha=0.2, width=0.5, edge_color='gray',
|
||||
arrows=True, arrowsize=8)
|
||||
|
||||
# 按类型绘制节点
|
||||
node_types = {}
|
||||
for node, data in G.nodes(data=True):
|
||||
nt = data.get('node_type', 'Unknown')
|
||||
node_types.setdefault(nt, []).append(node)
|
||||
|
||||
for nt, nodes in node_types.items():
|
||||
color = NODE_COLORS.get(nt, '#CCCCCC')
|
||||
size = NODE_SIZES.get(nt, 150)
|
||||
nx.draw_networkx_nodes(G, pos, nodelist=nodes, node_color=color,
|
||||
node_size=size, alpha=0.8, edgecolors='white', linewidths=1)
|
||||
|
||||
# 标签（只显示非Article类型）
|
||||
labels = {n: d['label'] for n, d in G.nodes(data=True)
|
||||
if d.get('node_type') != 'Article'}
|
||||
if len(labels) <= 150:
|
||||
nx.draw_networkx_labels(G, pos, labels=labels, font_size=7, font_weight='bold')
|
||||
|
||||
# 图例
|
||||
patches = [mpatches.Patch(color=NODE_COLORS[nt], label=nt)
|
||||
for nt in node_types if nt in NODE_COLORS]
|
||||
plt.legend(handles=patches, loc='upper right', fontsize=10, framealpha=0.9)
|
||||
|
||||
plt.title(title, fontsize=16, fontweight='bold', pad=20)
|
||||
plt.axis('off')
|
||||
plt.tight_layout()
|
||||
plt.savefig(output_path, dpi=150, bbox_inches='tight')
|
||||
plt.close()
|
||||
print(f"保存: {output_path}")
|
||||
|
||||
|
||||
def draw_subgraphs(G, output_dir):
|
||||
"""绘制子图"""
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 1. 文档层次图
|
||||
doc_types = ['Law', 'AdministrativeRegulation', 'DepartmentalRule', 'PolicyDocument', 'Chapter']
|
||||
G1_nodes = [n for n, d in G.nodes(data=True) if d.get('node_type') in doc_types]
|
||||
if G1_nodes:
|
||||
G1 = G.subgraph(G1_nodes).copy()
|
||||
draw_graph(G1, output_dir / 'kg_document_hierarchy.png', '法规文档层次结构')
|
||||
|
||||
# 2. 引用网络
|
||||
ref_rels = ['CITES', 'IMPLEMENTS', 'AMENDS', 'REPLACES', 'SUPPLEMENTS']
|
||||
ref_edges = [(u, v) for u, v, d in G.edges(data=True) if d.get('rel_type') in ref_rels]
|
||||
if ref_edges:
|
||||
G2 = G.edge_subgraph(ref_edges).copy()
|
||||
draw_graph(G2, output_dir / 'kg_citation_network.png', '法规引用网络')
|
||||
|
||||
# 3. 主题规制图
|
||||
subject_types = ['LegalSubject', 'SpatialConcept']
|
||||
subject_nodes = [n for n, d in G.nodes(data=True) if d.get('node_type') in subject_types]
|
||||
if subject_nodes:
|
||||
neighbors = set(subject_nodes)
|
||||
for node in subject_nodes:
|
||||
neighbors.update(G.predecessors(node))
|
||||
neighbors.update(G.successors(node))
|
||||
G3 = G.subgraph(neighbors).copy()
|
||||
draw_graph(G3, output_dir / 'kg_subject_regulation.png', '法规主题规制关系')
|
||||
|
||||
# 4. 机构关系图
|
||||
gov_nodes = [n for n, d in G.nodes(data=True) if d.get('node_type') == 'GovernmentBody']
|
||||
if gov_nodes:
|
||||
neighbors = set(gov_nodes)
|
||||
for node in gov_nodes:
|
||||
neighbors.update(G.predecessors(node))
|
||||
neighbors.update(G.successors(node))
|
||||
G4 = G.subgraph(neighbors).copy()
|
||||
draw_graph(G4, output_dir / 'kg_government_body.png', '政府机构关系')
|
||||
|
||||
|
||||
def print_statistics(G):
|
||||
"""打印统计"""
|
||||
print(f"\n{'='*60}")
|
||||
print("图谱统计")
|
||||
print(f"{'='*60}")
|
||||
print(f"节点: {G.number_of_nodes()}, 边: {G.number_of_edges()}")
|
||||
|
||||
type_counts = {}
|
||||
for _, d in G.nodes(data=True):
|
||||
nt = d.get('node_type', 'Unknown')
|
||||
type_counts[nt] = type_counts.get(nt, 0) + 1
|
||||
|
||||
print("\n节点类型:")
|
||||
for nt, c in sorted(type_counts.items()):
|
||||
print(f" {nt}: {c} ({c/G.number_of_nodes()*100:.1f}%)")
|
||||
|
||||
rel_counts = {}
|
||||
for _, _, d in G.edges(data=True):
|
||||
rt = d.get('rel_type', 'Unknown')
|
||||
rel_counts[rt] = rel_counts.get(rt, 0) + 1
|
||||
|
||||
print("\n关系类型:")
|
||||
for rt, c in sorted(rel_counts.items()):
|
||||
print(f" {rt}: {c}")
|
||||
|
||||
degrees = [d for _, d in G.degree()]
|
||||
print(f"\n平均连接度: {np.mean(degrees):.2f}")
|
||||
top = sorted(G.degree(), key=lambda x: x[1], reverse=True)[:10]
|
||||
print("连接度最高:")
|
||||
for node, deg in top:
|
||||
data = G.nodes[node]
|
||||
print(f" [{data.get('node_type')}] {data.get('full_label', node)}: {deg}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
|
||||
def main():
|
||||
setup_chinese_font()
|
||||
|
||||
base = Path(r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output")
|
||||
|
||||
# 优先使用合并后的文件
|
||||
nodes_csv = base / 'nodes_merged.csv' if (base / 'nodes_merged.csv').exists() else base / 'nodes_structured.csv'
|
||||
rels_csv = base / 'rels_merged.csv' if (base / 'rels_merged.csv').exists() else base / 'rels_structured.csv'
|
||||
|
||||
nodes_df, rels_df = load_data(nodes_csv, rels_csv)
|
||||
G = build_graph(nodes_df, rels_df)
|
||||
print_statistics(G)
|
||||
draw_subgraphs(G, base / 'visualizations')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,176 @@
|
||||
"""
|
||||
实体规范化器 - 去重、ID生成和规范化
|
||||
"""
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
import logging
|
||||
from typing import Dict, List, Any, Optional
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LegalEntityNormalizer:
|
||||
"""法规实体规范化器"""
|
||||
|
||||
# 法律领域常见同义词映射
|
||||
SYNONYM_MAP = {
|
||||
'城乡规划': '城市规划',
|
||||
'住建部': '住房和城乡建设部',
|
||||
'建设部': '住房和城乡建设部',
|
||||
'自然资源部': '自然资源部',
|
||||
'国土部': '自然资源部',
|
||||
'国土资源部': '自然资源部',
|
||||
'环保部': '生态环境部',
|
||||
'环境保护部': '生态环境部',
|
||||
'国家发改委': '国家发展和改革委员会',
|
||||
'发改委': '国家发展和改革委员会',
|
||||
'国务院': '国务院',
|
||||
# 第一轮旧名称 → 第二轮新名称
|
||||
'GovernmentBody': 'Agency',
|
||||
'LegalSubject': 'LegalObject',
|
||||
'SpatialConcept': 'SpatialObject',
|
||||
'AdministrativeProcedure': 'Procedure',
|
||||
# 城市更新领域同义词
|
||||
'老旧小区': '老旧小区',
|
||||
'城中村': '城中村',
|
||||
'棚户区': '棚户区',
|
||||
'旧城改造': '城市更新',
|
||||
'旧区改造': '城市更新',
|
||||
'三旧改造': '城市更新',
|
||||
}
|
||||
|
||||
def __init__(self, ontology_file: str = None):
|
||||
self.entity_registry = {}
|
||||
self.text_to_id_map = {}
|
||||
self.type_counters = {}
|
||||
|
||||
if ontology_file:
|
||||
self._load_ontology(ontology_file)
|
||||
|
||||
logger.info("LegalEntityNormalizer 初始化完成")
|
||||
|
||||
def _load_ontology(self, ontology_file: str):
|
||||
if yaml is None:
|
||||
return
|
||||
path = Path(ontology_file)
|
||||
if path.exists():
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
self.ontology = yaml.safe_load(f)
|
||||
|
||||
def normalize_text(self, text: str) -> str:
|
||||
"""标准化文本"""
|
||||
if not text:
|
||||
return ""
|
||||
text = text.strip()
|
||||
text = text.replace(' ', ' ').replace('\xa0', ' ')
|
||||
return text
|
||||
|
||||
def generate_entity_id(self, entity_type: str, entity_text: str) -> str:
|
||||
"""生成实体ID"""
|
||||
type_prefixes = {
|
||||
# 第二轮本体：城市更新法规政策工具箱
|
||||
'Agency': 'AGENCY',
|
||||
'LegalObject': 'OBJ',
|
||||
'SpatialObject': 'SPAT',
|
||||
'RenewalScene': 'SCENE',
|
||||
'PolicyTool': 'TOOL',
|
||||
'ToolCategory': 'TCAT',
|
||||
'Procedure': 'PROC',
|
||||
'Obligation': 'OBL',
|
||||
'Condition': 'COND',
|
||||
'Constraint': 'CONST',
|
||||
'Penalty': 'PEN',
|
||||
'TimePoint': 'TIME',
|
||||
'Region': 'REGION',
|
||||
# 兼容第一轮旧名称
|
||||
'GovernmentBody': 'AGENCY',
|
||||
'LegalSubject': 'OBJ',
|
||||
'SpatialConcept': 'SPAT',
|
||||
'AdministrativeProcedure': 'PROC',
|
||||
}
|
||||
prefix = type_prefixes.get(entity_type, 'ENT')
|
||||
|
||||
if entity_type not in self.type_counters:
|
||||
self.type_counters[entity_type] = 0
|
||||
self.type_counters[entity_type] += 1
|
||||
|
||||
text_hash = abs(hash(entity_text)) % 100000
|
||||
counter = self.type_counters[entity_type]
|
||||
return f"{prefix}-{text_hash:05d}-{counter}"
|
||||
|
||||
def normalize_entity(self, raw_entity: Dict[str, Any]) -> Optional[str]:
|
||||
"""规范化单个实体"""
|
||||
entity_text = raw_entity.get('text', '')
|
||||
entity_type = raw_entity.get('type', '')
|
||||
attributes = raw_entity.get('attributes', {})
|
||||
|
||||
if not entity_text or not entity_type:
|
||||
return None
|
||||
|
||||
# 标准化文本
|
||||
normalized_text = self.normalize_text(entity_text)
|
||||
|
||||
# 同义词映射
|
||||
if normalized_text in self.SYNONYM_MAP:
|
||||
normalized_text = self.SYNONYM_MAP[normalized_text]
|
||||
|
||||
# 检查是否已存在
|
||||
key = f"{entity_type}:{normalized_text}"
|
||||
if key in self.text_to_id_map:
|
||||
return self.text_to_id_map[key]
|
||||
|
||||
# 生成新ID
|
||||
entity_id = self.generate_entity_id(entity_type, normalized_text)
|
||||
|
||||
# 注册
|
||||
self.entity_registry[entity_id] = {
|
||||
'canonical_name': normalized_text,
|
||||
'display_name': entity_text,
|
||||
'type': entity_type,
|
||||
'attributes': attributes,
|
||||
}
|
||||
self.text_to_id_map[key] = entity_id
|
||||
|
||||
return entity_id
|
||||
|
||||
def normalize_batch(self, extraction_results: List[Dict]) -> Dict[str, str]:
|
||||
"""批量规范化"""
|
||||
for result in extraction_results:
|
||||
if not result or 'entities' not in result:
|
||||
continue
|
||||
for entity in result.get('entities', []):
|
||||
self.normalize_entity(entity)
|
||||
|
||||
logger.info(f"规范化完成: {len(self.entity_registry)} 个唯一实体")
|
||||
return self.text_to_id_map.copy()
|
||||
|
||||
def get_entity_nodes(self) -> List[Dict[str, Any]]:
|
||||
"""获取所有实体节点"""
|
||||
nodes = []
|
||||
for entity_id, entity_data in self.entity_registry.items():
|
||||
node = {
|
||||
'id': entity_id,
|
||||
'label': entity_data['display_name'],
|
||||
'type': entity_data['type'],
|
||||
'properties': json.dumps(entity_data['attributes'], ensure_ascii=False)
|
||||
}
|
||||
nodes.append(node)
|
||||
return nodes
|
||||
|
||||
def get_statistics(self) -> Dict[str, Any]:
|
||||
"""获取统计信息"""
|
||||
stats = {
|
||||
'total_entities': len(self.entity_registry),
|
||||
'entities_by_type': {},
|
||||
}
|
||||
for entity_data in self.entity_registry.values():
|
||||
et = entity_data['type']
|
||||
stats['entities_by_type'][et] = stats['entities_by_type'].get(et, 0) + 1
|
||||
return stats
|
||||
@@ -0,0 +1,189 @@
|
||||
"""
|
||||
关系构建器 - 从LLM抽取结果构建关系
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RelationshipBuilder:
|
||||
"""关系构建器"""
|
||||
|
||||
# LLM关系类型到大写关系类型的映射
|
||||
REL_TYPE_MAP = {
|
||||
# 文档层次
|
||||
'has_chapter': 'HAS_CHAPTER',
|
||||
'has_section': 'HAS_SECTION',
|
||||
'has_article': 'HAS_ARTICLE',
|
||||
'has_clause': 'HAS_CLAUSE',
|
||||
'article_in_document': 'ARTICLE_IN_DOCUMENT',
|
||||
# 法条依据
|
||||
'cites': 'CITES',
|
||||
'implements': 'IMPLEMENTS',
|
||||
'amends': 'AMENDS',
|
||||
'replaces': 'REPLACES',
|
||||
'supplements': 'SUPPLEMENTS',
|
||||
'basis_for_planning': 'BASIS_FOR_PLANNING',
|
||||
# 主体关系
|
||||
'issued_by': 'ISSUED_BY',
|
||||
'implemented_by': 'IMPLEMENTED_BY',
|
||||
'assigned_to': 'ASSIGNED_TO',
|
||||
# 语义规制
|
||||
'regulates': 'REGULATES',
|
||||
'defines_object': 'DEFINES_OBJECT',
|
||||
'applies_to': 'APPLIES_TO',
|
||||
'sets_obligation': 'SETS_OBLIGATION',
|
||||
'sets_condition': 'SETS_CONDITION',
|
||||
'sets_constraint': 'SETS_CONSTRAINT',
|
||||
'prescribes_penalty': 'PRESCRIBES_PENALTY',
|
||||
# 政策工具
|
||||
'extracts_tool': 'EXTRACTS_TOOL',
|
||||
'belongs_to_category': 'BELONGS_TO_CATEGORY',
|
||||
'applies_to_scene': 'APPLIES_TO_SCENE',
|
||||
'targets_object': 'TARGETS_OBJECT',
|
||||
'coordinates_with': 'COORDINATES_WITH',
|
||||
'supports_tool': 'SUPPORTS_TOOL',
|
||||
# 程序
|
||||
'requires_procedure': 'REQUIRES_PROCEDURE',
|
||||
'procedure_for': 'PROCEDURE_FOR',
|
||||
'precedes_procedure': 'PRECEDES_PROCEDURE',
|
||||
# 辅助
|
||||
'governs_region': 'GOVERNS_REGION',
|
||||
'effective_timeline': 'EFFECTIVE_TIMELINE',
|
||||
'standard_for': 'STANDARD_FOR',
|
||||
'article_effective_timeline': 'ARTICLE_EFFECTIVE_TIMELINE',
|
||||
# 兼容第一轮旧名称
|
||||
'defines_spatial': 'DEFINES_OBJECT',
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.relationships = []
|
||||
|
||||
def build_from_extraction(
|
||||
self,
|
||||
extraction_results: List[Dict],
|
||||
entity_id_map: Dict[str, str],
|
||||
source_doc_id: str = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
从LLM抽取结果构建关系
|
||||
|
||||
Args:
|
||||
extraction_results: LLM抽取结果列表
|
||||
entity_id_map: 文本到ID的映射
|
||||
source_doc_id: 源文档ID
|
||||
"""
|
||||
relationships = []
|
||||
rel_counter = 0
|
||||
|
||||
for result in extraction_results:
|
||||
if not result or 'relationships' not in result:
|
||||
continue
|
||||
|
||||
# 建立实体文本到ID的快速查找
|
||||
local_entity_map = {}
|
||||
if 'entities' in result:
|
||||
for entity in result['entities']:
|
||||
text = entity.get('text', '')
|
||||
etype = entity.get('type', '')
|
||||
key = f"{etype}:{text}"
|
||||
if key in entity_id_map:
|
||||
local_entity_map[text] = entity_id_map[key]
|
||||
|
||||
for rel in result.get('relationships', []):
|
||||
from_text = rel.get('from_entity', '')
|
||||
to_text = rel.get('to_entity', '')
|
||||
rel_type = rel.get('type', '')
|
||||
context = rel.get('context', '')
|
||||
confidence = rel.get('confidence', 0.8)
|
||||
|
||||
# 查找实体ID
|
||||
from_id = local_entity_map.get(from_text)
|
||||
to_id = local_entity_map.get(to_text)
|
||||
|
||||
if not from_id or not to_id:
|
||||
continue
|
||||
|
||||
rel_type_upper = self.REL_TYPE_MAP.get(rel_type, rel_type.upper())
|
||||
|
||||
rel_counter += 1
|
||||
relationships.append({
|
||||
'source': from_id,
|
||||
'target': to_id,
|
||||
'type': rel_type_upper,
|
||||
'properties': json.dumps({
|
||||
'context': context,
|
||||
'confidence': confidence,
|
||||
'source_doc': source_doc_id,
|
||||
}, ensure_ascii=False)
|
||||
})
|
||||
|
||||
self.relationships.extend(relationships)
|
||||
logger.info(f"构建了 {len(relationships)} 个关系")
|
||||
return relationships
|
||||
|
||||
def build_reference_relationships(
|
||||
self,
|
||||
resolved_refs: List[Dict],
|
||||
doc_id_map: Dict[str, str]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
从引用解析结果构建引用关系
|
||||
|
||||
Args:
|
||||
resolved_refs: CitationResolver解析的引用列表
|
||||
doc_id_map: 文档标题到ID的映射
|
||||
"""
|
||||
relationships = []
|
||||
|
||||
for ref in resolved_refs:
|
||||
source_title = ref.get('source_title', '')
|
||||
target_id = ref.get('target_id', '')
|
||||
rel_type = ref.get('relation_type', 'CITES')
|
||||
context = ref.get('context', '')
|
||||
|
||||
source_id = doc_id_map.get(source_title)
|
||||
if not source_id:
|
||||
continue
|
||||
|
||||
relationships.append({
|
||||
'source': source_id,
|
||||
'target': target_id,
|
||||
'type': rel_type,
|
||||
'properties': json.dumps({'context': context}, ensure_ascii=False)
|
||||
})
|
||||
|
||||
logger.info(f"构建了 {len(relationships)} 个引用关系")
|
||||
return relationships
|
||||
|
||||
def validate_relationships(
|
||||
self,
|
||||
relationships: List[Dict],
|
||||
valid_node_ids: set
|
||||
) -> Dict[str, Any]:
|
||||
"""验证关系数据"""
|
||||
valid = 0
|
||||
invalid = 0
|
||||
broken_links = []
|
||||
|
||||
for rel in relationships:
|
||||
source = rel.get('source', '')
|
||||
target = rel.get('target', '')
|
||||
|
||||
if source in valid_node_ids and target in valid_node_ids:
|
||||
valid += 1
|
||||
else:
|
||||
invalid += 1
|
||||
if source not in valid_node_ids:
|
||||
broken_links.append(f"source not found: {source}")
|
||||
if target not in valid_node_ids:
|
||||
broken_links.append(f"target not found: {target}")
|
||||
|
||||
return {
|
||||
'valid_relationships': valid,
|
||||
'invalid_relationships': invalid,
|
||||
'broken_links': broken_links[:50],
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
"""
|
||||
深度法规实体抽取管道
|
||||
协调LLM抽取、实体规范化、关系构建
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import pandas as pd
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Any
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from knowledge_extraction.llm_legal_extractor import LegalLLMExtractor
|
||||
from knowledge_extraction.citation_resolver import CitationResolver
|
||||
from data_processing.entity_normalizer import LegalEntityNormalizer
|
||||
from data_processing.relationship_builder import RelationshipBuilder
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DeepExtractionPipeline:
|
||||
"""深度法规实体抽取管道"""
|
||||
|
||||
def __init__(self, config_file: str):
|
||||
self.config_file = config_file
|
||||
self.extractor = LegalLLMExtractor(config_file)
|
||||
self.config = self.extractor.config
|
||||
self.normalizer = LegalEntityNormalizer()
|
||||
self.builder = RelationshipBuilder()
|
||||
self.project_root = Path(__file__).parent.parent
|
||||
|
||||
def load_structured_documents(self, input_file: str) -> List[Dict]:
|
||||
"""加载结构化文档数据"""
|
||||
with open(input_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
return data['documents']
|
||||
|
||||
def _prepare_sections(self, documents: List[Dict]) -> List[Dict]:
|
||||
"""准备章节级别的抽取单元"""
|
||||
sections = []
|
||||
|
||||
for doc in documents:
|
||||
doc_title = doc.get('title', '')
|
||||
chapters = doc.get('chapters', [])
|
||||
|
||||
for chapter in chapters:
|
||||
# 将章节下所有条文合并为一个抽取单元
|
||||
texts = []
|
||||
for art in chapter.get('articles', []):
|
||||
art_text = art.get('full_text', art.get('text', ''))
|
||||
if art_text:
|
||||
texts.append(art_text)
|
||||
|
||||
for section in chapter.get('sections', []):
|
||||
for art in section.get('articles', []):
|
||||
art_text = art.get('full_text', art.get('text', ''))
|
||||
if art_text:
|
||||
texts.append(art_text)
|
||||
|
||||
combined_text = '\n'.join(texts)
|
||||
|
||||
if len(combined_text.strip()) > 20:
|
||||
sections.append({
|
||||
'text': combined_text,
|
||||
'document_title': doc_title,
|
||||
'section_title': chapter.get('title', ''),
|
||||
'doc_id': doc.get('doc_id', ''),
|
||||
})
|
||||
|
||||
logger.info(f"准备了 {len(sections)} 个章节抽取单元")
|
||||
return sections
|
||||
|
||||
def _prepare_documents_for_references(self, documents: List[Dict]) -> List[Dict]:
|
||||
"""准备文档级引用抽取"""
|
||||
doc_list = []
|
||||
for doc in documents:
|
||||
doc_list.append({
|
||||
'title': doc.get('title', ''),
|
||||
'text': doc.get('text_preview', ''), # 需要原始文本
|
||||
'doc_id': doc.get('doc_id', ''),
|
||||
})
|
||||
return doc_list
|
||||
|
||||
async def run(
|
||||
self,
|
||||
structured_input: str,
|
||||
raw_input: str,
|
||||
output_dir: str,
|
||||
max_sections: int = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
执行完整抽取流程
|
||||
|
||||
Args:
|
||||
structured_input: 结构化文档JSON路径
|
||||
raw_input: 原始文档JSON路径（用于引用抽取）
|
||||
output_dir: 输出目录
|
||||
max_sections: 最大处理章节数（测试用）
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 1. 加载数据
|
||||
logger.info("加载结构化文档...")
|
||||
documents = self.load_structured_documents(structured_input)
|
||||
logger.info(f"加载 {len(documents)} 个文档")
|
||||
|
||||
# 2. 准备章节抽取单元
|
||||
sections = self._prepare_sections(documents)
|
||||
if max_sections:
|
||||
sections = sections[:max_sections]
|
||||
logger.info(f"限制处理: {max_sections} 个章节")
|
||||
|
||||
# 3. 加载已有的结构化节点（用于引用解析）
|
||||
nodes_csv = output_path / 'nodes_structured.csv'
|
||||
known_docs = []
|
||||
if nodes_csv.exists():
|
||||
nodes_df = pd.read_csv(nodes_csv, encoding='utf-8-sig')
|
||||
known_docs = [
|
||||
{'id': row['id'], 'label': row['label'], 'type': row['type']}
|
||||
for _, row in nodes_df.iterrows()
|
||||
if row['type'] in ['Law', 'AdministrativeRegulation', 'DepartmentalRule', 'PolicyDocument']
|
||||
]
|
||||
logger.info(f"已知文档节点: {len(known_docs)}")
|
||||
|
||||
# 4. LLM章节级抽取
|
||||
logger.info("开始LLM章节级抽取...")
|
||||
extraction_results = await self.extractor.batch_extract_sections(
|
||||
sections,
|
||||
concurrent=self.extractor.config.get('batch_processing', {}).get('concurrent_requests', 5)
|
||||
)
|
||||
|
||||
# 5. 实体规范化
|
||||
logger.info("规范化实体...")
|
||||
entity_id_map = self.normalizer.normalize_batch(extraction_results)
|
||||
entity_nodes = self.normalizer.get_entity_nodes()
|
||||
stats = self.normalizer.get_statistics()
|
||||
logger.info(f"规范化后实体: {stats['total_entities']}")
|
||||
|
||||
# 6. 关系构建
|
||||
logger.info("构建关系...")
|
||||
relationships = self.builder.build_from_extraction(
|
||||
extraction_results, entity_id_map
|
||||
)
|
||||
|
||||
# 7. 文档级引用抽取
|
||||
logger.info("开始文档级引用抽取...")
|
||||
# 加载原始文本用于引用抽取
|
||||
with open(raw_input, 'r', encoding='utf-8') as f:
|
||||
raw_data = json.load(f)
|
||||
|
||||
doc_id_map = {d['label']: d['id'] for d in known_docs}
|
||||
# 补充从文档数据中获取原始文本
|
||||
docs_for_ref = []
|
||||
raw_docs_map = {d['filename']: d for d in raw_data['documents']}
|
||||
|
||||
for doc in documents:
|
||||
filename = doc.get('filename', '')
|
||||
raw_doc = raw_docs_map.get(filename, {})
|
||||
raw_text = raw_doc.get('raw_text', '')
|
||||
if raw_text and len(raw_text) > 50:
|
||||
docs_for_ref.append({
|
||||
'title': doc['title'],
|
||||
'text': raw_text[:5000], # 截取前5000字符
|
||||
})
|
||||
|
||||
ref_results = await self.extractor.batch_extract_references(docs_for_ref, concurrent=3)
|
||||
|
||||
# 8. 引用解析
|
||||
logger.info("解析法规引用...")
|
||||
resolver = CitationResolver(known_docs)
|
||||
|
||||
all_references = []
|
||||
for result in ref_results:
|
||||
if not result or 'references' not in result:
|
||||
continue
|
||||
doc_title = result.get('metadata', {}).get('document_title', '')
|
||||
for ref in result['references']:
|
||||
ref['document_title'] = doc_title
|
||||
all_references.append(ref)
|
||||
|
||||
resolved_refs, unresolved_refs = resolver.resolve_batch(all_references)
|
||||
ref_relationships = self.builder.build_reference_relationships(resolved_refs, doc_id_map)
|
||||
relationships.extend(ref_relationships)
|
||||
|
||||
# 9. 保存结果
|
||||
logger.info("保存结果...")
|
||||
|
||||
# 从配置获取输出文件名
|
||||
output_cfg = self.config.get('output', {})
|
||||
llm_nodes_name = output_cfg.get('nodes_file', 'output/nodes_llm_v2.csv').split('/')[-1]
|
||||
llm_rels_name = output_cfg.get('relationships_file', 'output/rels_llm_v2.csv').split('/')[-1]
|
||||
report_name = output_cfg.get('report_file', 'output/extraction_report_v2.md').split('/')[-1]
|
||||
|
||||
# LLM节点
|
||||
llm_nodes_df = pd.DataFrame(entity_nodes)
|
||||
llm_nodes_path = output_path / llm_nodes_name
|
||||
llm_nodes_df.to_csv(llm_nodes_path, index=False, encoding='utf-8-sig')
|
||||
|
||||
# LLM关系
|
||||
llm_rels_df = pd.DataFrame(relationships)
|
||||
llm_rels_path = output_path / llm_rels_name
|
||||
llm_rels_df.to_csv(llm_rels_path, index=False, encoding='utf-8-sig')
|
||||
|
||||
# 10. 合并
|
||||
logger.info("合并结构化和LLM结果...")
|
||||
if nodes_csv.exists():
|
||||
structured_nodes = pd.read_csv(nodes_csv, encoding='utf-8-sig')
|
||||
merged_nodes = pd.concat([structured_nodes, llm_nodes_df], ignore_index=True)
|
||||
merged_nodes = merged_nodes.drop_duplicates(subset=['id'], keep='first')
|
||||
else:
|
||||
merged_nodes = llm_nodes_df
|
||||
|
||||
rels_csv = output_path / 'rels_structured.csv'
|
||||
if rels_csv.exists():
|
||||
structured_rels = pd.read_csv(rels_csv, encoding='utf-8-sig')
|
||||
merged_rels = pd.concat([structured_rels, llm_rels_df], ignore_index=True)
|
||||
else:
|
||||
merged_rels = llm_rels_df
|
||||
|
||||
merged_nodes_path = output_path / 'nodes_merged_v2.csv'
|
||||
merged_rels_path = output_path / 'rels_merged_v2.csv'
|
||||
merged_nodes.to_csv(merged_nodes_path, index=False, encoding='utf-8-sig')
|
||||
merged_rels.to_csv(merged_rels_path, index=False, encoding='utf-8-sig')
|
||||
|
||||
# 验证
|
||||
valid_ids = set(merged_nodes['id'].tolist())
|
||||
validation = self.builder.validate_relationships(
|
||||
merged_rels.to_dict('records'), valid_ids
|
||||
)
|
||||
|
||||
# 生成报告
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
self._generate_report(
|
||||
output_path, sections, extraction_results, entity_nodes,
|
||||
relationships, resolved_refs, unresolved_refs, validation, duration,
|
||||
report_name=report_name
|
||||
)
|
||||
|
||||
result = {
|
||||
'total_sections': len(sections),
|
||||
'total_entities': len(entity_nodes),
|
||||
'total_relationships': len(relationships),
|
||||
'total_references_resolved': len(resolved_refs),
|
||||
'total_references_unresolved': len(unresolved_refs),
|
||||
'duration_seconds': duration,
|
||||
}
|
||||
|
||||
logger.info(f"\n抽取完成！耗时: {duration:.1f}s")
|
||||
logger.info(f"章节: {result['total_sections']}, 实体: {result['total_entities']}, "
|
||||
f"关系: {result['total_relationships']}, 引用: {result['total_references_resolved']}")
|
||||
|
||||
return result
|
||||
|
||||
def _generate_report(self, output_path, sections, extraction_results,
|
||||
entity_nodes, relationships, resolved_refs,
|
||||
unresolved_refs, validation, duration,
|
||||
report_name='extraction_report_v2.md'):
|
||||
"""生成抽取报告"""
|
||||
lines = ["# 第二轮深度抽取报告（城市更新政策工具箱本体）\n"]
|
||||
lines.append(f"**生成时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
|
||||
lines.append(f"**耗时**: {duration:.1f}秒\n\n")
|
||||
|
||||
lines.append("## 统计\n\n")
|
||||
lines.append(f"- 处理章节: {len(sections)}\n")
|
||||
lines.append(f"- 抽取实体: {len(entity_nodes)}\n")
|
||||
lines.append(f"- 构建关系: {len(relationships)}\n")
|
||||
lines.append(f"- 解析引用: {len(resolved_refs)} 成功, {len(unresolved_refs)} 未匹配\n")
|
||||
|
||||
# 实体类型分布
|
||||
lines.append("\n## 实体类型分布\n\n")
|
||||
type_counts = {}
|
||||
for node in entity_nodes:
|
||||
t = node['type']
|
||||
type_counts[t] = type_counts.get(t, 0) + 1
|
||||
lines.append("| 类型 | 数量 |\n|------|------|\n")
|
||||
for t, c in sorted(type_counts.items()):
|
||||
lines.append(f"| {t} | {c} |\n")
|
||||
|
||||
# 关系类型分布
|
||||
lines.append("\n## 关系类型分布\n\n")
|
||||
rel_type_counts = {}
|
||||
for rel in relationships:
|
||||
t = rel['type']
|
||||
rel_type_counts[t] = rel_type_counts.get(t, 0) + 1
|
||||
lines.append("| 类型 | 数量 |\n|------|------|\n")
|
||||
for t, c in sorted(rel_type_counts.items()):
|
||||
lines.append(f"| {t} | {c} |\n")
|
||||
|
||||
# 数据质量
|
||||
lines.append("\n## 数据质量\n\n")
|
||||
lines.append(f"- 有效关系: {validation['valid_relationships']}\n")
|
||||
lines.append(f"- 无效关系: {validation['invalid_relationships']}\n")
|
||||
|
||||
report_path = output_path / report_name
|
||||
with open(report_path, 'w', encoding='utf-8') as f:
|
||||
f.writelines(lines)
|
||||
logger.info(f"报告已保存: {report_path}")
|
||||
|
||||
|
||||
async def main():
|
||||
config_file = r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\config\deep_extraction_config.yaml"
|
||||
structured_input = r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output\structured_documents.json"
|
||||
raw_input = r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output\all_documents.json"
|
||||
output_dir = r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output"
|
||||
|
||||
pipeline = DeepExtractionPipeline(config_file)
|
||||
|
||||
# 测试模式：只处理前20个章节
|
||||
import sys
|
||||
max_sections = 20
|
||||
if '--full' in sys.argv:
|
||||
max_sections = None
|
||||
|
||||
results = await pipeline.run(
|
||||
structured_input, raw_input, output_dir,
|
||||
max_sections=max_sections
|
||||
)
|
||||
|
||||
print("\n结果:")
|
||||
print(json.dumps(results, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,134 @@
|
||||
"""
|
||||
法规引用解析器 - 将文本中的法规引用映射到已知文档节点
|
||||
"""
|
||||
|
||||
import re
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CitationResolver:
|
||||
"""法规引用解析器"""
|
||||
|
||||
def __init__(self, known_documents: List[Dict]):
|
||||
"""
|
||||
Args:
|
||||
known_documents: 已知文档列表，每个包含 id, label, type
|
||||
"""
|
||||
self.known_docs = {}
|
||||
self.name_variants = {}
|
||||
|
||||
for doc in known_documents:
|
||||
name = doc['label']
|
||||
doc_id = doc['id']
|
||||
self.known_docs[name] = doc_id
|
||||
|
||||
# 生成名称变体
|
||||
variants = self._generate_variants(name)
|
||||
for variant in variants:
|
||||
self.name_variants[variant] = doc_id
|
||||
|
||||
logger.info(f"CitationResolver 初始化: {len(self.known_docs)} 个已知文档")
|
||||
|
||||
def _generate_variants(self, name: str) -> List[str]:
|
||||
"""生成法规名称变体"""
|
||||
variants = [name]
|
||||
|
||||
# 去掉书名号
|
||||
cleaned = name.replace('《', '').replace('》', '')
|
||||
if cleaned != name:
|
||||
variants.append(cleaned)
|
||||
|
||||
# 去掉"中华人民共和国"前缀
|
||||
if name.startswith('中华人民共和国'):
|
||||
short = name.replace('中华人民共和国', '')
|
||||
variants.append(short)
|
||||
|
||||
# 简称
|
||||
if '实施条例' in name:
|
||||
variants.append(name.replace('实施条例', '条例'))
|
||||
if '实施办法' in name:
|
||||
variants.append(name.replace('实施办法', '办法'))
|
||||
|
||||
return list(set(variants))
|
||||
|
||||
def resolve(self, referenced_name: str, threshold: float = 0.7) -> Optional[str]:
|
||||
"""
|
||||
解析引用名称到文档ID
|
||||
|
||||
Args:
|
||||
referenced_name: 被引用的法规名称
|
||||
threshold: 相似度阈值
|
||||
|
||||
Returns:
|
||||
文档ID，未找到返回None
|
||||
"""
|
||||
if not referenced_name:
|
||||
return None
|
||||
|
||||
# 清理名称
|
||||
cleaned = referenced_name.strip()
|
||||
cleaned = re.sub(r'[《》]', '', cleaned)
|
||||
|
||||
# 精确匹配
|
||||
if cleaned in self.known_docs:
|
||||
return self.known_docs[cleaned]
|
||||
if cleaned in self.name_variants:
|
||||
return self.name_variants[cleaned]
|
||||
|
||||
# 模糊匹配
|
||||
best_match = None
|
||||
best_score = 0
|
||||
|
||||
for known_name, doc_id in self.known_docs.items():
|
||||
score = SequenceMatcher(None, cleaned, known_name).ratio()
|
||||
if score > best_score and score >= threshold:
|
||||
best_score = score
|
||||
best_match = doc_id
|
||||
|
||||
if best_match:
|
||||
logger.info(f"模糊匹配: '{referenced_name}' -> {best_match} (score={best_score:.2f})")
|
||||
|
||||
return best_match
|
||||
|
||||
def resolve_batch(self, references: List[Dict]) -> List[Tuple[str, str]]:
|
||||
"""
|
||||
批量解析引用
|
||||
|
||||
Returns:
|
||||
List of (source_doc_title, target_doc_id) pairs
|
||||
"""
|
||||
resolved = []
|
||||
unresolved = []
|
||||
|
||||
for ref in references:
|
||||
doc_title = ref.get('document_title', '')
|
||||
referenced_law = ref.get('referenced_law', '')
|
||||
relation_type = ref.get('relation_type', 'cites')
|
||||
context = ref.get('context', '')
|
||||
|
||||
target_id = self.resolve(referenced_law)
|
||||
|
||||
if target_id:
|
||||
resolved.append({
|
||||
'source_title': doc_title,
|
||||
'target_id': target_id,
|
||||
'target_name': referenced_law,
|
||||
'relation_type': relation_type.upper(),
|
||||
'context': context,
|
||||
})
|
||||
else:
|
||||
unresolved.append({
|
||||
'source_title': doc_title,
|
||||
'target_name': referenced_law,
|
||||
'relation_type': relation_type,
|
||||
})
|
||||
|
||||
logger.info(f"引用解析完成: {len(resolved)} 成功, {len(unresolved)} 未匹配")
|
||||
|
||||
return resolved, unresolved
|
||||
@@ -0,0 +1,303 @@
|
||||
"""
|
||||
基于DeepSeek的法规知识抽取器
|
||||
从法条文本中抽取法律实体和关系
|
||||
使用OpenAI SDK直接调用DeepSeek API
|
||||
"""
|
||||
|
||||
import os
|
||||
# 解决Windows上httpx代理检测导致的连接失败
|
||||
os.environ['NO_PROXY'] = '*'
|
||||
|
||||
import yaml
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
raise ImportError("请安装: pip install openai")
|
||||
|
||||
|
||||
class LegalLLMExtractor:
|
||||
"""基于DeepSeek的法规知识抽取器"""
|
||||
|
||||
def __init__(self, config_file: str):
|
||||
self.config = self._load_config(config_file)
|
||||
self.api_key = self._load_api_key()
|
||||
|
||||
self.client = OpenAI(
|
||||
api_key=self.api_key,
|
||||
base_url='https://api.deepseek.com',
|
||||
)
|
||||
|
||||
self.logger = logging.getLogger(__name__)
|
||||
|
||||
def _load_config(self, config_file: str) -> Dict:
|
||||
config_path = Path(config_file)
|
||||
if not config_path.exists():
|
||||
config_path = Path(__file__).parent.parent.parent / 'config' / 'deep_extraction_config.yaml'
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
def _load_api_key(self) -> str:
|
||||
api_file = Path(__file__).parent.parent.parent / 'config' / 'api_keys.yaml'
|
||||
if api_file.exists():
|
||||
with open(api_file, 'r', encoding='utf-8') as f:
|
||||
api_config = yaml.safe_load(f)
|
||||
key = api_config.get('deepseek_api_key', '')
|
||||
if key and key != 'YOUR_API_KEY_HERE':
|
||||
return key
|
||||
key = os.environ.get('DEEPSEEK_API_KEY', '')
|
||||
if key:
|
||||
return key
|
||||
raise ValueError("未找到DeepSeek API密钥，请配置 config/api_keys.yaml 或设置 DEEPSEEK_API_KEY 环境变量")
|
||||
|
||||
def _extract_json(self, response_text: str) -> str:
|
||||
"""从响应中提取JSON"""
|
||||
if '```json' in response_text:
|
||||
start = response_text.find('```json') + 7
|
||||
end = response_text.find('```', start)
|
||||
if end > start:
|
||||
return response_text[start:end].strip()
|
||||
elif '```' in response_text:
|
||||
start = response_text.find('```') + 3
|
||||
end = response_text.find('```', start)
|
||||
if end > start:
|
||||
content = response_text[start:end].strip()
|
||||
if content.startswith('json'):
|
||||
content = content[4:].strip()
|
||||
return content
|
||||
return response_text.strip()
|
||||
|
||||
def _call_api(self, prompt: str, max_retries: int = 3) -> Optional[str]:
|
||||
"""同步调用API"""
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.config['llm']['model'],
|
||||
messages=[{'role': 'user', 'content': prompt}],
|
||||
temperature=self.config['llm']['temperature'],
|
||||
max_tokens=self.config['llm']['max_tokens'],
|
||||
)
|
||||
return response.choices[0].message.content
|
||||
except Exception as e:
|
||||
self.logger.error(f"API调用失败 (attempt {attempt+1}/{max_retries}): {e}")
|
||||
if attempt == max_retries - 1:
|
||||
return None
|
||||
import time
|
||||
time.sleep(self.config['llm'].get('retry_delay', 2))
|
||||
|
||||
async def _call_api_async(self, prompt: str, max_retries: int = 3) -> Optional[str]:
|
||||
"""异步调用API"""
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(None, lambda: self._call_api(prompt, max_retries))
|
||||
|
||||
async def extract_from_section(
|
||||
self,
|
||||
section_text: str,
|
||||
document_title: str,
|
||||
section_title: str,
|
||||
max_retries: int = 3
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""从章节文本中抽取实体和关系（基于动态配置的本体）"""
|
||||
if len(section_text) > 5000:
|
||||
section_text = section_text[:5000]
|
||||
|
||||
# 从配置动态构建实体类型列表
|
||||
entity_prompts = self.config.get('entity_type_prompts', {})
|
||||
entity_types_desc = '\n'.join(
|
||||
f"- {etype}：{desc}"
|
||||
for etype, desc in entity_prompts.items()
|
||||
)
|
||||
|
||||
# 从配置动态构建关系类型列表
|
||||
rel_prompts = self.config.get('relationship_type_prompts', {})
|
||||
rel_types_desc = '\n'.join(
|
||||
f"- {rtype}：{desc}"
|
||||
for rtype, desc in rel_prompts.items()
|
||||
)
|
||||
|
||||
# 边界规则
|
||||
boundary_rules = self.config.get('extraction_tips', {}).get('entity_boundary_rules', '')
|
||||
do_not = self.config.get('extraction_tips', {}).get('do_not_extract', '')
|
||||
|
||||
prompt = f"""你是一位中国城市更新法规政策分析专家。请从以下法规章节中识别实体和关系。
|
||||
|
||||
所属法规：{document_title}
|
||||
章节名称：{section_title}
|
||||
|
||||
文本内容：
|
||||
{section_text}
|
||||
|
||||
## 需要识别的实体类型
|
||||
|
||||
{entity_types_desc}
|
||||
|
||||
## 需要识别的关系类型
|
||||
|
||||
{rel_types_desc}
|
||||
|
||||
{boundary_rules}
|
||||
|
||||
{do_not}
|
||||
|
||||
## 输出JSON格式
|
||||
|
||||
严格输出以下JSON格式，不要输出其他内容：
|
||||
{{
|
||||
"entities": [
|
||||
{{
|
||||
"text": "实体原文表述",
|
||||
"type": "实体类型（必须是上述类型之一）",
|
||||
"attributes": {{
|
||||
"description": "实体简要描述"
|
||||
}}
|
||||
}}
|
||||
],
|
||||
"relationships": [
|
||||
{{
|
||||
"from_entity": "源实体文本",
|
||||
"from_type": "源实体类型",
|
||||
"to_entity": "目标实体文本",
|
||||
"to_type": "目标实体类型",
|
||||
"type": "关系类型（必须是上述类型之一）",
|
||||
"context": "关系上下文原文",
|
||||
"confidence": 0.9
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
## 注意事项
|
||||
- 只抽取文本中明确提到的实体和关系，不要推测
|
||||
- 关系的from_entity和to_entity必须是entities中出现的实体
|
||||
- 保持原文表述，不要改写实体名称
|
||||
- confidence范围0.0-1.0，表示抽取的可信度
|
||||
- 重点关注PolicyTool（政策工具）的识别，这是本次抽取的核心目标
|
||||
- 确保输出有效JSON，不要输出JSON以外的内容"""
|
||||
|
||||
response_text = await self._call_api_async(prompt, max_retries)
|
||||
if not response_text:
|
||||
return None
|
||||
|
||||
try:
|
||||
json_text = self._extract_json(response_text)
|
||||
result = json.loads(json_text)
|
||||
result['metadata'] = {
|
||||
'document_title': document_title,
|
||||
'section_title': section_title,
|
||||
'extraction_time': datetime.now().isoformat(),
|
||||
}
|
||||
return result
|
||||
except json.JSONDecodeError as e:
|
||||
self.logger.warning(f"JSON解析失败: {e}")
|
||||
return None
|
||||
|
||||
async def extract_document_references(
|
||||
self,
|
||||
document_title: str,
|
||||
full_text: str,
|
||||
max_retries: int = 3
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""从文档全文中抽取跨文档引用关系"""
|
||||
text_sample = full_text[:3000]
|
||||
if len(full_text) > 5000:
|
||||
text_sample += '\n...\n' + full_text[-2000:]
|
||||
|
||||
prompt = f"""你是一位中国法律专家。请从以下法规文本中识别引用的其他法规文件。
|
||||
|
||||
法规名称：{document_title}
|
||||
|
||||
文本片段：
|
||||
{text_sample}
|
||||
|
||||
请识别以下引用关系：
|
||||
- cites（引用）：文本中提到的其他法规
|
||||
- implements（实施）：文本声明为实施某上位法而制定
|
||||
- amends（修正）：文本对其他法规的修改
|
||||
- replaces（替代）：文本声明替代或废止其他法规
|
||||
|
||||
输出JSON格式：
|
||||
{{
|
||||
"references": [
|
||||
{{
|
||||
"referenced_law": "被引用的法规名称",
|
||||
"relation_type": "cites/implements/amends/replaces",
|
||||
"context": "引用上下文原文"
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
注意：只识别文本中明确提到的法规名称（书名号《》内的名称），不要推测。"""
|
||||
|
||||
response_text = await self._call_api_async(prompt, max_retries)
|
||||
if not response_text:
|
||||
return None
|
||||
|
||||
try:
|
||||
json_text = self._extract_json(response_text)
|
||||
result = json.loads(json_text)
|
||||
result['metadata'] = {
|
||||
'document_title': document_title,
|
||||
'extraction_time': datetime.now().isoformat(),
|
||||
}
|
||||
return result
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
async def batch_extract_sections(
|
||||
self,
|
||||
sections: List[Dict[str, str]],
|
||||
concurrent: int = 5,
|
||||
progress_callback=None
|
||||
) -> List[Optional[Dict]]:
|
||||
"""批量抽取章节实体"""
|
||||
results = []
|
||||
batch_size = concurrent
|
||||
|
||||
for i in range(0, len(sections), batch_size):
|
||||
batch = sections[i:i + batch_size]
|
||||
batch_num = i // batch_size + 1
|
||||
total_batches = (len(sections) + batch_size - 1) // batch_size
|
||||
|
||||
self.logger.info(f"处理章节批次 {batch_num}/{total_batches}: {len(batch)} 个")
|
||||
|
||||
tasks = [
|
||||
self.extract_from_section(
|
||||
s['text'], s['document_title'], s['section_title']
|
||||
)
|
||||
for s in batch
|
||||
]
|
||||
batch_results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
results.extend(batch_results)
|
||||
|
||||
if progress_callback:
|
||||
await progress_callback(batch_num, total_batches, batch_results, batch)
|
||||
|
||||
return results
|
||||
|
||||
async def batch_extract_references(
|
||||
self,
|
||||
documents: List[Dict[str, str]],
|
||||
concurrent: int = 3
|
||||
) -> List[Optional[Dict]]:
|
||||
"""批量抽取文档引用"""
|
||||
results = []
|
||||
batch_size = concurrent
|
||||
|
||||
for i in range(0, len(documents), batch_size):
|
||||
batch = documents[i:i + batch_size]
|
||||
|
||||
tasks = [
|
||||
self.extract_document_references(d['title'], d['text'])
|
||||
for d in batch
|
||||
]
|
||||
batch_results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
results.extend(batch_results)
|
||||
|
||||
self.logger.info(f"引用抽取进度: {min(i+batch_size, len(documents))}/{len(documents)}")
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,166 @@
|
||||
"""
|
||||
城市规划法律法规知识图谱构建 - 主入口
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
# 添加项目路径
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
|
||||
def step1_read_documents():
|
||||
"""Step1: 读取Word文档"""
|
||||
from importlib import import_module
|
||||
# 动态导入scripts目录下的模块
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'scripts'))
|
||||
docx_reader = import_module('docx_reader')
|
||||
|
||||
source_dir = r"E:\Project\SI\2026_KG_PlanningLaw\data\城市规划法律法规\城市更新法规数据库"
|
||||
output_path = r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output\all_documents.json"
|
||||
|
||||
reader = docx_reader.DocxReader(source_dir)
|
||||
reader.read_all_documents()
|
||||
reader.save_to_json(output_path)
|
||||
reader.print_summary()
|
||||
|
||||
|
||||
def step2_extract_metadata():
|
||||
"""Step2: 解析法规元数据"""
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'scripts'))
|
||||
metadata_extractor = import_module('legal_metadata_extractor')
|
||||
|
||||
input_path = r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output\all_documents.json"
|
||||
output_path = r"E:\Project\SI\2026_KG_PlanningLaw\dofile\kg_project\output\structured_documents.json"
|
||||
|
||||
extractor = metadata_extractor.LegalMetadataExtractor()
|
||||
extractor.process_all(input_path, output_path)
|
||||
|
||||
|
||||
def step3_generate_csv():
|
||||
"""Step3: 生成结构化CSV"""
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'scripts'))
|
||||
extract_csv = import_module('extract_legal_csv')
|
||||
extract_csv.main()
|
||||
|
||||
|
||||
def step4_deep_extraction():
|
||||
"""Step4: LLM深度抽取"""
|
||||
from deep_extraction_pipeline import DeepExtractionPipeline
|
||||
|
||||
config_file = str(Path(__file__).parent.parent / 'config' / 'deep_extraction_config.yaml')
|
||||
structured_input = str(Path(__file__).parent.parent / 'output' / 'structured_documents.json')
|
||||
raw_input = str(Path(__file__).parent.parent / 'output' / 'all_documents.json')
|
||||
output_dir = str(Path(__file__).parent.parent / 'output')
|
||||
|
||||
pipeline = DeepExtractionPipeline(config_file)
|
||||
|
||||
results = asyncio.run(pipeline.run(
|
||||
structured_input, raw_input, output_dir,
|
||||
max_sections=None # 完整抽取
|
||||
))
|
||||
|
||||
print("\n结果:")
|
||||
print(json.dumps(results, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
def step5_visualize():
|
||||
"""Step5: 可视化"""
|
||||
from importlib import import_module
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'scripts'))
|
||||
visualize = import_module('visualize_legal_kg')
|
||||
visualize.main()
|
||||
|
||||
|
||||
def show_status():
|
||||
"""显示项目状态"""
|
||||
output_dir = Path(__file__).parent.parent / 'output'
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("城市规划法律法规知识图谱 - 项目状态")
|
||||
print("=" * 60)
|
||||
|
||||
files = {
|
||||
'all_documents.json': 'Step1: Word文档读取',
|
||||
'structured_documents.json': 'Step2: 法规元数据解析',
|
||||
'nodes_structured.csv': 'Step3: 结构化节点',
|
||||
'rels_structured.csv': 'Step3: 结构化关系',
|
||||
'nodes_llm.csv': 'Step4: LLM抽取节点',
|
||||
'rels_llm.csv': 'Step4: LLM抽取关系',
|
||||
'nodes_merged.csv': 'Step5: 合并节点',
|
||||
'rels_merged.csv': 'Step5: 合并关系',
|
||||
}
|
||||
|
||||
for filename, description in files.items():
|
||||
filepath = output_dir / filename
|
||||
if filepath.exists():
|
||||
import os
|
||||
size = os.path.getsize(filepath)
|
||||
print(f" [OK] {description} ({filename}, {size:,} bytes)")
|
||||
else:
|
||||
print(f" [--] {description} ({filename}, 未生成)")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
def importlib_safe_import(module_name):
|
||||
from importlib import import_module
|
||||
return import_module(module_name)
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='城市规划法律法规知识图谱构建')
|
||||
parser.add_argument('--step', type=int, choices=[1, 2, 3, 4, 5], help='执行指定步骤')
|
||||
parser.add_argument('--full', action='store_true', help='运行完整流程')
|
||||
parser.add_argument('--status', action='store_true', help='查看项目状态')
|
||||
parser.add_argument('--test', action='store_true', help='测试模式(少量数据)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.status:
|
||||
show_status()
|
||||
return
|
||||
|
||||
steps = {
|
||||
1: ("读取Word文档", step1_read_documents),
|
||||
2: ("解析法规元数据", step2_extract_metadata),
|
||||
3: ("生成结构化CSV", step3_generate_csv),
|
||||
4: ("LLM深度抽取", step4_deep_extraction),
|
||||
5: ("可视化", step5_visualize),
|
||||
}
|
||||
|
||||
if args.step:
|
||||
name, func = steps[args.step]
|
||||
print(f"\n{'='*60}")
|
||||
print(f"执行 Step {args.step}: {name}")
|
||||
print(f"{'='*60}")
|
||||
func()
|
||||
elif args.full:
|
||||
print("\n运行完整流程...")
|
||||
for step_num, (name, func) in steps.items():
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Step {step_num}: {name}")
|
||||
print(f"{'='*60}")
|
||||
try:
|
||||
func()
|
||||
except Exception as e:
|
||||
print(f"Step {step_num} 失败: {e}")
|
||||
break
|
||||
else:
|
||||
print("请指定操作:")
|
||||
print(" python src/main.py --step 1 # 读取Word文档")
|
||||
print(" python src/main.py --step 2 # 解析法规元数据")
|
||||
print(" python src/main.py --step 3 # 生成结构化CSV")
|
||||
print(" python src/main.py --step 4 # LLM深度抽取")
|
||||
print(" python src/main.py --step 5 # 可视化")
|
||||
print(" python src/main.py --full # 运行完整流程")
|
||||
print(" python src/main.py --status # 查看项目状态")
|
||||
print(" python src/main.py --test # 测试模式")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,29 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
echo ========================================
|
||||
echo 城市规划法律法规知识图谱构建工具
|
||||
echo ========================================
|
||||
echo.
|
||||
echo 请选择操作:
|
||||
echo 1. 读取Word文档 (Step1)
|
||||
echo 2. 解析法规元数据 (Step2)
|
||||
echo 3. 生成结构化CSV (Step3)
|
||||
echo 4. LLM深度抽取 (Step4)
|
||||
echo 5. 实体规范化与合并 (Step5)
|
||||
echo 6. 可视化 (Step6)
|
||||
echo 7. 运行完整流程
|
||||
echo 8. 查看进度
|
||||
echo.
|
||||
|
||||
set /p choice=请输入选项:
|
||||
|
||||
if "%choice%"=="1" python scripts/docx_reader.py
|
||||
if "%choice%"=="2" python scripts/legal_metadata_extractor.py
|
||||
if "%choice%"=="3" python scripts/extract_legal_csv.py
|
||||
if "%choice%"=="4" python src/deep_extraction_pipeline.py
|
||||
if "%choice%"=="5" python src/data_processing/merge_data.py
|
||||
if "%choice%"=="6" python scripts/visualize_legal_kg.py
|
||||
if "%choice%"=="7" python src/main.py --full
|
||||
if "%choice%"=="8" python src/main.py --status
|
||||
|
||||
pause
|
||||
Reference in New Issue
Block a user