init: KG_ICH 项目初始化
- data/: 非遗地理编码数据（GIS shapefile + CSV） - dofile/kg_project/: 知识图谱构建代码（纳入主仓库） - dofile/visulization/: 可视化数据与路线图 - officefile/: 文献、草稿、bib 文档 - officefile/latex/: Overleaf 同步目录（独立管理，不纳入） - output/: 输出目录 - logs/: 日志目录
This commit is contained in:
@@ -0,0 +1,375 @@
|
||||
"""
|
||||
数据预处理脚本 - 读取黑龙江非遗Excel数据
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any
|
||||
import logging
|
||||
|
||||
class ExcelDataReader:
|
||||
"""Excel数据读取器"""
|
||||
|
||||
def __init__(self, excel_path: str):
|
||||
"""
|
||||
初始化数据读取器
|
||||
|
||||
Args:
|
||||
excel_path: Excel文件路径
|
||||
"""
|
||||
self.excel_path = Path(excel_path)
|
||||
self.data = None
|
||||
self.logger = self._setup_logger()
|
||||
|
||||
def _setup_logger(self):
|
||||
"""设置日志"""
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
return logging.getLogger(__name__)
|
||||
|
||||
def read_excel(self) -> pd.DataFrame:
|
||||
"""
|
||||
读取Excel文件
|
||||
|
||||
Returns:
|
||||
DataFrame: 数据框
|
||||
"""
|
||||
try:
|
||||
self.logger.info(f"开始读取Excel文件: {self.excel_path}")
|
||||
|
||||
# 读取Excel文件
|
||||
self.data = pd.read_excel(self.excel_path)
|
||||
|
||||
self.logger.info(f"成功读取 {len(self.data)} 行数据")
|
||||
self.logger.info(f"列名: {list(self.data.columns)}")
|
||||
|
||||
# 显示前5行
|
||||
self.logger.info("\n前5行数据:")
|
||||
self.logger.info(self.data.head())
|
||||
|
||||
return self.data
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"读取Excel文件失败: {str(e)}")
|
||||
raise
|
||||
|
||||
def analyze_data(self) -> Dict[str, Any]:
|
||||
"""
|
||||
分析数据
|
||||
|
||||
Returns:
|
||||
Dict: 分析结果
|
||||
"""
|
||||
if self.data is None:
|
||||
raise ValueError("请先读取Excel文件")
|
||||
|
||||
analysis = {
|
||||
"total_records": len(self.data),
|
||||
"columns": list(self.data.columns),
|
||||
"column_types": {col: str(dtype) for col, dtype in self.data.dtypes.items()},
|
||||
"missing_values": self.data.isnull().sum().to_dict(),
|
||||
"statistics": {}
|
||||
}
|
||||
|
||||
# 分析类别分布
|
||||
if '类别' in self.data.columns:
|
||||
category_counts = self.data['类别'].value_counts()
|
||||
analysis['category_distribution'] = category_counts.to_dict()
|
||||
|
||||
# 分析级别分布
|
||||
if '项目级别' in self.data.columns:
|
||||
level_counts = self.data['项目级别'].value_counts()
|
||||
analysis['level_distribution'] = level_counts.to_dict()
|
||||
|
||||
# 分析地域分布
|
||||
if '项目申报单位/地区' in self.data.columns:
|
||||
location_counts = self.data['项目申报单位/地区'].value_counts()
|
||||
analysis['location_distribution'] = location_counts.head(20).to_dict()
|
||||
|
||||
# 传承人覆盖率
|
||||
if '代表性传承人' in self.data.columns:
|
||||
has_inheritor = self.data['代表性传承人'].notna().sum()
|
||||
analysis['inheritor_coverage'] = {
|
||||
"total": len(self.data),
|
||||
"has_inheritor": int(has_inheritor),
|
||||
"coverage_rate": float(has_inheritor / len(self.data) * 100)
|
||||
}
|
||||
|
||||
return analysis
|
||||
|
||||
def clean_data(self) -> pd.DataFrame:
|
||||
"""
|
||||
清洗数据
|
||||
|
||||
Returns:
|
||||
DataFrame: 清洗后的数据
|
||||
"""
|
||||
if self.data is None:
|
||||
raise ValueError("请先读取Excel文件")
|
||||
|
||||
self.logger.info("开始清洗数据")
|
||||
|
||||
# 去除空行
|
||||
original_len = len(self.data)
|
||||
self.data = self.data.dropna(how='all')
|
||||
self.logger.info(f"去除空行: {original_len} -> {len(self.data)}")
|
||||
|
||||
# 填充缺失值
|
||||
for col in self.data.columns:
|
||||
if self.data[col].dtype == 'object':
|
||||
self.data[col] = self.data[col].fillna('')
|
||||
else:
|
||||
self.data[col] = self.data[col].fillna(0)
|
||||
|
||||
# 去重（基于项目名称、项目批次、项目保护单位、代表性传承人四个字段）
|
||||
dedup_cols = ['项目名称', '项目批次', '项目保护单位', '代表性传承人']
|
||||
existing_cols = [col for col in dedup_cols if col in self.data.columns]
|
||||
|
||||
if existing_cols:
|
||||
before_dedup = len(self.data)
|
||||
self.data = self.data.drop_duplicates(subset=existing_cols, keep='first')
|
||||
duplicate_count = before_dedup - len(self.data)
|
||||
dedup_rate = (duplicate_count / before_dedup * 100) if before_dedup > 0 else 0
|
||||
self.logger.info(f"去重（基于{len(existing_cols)}个字段: {', '.join(existing_cols)}）: {before_dedup} -> {len(self.data)} (删除{duplicate_count}条，去重率{dedup_rate:.1f}%)")
|
||||
|
||||
return self.data
|
||||
|
||||
def convert_to_kg_format(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
转换为知识图谱格式
|
||||
|
||||
Returns:
|
||||
List[Dict]: 知识图谱节点列表
|
||||
"""
|
||||
if self.data is None:
|
||||
raise ValueError("请先读取Excel文件")
|
||||
|
||||
self.logger.info("转换为知识图谱格式")
|
||||
|
||||
kg_nodes = []
|
||||
|
||||
for idx, row in self.data.iterrows():
|
||||
try:
|
||||
# 创建非遗项目节点
|
||||
project_node = {
|
||||
"id": f"ICH-{idx:04d}",
|
||||
"type": "ICH_Project",
|
||||
"properties": {
|
||||
"project_id": f"ICH-{idx:04d}",
|
||||
"name": str(row.get('项目名称', '')),
|
||||
"level": str(row.get('项目级别', '')),
|
||||
"batch": str(row.get('批次号', '')),
|
||||
"category": str(row.get('类别', '')),
|
||||
"declaration_area": str(row.get('项目申报单位/地区', '')),
|
||||
"description": str(row.get('备注', '')),
|
||||
"source_row": idx + 2 # Excel行号（从2开始，第一行是表头）
|
||||
}
|
||||
}
|
||||
|
||||
kg_nodes.append(project_node)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"转换第{idx}行数据失败: {str(e)}")
|
||||
continue
|
||||
|
||||
self.logger.info(f"成功转换 {len(kg_nodes)} 个节点")
|
||||
return kg_nodes
|
||||
|
||||
def extract_inheritors(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
提取传承人信息
|
||||
|
||||
Returns:
|
||||
List[Dict]: 传承人节点列表
|
||||
"""
|
||||
if self.data is None:
|
||||
raise ValueError("请先读取Excel文件")
|
||||
|
||||
self.logger.info("提取传承人信息")
|
||||
|
||||
inheritors = []
|
||||
inheritor_id = 0
|
||||
|
||||
for idx, row in self.data.iterrows():
|
||||
inheritor_names = row.get('代表性传承人', '')
|
||||
if pd.isna(inheritor_names) or not str(inheritor_names).strip():
|
||||
continue
|
||||
|
||||
# 处理多个传承人（用顿号分隔）
|
||||
names = str(inheritor_names).replace('、', ',').replace('，', ',').split(',')
|
||||
|
||||
for name in names:
|
||||
name = name.strip()
|
||||
if not name:
|
||||
continue
|
||||
|
||||
inheritor_id += 1
|
||||
inheritor_node = {
|
||||
"id": f"INH-{inheritor_id:04d}",
|
||||
"type": "Inheritor",
|
||||
"properties": {
|
||||
"inheritor_id": f"INH-{inheritor_id:04d}",
|
||||
"name": name,
|
||||
"project_name": str(row.get('项目名称', '')),
|
||||
"source_row": idx + 2
|
||||
}
|
||||
}
|
||||
|
||||
inheritors.append(inheritor_node)
|
||||
|
||||
self.logger.info(f"提取了 {len(inheritors)} 个传承人")
|
||||
return inheritors
|
||||
|
||||
def extract_relations(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
提取关系
|
||||
|
||||
Returns:
|
||||
List[Dict]: 关系列表
|
||||
"""
|
||||
if self.data is None:
|
||||
raise ValueError("请先读取Excel文件")
|
||||
|
||||
self.logger.info("提取关系")
|
||||
|
||||
relations = []
|
||||
relation_id = 0
|
||||
|
||||
for idx, row in self.data.iterrows():
|
||||
project_id = f"ICH-{idx:04d}"
|
||||
|
||||
# 项目-类别关系
|
||||
category = row.get('类别', '')
|
||||
if pd.notna(category) and str(category).strip():
|
||||
relation_id += 1
|
||||
relations.append({
|
||||
"id": f"REL-{relation_id:04d}",
|
||||
"type": "belongs_to",
|
||||
"from": project_id,
|
||||
"to": f"CAT-{str(category)}",
|
||||
"properties": {}
|
||||
})
|
||||
|
||||
# 项目-传承人关系
|
||||
inheritor_names = row.get('代表性传承人', '')
|
||||
if pd.notna(inheritor_names) and str(inheritor_names).strip():
|
||||
names = str(inheritor_names).replace('、', ',').replace('，', ',').split(',')
|
||||
for name in names:
|
||||
name = name.strip()
|
||||
if name:
|
||||
relation_id += 1
|
||||
relations.append({
|
||||
"id": f"REL-{relation_id:04d}",
|
||||
"type": "has_inheritor",
|
||||
"from": project_id,
|
||||
"to": f"INH-{name}", # 简化处理
|
||||
"properties": {}
|
||||
})
|
||||
|
||||
self.logger.info(f"提取了 {len(relations)} 个关系")
|
||||
return relations
|
||||
|
||||
def save_analysis_report(self, output_path: str):
|
||||
"""
|
||||
保存分析报告
|
||||
|
||||
Args:
|
||||
output_path: 输出路径
|
||||
"""
|
||||
analysis = self.analyze_data()
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(analysis, f, indent=2, ensure_ascii=False)
|
||||
|
||||
self.logger.info(f"分析报告已保存到: {output_path}")
|
||||
|
||||
def save_kg_data(self, nodes: List[Dict], relations: List[Dict], output_dir: str):
|
||||
"""
|
||||
保存知识图谱数据
|
||||
|
||||
Args:
|
||||
nodes: 节点列表
|
||||
relations: 关系列表
|
||||
output_dir: 输出目录
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 保存节点
|
||||
nodes_file = output_path / "nodes.json"
|
||||
with open(nodes_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(nodes, f, indent=2, ensure_ascii=False)
|
||||
self.logger.info(f"节点已保存到: {nodes_file}")
|
||||
|
||||
# 保存关系
|
||||
relations_file = output_path / "relations.json"
|
||||
with open(relations_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(relations, f, indent=2, ensure_ascii=False)
|
||||
self.logger.info(f"关系已保存到: {relations_file}")
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
# 配置
|
||||
excel_path = r"E:\Project\2026_KG_ICH\data\黑龙江国家级和省级非遗名单.xlsx"
|
||||
output_dir = r"E:\Project\2026_KG_ICH\dofile\kg_project\output"
|
||||
|
||||
# 创建读取器
|
||||
reader = ExcelDataReader(excel_path)
|
||||
|
||||
# 读取数据
|
||||
reader.read_excel()
|
||||
|
||||
# 分析数据
|
||||
print("\n" + "="*60)
|
||||
print("数据分析")
|
||||
print("="*60)
|
||||
analysis = reader.analyze_data()
|
||||
print(f"\n总记录数: {analysis['total_records']}")
|
||||
print(f"\n类别分布:")
|
||||
for cat, count in analysis.get('category_distribution', {}).items():
|
||||
print(f" {cat}: {count}")
|
||||
|
||||
print(f"\n传承人覆盖率: {analysis.get('inheritor_coverage', {}).get('coverage_rate', 0):.1f}%")
|
||||
|
||||
# 保存分析报告
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
reader.save_analysis_report(f"{output_dir}/data_analysis.json")
|
||||
|
||||
# 清洗数据
|
||||
print("\n" + "="*60)
|
||||
print("数据清洗")
|
||||
print("="*60)
|
||||
reader.clean_data()
|
||||
|
||||
# 转换为知识图谱格式
|
||||
print("\n" + "="*60)
|
||||
print("转换为知识图谱格式")
|
||||
print("="*60)
|
||||
|
||||
nodes = reader.convert_to_kg_format()
|
||||
inheritors = reader.extract_inheritors()
|
||||
relations = reader.extract_relations()
|
||||
|
||||
# 合并所有节点
|
||||
all_nodes = nodes + inheritors
|
||||
|
||||
print(f"\n节点统计:")
|
||||
print(f" 非遗项目节点: {len(nodes)}")
|
||||
print(f" 传承人节点: {len(inheritors)}")
|
||||
print(f" 总节点数: {len(all_nodes)}")
|
||||
print(f" 关系数: {len(relations)}")
|
||||
|
||||
# 保存知识图谱数据
|
||||
reader.save_kg_data(all_nodes, relations, output_dir)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("处理完成！")
|
||||
print("="*60)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,438 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
从Excel提取数据生成知识图谱CSV文件
|
||||
保持原始数据表述不变
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
import re
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def extract_batches(df):
|
||||
"""提取所有唯一的批次（保持原始表述）"""
|
||||
batches = {}
|
||||
batch_counter = {}
|
||||
|
||||
for batch_str in df['项目批次'].dropna().unique():
|
||||
if not batch_str or str(batch_str).strip() == '':
|
||||
continue
|
||||
|
||||
# 解析批次字符串（可能包含多个批次）
|
||||
batch_items = re.split(r'[,、，]', str(batch_str))
|
||||
|
||||
for item in batch_items:
|
||||
item = item.strip()
|
||||
if not item:
|
||||
continue
|
||||
|
||||
# 生成批次ID（使用原始表述的哈希）
|
||||
if item not in batch_counter:
|
||||
batch_counter[item] = 1
|
||||
else:
|
||||
batch_counter[item] += 1
|
||||
|
||||
batch_id = f"BATCH-{abs(hash(item)) % 100000:05d}"
|
||||
|
||||
if batch_id not in batches:
|
||||
batches[batch_id] = {
|
||||
'name': item, # 保持原始表述
|
||||
'original_string': item
|
||||
}
|
||||
|
||||
return batches
|
||||
|
||||
|
||||
def parse_inheritor_field(inheritor_str):
|
||||
"""解析传承人字段，保持原始表述（如"吴明新(国)"）"""
|
||||
if not inheritor_str or str(inheritor_str).strip() in ['无', '']:
|
||||
return []
|
||||
|
||||
# 按顿号、逗号分割，保持原始表述
|
||||
inheritors = re.split(r'[、，,\n]', str(inheritor_str))
|
||||
inheritors = [inh.strip() for inh in inheritors if inh.strip() and inh.strip() != '无']
|
||||
return inheritors
|
||||
|
||||
|
||||
def extract_inheritors(df):
|
||||
"""提取所有唯一传承人（保持原始表述）"""
|
||||
inheritors = {}
|
||||
inheritor_counter = {}
|
||||
|
||||
for idx, row in df.iterrows():
|
||||
inheritor_str = row.get('代表性传承人', '')
|
||||
if not inheritor_str or str(inheritor_str).strip() in ['无', '']:
|
||||
continue
|
||||
|
||||
# 解析传承人列表
|
||||
inheritor_names = parse_inheritor_field(inheritor_str)
|
||||
|
||||
for name in inheritor_names:
|
||||
# 使用原始名称（包括括号）作为key
|
||||
if name not in inheritor_counter:
|
||||
inheritor_counter[name] = 1
|
||||
else:
|
||||
inheritor_counter[name] += 1
|
||||
|
||||
# 生成传承人ID
|
||||
inheritor_id = f"INH-{abs(hash(name)) % 100000:05d}-{inheritor_counter[name]}"
|
||||
|
||||
if inheritor_id not in inheritors:
|
||||
inheritors[inheritor_id] = {
|
||||
'name': name # 保持原始表述，如"吴明新(国)"
|
||||
}
|
||||
|
||||
return inheritors
|
||||
|
||||
|
||||
def extract_institutions(df):
|
||||
"""提取所有唯一保护机构（保持原始表述）"""
|
||||
institutions = {}
|
||||
inst_counter = {}
|
||||
|
||||
for idx, row in df.iterrows():
|
||||
inst_str = row.get('项目保护单位', '')
|
||||
if not inst_str or str(inst_str).strip() in ['', '无']:
|
||||
continue
|
||||
|
||||
inst_name = str(inst_str).strip()
|
||||
|
||||
# 使用原始机构名
|
||||
if inst_name not in inst_counter:
|
||||
inst_counter[inst_name] = 1
|
||||
else:
|
||||
inst_counter[inst_name] += 1
|
||||
|
||||
# 生成机构ID
|
||||
inst_id = f"INST-{abs(hash(inst_name)) % 100000:05d}-{inst_counter[inst_name]}"
|
||||
|
||||
if inst_id not in institutions:
|
||||
institutions[inst_id] = {
|
||||
'name': inst_name # 保持原始表述
|
||||
}
|
||||
|
||||
return institutions
|
||||
|
||||
|
||||
def parse_batch_field(batch_str, batches_dict):
|
||||
"""解析批次字段，返回批次ID列表"""
|
||||
if not batch_str or str(batch_str).strip() == '':
|
||||
return []
|
||||
|
||||
batches = re.split(r'[,、，]', str(batch_str))
|
||||
batch_ids = []
|
||||
|
||||
for batch in batches:
|
||||
batch = batch.strip()
|
||||
if batch:
|
||||
# 查找对应的批次ID
|
||||
for batch_id, batch_info in batches_dict.items():
|
||||
if batch_info['name'] == batch:
|
||||
batch_ids.append(batch_id)
|
||||
break
|
||||
|
||||
return batch_ids
|
||||
|
||||
|
||||
def get_institution_id(institution_name, institutions_dict):
|
||||
"""获取机构ID"""
|
||||
if not institution_name or str(institution_name).strip() in ['', '无']:
|
||||
return None
|
||||
|
||||
# 从已提取的机构中查找
|
||||
for inst_id, inst_info in institutions_dict.items():
|
||||
if inst_info['name'] == str(institution_name).strip():
|
||||
return inst_id
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_inheritor_id(inheritor_name, inheritors_dict):
|
||||
"""获取传承人ID"""
|
||||
if not inheritor_name or not inheritor_name.strip():
|
||||
return None
|
||||
|
||||
# 从已提取的传承人中查找
|
||||
for inheritor_id, inheritor_info in inheritors_dict.items():
|
||||
if inheritor_info['name'] == inheritor_name:
|
||||
return inheritor_id
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def clean_data(df):
|
||||
"""数据清洗"""
|
||||
# 去除完全空白的行
|
||||
df = df.dropna(how='all')
|
||||
|
||||
# 填充缺失值为空字符串
|
||||
for col in df.columns:
|
||||
if df[col].dtype == 'object':
|
||||
df[col] = df[col].fillna('')
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def build_nodes(df):
|
||||
"""构建节点数据（保持原始表述）"""
|
||||
nodes_list = []
|
||||
|
||||
print("正在构建节点...")
|
||||
|
||||
# 1. ICH_Project节点
|
||||
print(" - 构建ICH_Project节点...")
|
||||
for idx, row in df.iterrows():
|
||||
seq_num = row.get('总序号', idx + 1)
|
||||
project_name = row.get('项目名称', '')
|
||||
|
||||
# 确定级别（从批次字段推断）
|
||||
batch_str = str(row.get('项目批次', ''))
|
||||
level = '国家级' if '国家级' in batch_str else ('省级' if '省级' in batch_str else '')
|
||||
|
||||
nodes_list.append({
|
||||
'id': f"ICH-{int(seq_num)}",
|
||||
'label': str(project_name),
|
||||
'type': 'ICH_Project',
|
||||
'properties': json.dumps({
|
||||
'level': level,
|
||||
'category': row.get('类别', ''), # 保持原始表述，如"曲艺类"
|
||||
'batch': batch_str,
|
||||
'protection_unit': row.get('项目保护单位', '')
|
||||
}, ensure_ascii=False)
|
||||
})
|
||||
|
||||
# 2. Category节点（从数据中动态提取，保持原始表述）
|
||||
print(" - 构建Category节点...")
|
||||
unique_categories = df['类别'].dropna().unique()
|
||||
|
||||
for cat_name in unique_categories:
|
||||
# 使用类别名称作为ID（使用哈希避免特殊字符）
|
||||
cat_id = f"CAT-{abs(hash(cat_name)) % 100000:05d}"
|
||||
|
||||
nodes_list.append({
|
||||
'id': cat_id,
|
||||
'label': cat_name, # 保持原始表述，如"曲艺类"
|
||||
'type': 'Category',
|
||||
'properties': '{}'
|
||||
})
|
||||
|
||||
# 3. Batch节点（动态识别，保持原始表述）
|
||||
print(" - 构建Batch节点...")
|
||||
batches = extract_batches(df)
|
||||
for batch_id, batch_info in batches.items():
|
||||
nodes_list.append({
|
||||
'id': batch_id,
|
||||
'label': batch_info['name'], # 保持原始表述，如"国家级第1批"
|
||||
'type': 'Batch',
|
||||
'properties': json.dumps({
|
||||
'original_string': batch_info['original_string']
|
||||
}, ensure_ascii=False)
|
||||
})
|
||||
|
||||
# 4. Inheritor节点（动态识别，保持原始表述）
|
||||
print(" - 构建Inheritor节点...")
|
||||
inheritors = extract_inheritors(df)
|
||||
for inheritor_id, inheritor_info in inheritors.items():
|
||||
nodes_list.append({
|
||||
'id': inheritor_id,
|
||||
'label': inheritor_info['name'], # 保持原始表述，如"吴明新(国)"
|
||||
'type': 'Inheritor',
|
||||
'properties': '{}'
|
||||
})
|
||||
|
||||
# 5. Institution节点（动态识别，保持原始表述）
|
||||
print(" - 构建Institution节点...")
|
||||
institutions = extract_institutions(df)
|
||||
for inst_id, inst_info in institutions.items():
|
||||
nodes_list.append({
|
||||
'id': inst_id,
|
||||
'label': inst_info['name'], # 保持原始表述
|
||||
'type': 'Institution',
|
||||
'properties': '{}'
|
||||
})
|
||||
|
||||
return nodes_list, batches, inheritors, institutions
|
||||
|
||||
|
||||
def build_relations(df, batches, inheritors, institutions):
|
||||
"""构建关系数据"""
|
||||
relations_list = []
|
||||
|
||||
print("正在构建关系...")
|
||||
|
||||
# 建立名称到ID的快速查找映射
|
||||
category_map = {}
|
||||
for cat_name in df['类别'].dropna().unique():
|
||||
cat_id = f"CAT-{abs(hash(cat_name)) % 100000:05d}"
|
||||
category_map[cat_name] = cat_id
|
||||
|
||||
for idx, row in df.iterrows():
|
||||
seq_num = row.get('总序号', idx + 1)
|
||||
project_id = f"ICH-{int(seq_num)}"
|
||||
|
||||
# 1. 项目 → 类别
|
||||
category_name = row.get('类别', '')
|
||||
if category_name and category_name in category_map:
|
||||
category_id = category_map[category_name]
|
||||
relations_list.append({
|
||||
'source': project_id,
|
||||
'target': category_id,
|
||||
'type': 'BELONGS_TO',
|
||||
'properties': '{}'
|
||||
})
|
||||
|
||||
# 2. 项目 → 批次（支持多个批次）
|
||||
batch_str = row.get('项目批次', '')
|
||||
if batch_str:
|
||||
batch_ids = parse_batch_field(batch_str, batches)
|
||||
for batch_id in batch_ids:
|
||||
relations_list.append({
|
||||
'source': project_id,
|
||||
'target': batch_id,
|
||||
'type': 'SELECTED_IN_BATCH',
|
||||
'properties': '{}'
|
||||
})
|
||||
|
||||
# 3. 项目 → 保护机构
|
||||
institution_name = row.get('项目保护单位', '')
|
||||
if institution_name and str(institution_name).strip() not in ['', '无']:
|
||||
institution_id = get_institution_id(institution_name, institutions)
|
||||
if institution_id:
|
||||
relations_list.append({
|
||||
'source': project_id,
|
||||
'target': institution_id,
|
||||
'type': 'PROTECTED_BY',
|
||||
'properties': '{}'
|
||||
})
|
||||
|
||||
# 4. 项目 → 传承人（支持多个传承人）
|
||||
inheritor_str = row.get('代表性传承人', '')
|
||||
if inheritor_str:
|
||||
inheritor_names = parse_inheritor_field(inheritor_str)
|
||||
for name in inheritor_names:
|
||||
inheritor_id = get_inheritor_id(name, inheritors)
|
||||
if inheritor_id:
|
||||
relations_list.append({
|
||||
'source': project_id,
|
||||
'target': inheritor_id,
|
||||
'type': 'HAS_INHERITOR',
|
||||
'properties': '{}'
|
||||
})
|
||||
|
||||
return relations_list
|
||||
|
||||
|
||||
def generate_report(nodes_df, relations_df, output_dir):
|
||||
"""生成统计报告"""
|
||||
report_lines = []
|
||||
report_lines.append("# 知识图谱CSV提取报告\n")
|
||||
report_lines.append(f"生成时间: {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
|
||||
report_lines.append("---\n\n")
|
||||
|
||||
# 节点统计
|
||||
report_lines.append("## 节点统计\n\n")
|
||||
report_lines.append(f"**节点总数**: {len(nodes_df)}\n\n")
|
||||
|
||||
node_type_counts = nodes_df['type'].value_counts().sort_index()
|
||||
report_lines.append("| 节点类型 | 数量 | 占比 |\n")
|
||||
report_lines.append("|---------|------|------|\n")
|
||||
for node_type, count in node_type_counts.items():
|
||||
percentage = (count / len(nodes_df) * 100)
|
||||
report_lines.append(f"| {node_type} | {count} | {percentage:.1f}% |\n")
|
||||
|
||||
# 关系统计
|
||||
report_lines.append("\n## 关系统计\n\n")
|
||||
report_lines.append(f"**关系总数**: {len(relations_df)}\n\n")
|
||||
|
||||
rel_type_counts = relations_df['type'].value_counts().sort_index()
|
||||
report_lines.append("| 关系类型 | 数量 | 占比 |\n")
|
||||
report_lines.append("|---------|------|------|\n")
|
||||
for rel_type, count in rel_type_counts.items():
|
||||
percentage = (count / len(relations_df) * 100)
|
||||
report_lines.append(f"| {rel_type} | {count} | {percentage:.1f}% |\n")
|
||||
|
||||
# 保存报告
|
||||
report_path = output_dir / 'extraction_report.md'
|
||||
with open(report_path, 'w', encoding='utf-8') as f:
|
||||
f.writelines(report_lines)
|
||||
|
||||
print(f"\n报告已生成: {report_path}")
|
||||
|
||||
# 打印统计信息
|
||||
print("\n" + "="*50)
|
||||
print("数据提取完成")
|
||||
print("="*50)
|
||||
print(f"\n节点总数: {len(nodes_df)}")
|
||||
for node_type, count in node_type_counts.items():
|
||||
print(f" - {node_type}: {count}")
|
||||
print(f"\n关系总数: {len(relations_df)}")
|
||||
for rel_type, count in rel_type_counts.items():
|
||||
print(f" - {rel_type}: {count}")
|
||||
print("="*50)
|
||||
|
||||
|
||||
def extract_data_from_excel():
|
||||
"""从Excel提取数据并生成知识图谱CSV文件"""
|
||||
|
||||
print("开始从Excel提取数据...")
|
||||
|
||||
# 1. 读取Excel
|
||||
excel_path = r'E:\Project\2026_KG_ICH\data\黑龙江国家级和省级非遗名单.xlsx'
|
||||
print(f"读取文件: {excel_path}")
|
||||
|
||||
df = pd.read_excel(excel_path)
|
||||
print(f"原始数据: {len(df)} 行 x {len(df.columns)} 列")
|
||||
|
||||
# 2. 提取所需列
|
||||
columns = ['总序号', '类别', '项目名称', '项目批次', '项目保护单位', '代表性传承人']
|
||||
print(f"\n提取列: {', '.join(columns)}")
|
||||
|
||||
# 检查列是否存在
|
||||
available_cols = [col for col in columns if col in df.columns]
|
||||
if len(available_cols) < len(columns):
|
||||
missing = set(columns) - set(available_cols)
|
||||
print(f"警告: 以下列不存在: {missing}")
|
||||
|
||||
df = df[available_cols]
|
||||
print(f"提取后数据: {len(df)} 行 x {len(df.columns)} 列")
|
||||
|
||||
# 3. 数据清洗
|
||||
print("\n数据清洗...")
|
||||
df = clean_data(df)
|
||||
print(f"清洗后数据: {len(df)} 行")
|
||||
|
||||
# 4. 构建节点
|
||||
nodes_list, batches, inheritors, institutions = build_nodes(df)
|
||||
nodes_df = pd.DataFrame(nodes_list)
|
||||
print(f"节点数据: {len(nodes_df)} 行")
|
||||
|
||||
# 5. 构建关系
|
||||
relations_list = build_relations(df, batches, inheritors, institutions)
|
||||
relations_df = pd.DataFrame(relations_list)
|
||||
print(f"关系数据: {len(relations_df)} 行")
|
||||
|
||||
# 6. 保存CSV
|
||||
output_dir = Path(r'E:\Project\2026_KG_ICH\dofile\kg_project\output')
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"\n保存文件到: {output_dir}")
|
||||
|
||||
# 保存为UTF-8-BOM编码（Excel友好）
|
||||
nodes_path = output_dir / 'nodes.csv'
|
||||
relations_path = output_dir / 'rels.csv'
|
||||
|
||||
nodes_df.to_csv(nodes_path, index=False, encoding='utf-8-sig')
|
||||
relations_df.to_csv(relations_path, index=False, encoding='utf-8-sig')
|
||||
|
||||
print(f" - nodes.csv: {nodes_path}")
|
||||
print(f" - rels.csv: {relations_path}")
|
||||
|
||||
# 7. 生成统计报告
|
||||
generate_report(nodes_df, relations_df, output_dir)
|
||||
|
||||
return nodes_df, relations_df
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
extract_data_from_excel()
|
||||
@@ -0,0 +1,382 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
知识图谱可视化工具
|
||||
使用networkx和matplotlib绘制知识图谱
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
import networkx as nx
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib import font_manager
|
||||
import matplotlib.patches as mpatches
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
|
||||
|
||||
# 设置中文字体
|
||||
def setup_chinese_font():
|
||||
"""设置中文字体"""
|
||||
# 尝试多种中文字体
|
||||
chinese_fonts = [
|
||||
'Microsoft YaHei',
|
||||
'SimHei',
|
||||
'SimSun',
|
||||
'KaiTi',
|
||||
'FangSong',
|
||||
'STXihei',
|
||||
'STSong',
|
||||
'STKaiti',
|
||||
'STFangsong'
|
||||
]
|
||||
|
||||
for font in chinese_fonts:
|
||||
try:
|
||||
plt.rcParams['font.sans-serif'] = [font]
|
||||
plt.rcParams['axes.unicode_minus'] = False
|
||||
break
|
||||
except:
|
||||
continue
|
||||
|
||||
print(f"使用字体: {plt.rcParams['font.sans-serif'][0]}")
|
||||
|
||||
|
||||
# 节点类型颜色映射
|
||||
NODE_TYPE_COLORS = {
|
||||
'ICH_Project': '#FF6B6B', # 红色 - 项目
|
||||
'Category': '#4ECDC4', # 青色 - 类别
|
||||
'Batch': '#95E1D3', # 绿色 - 批次
|
||||
'Inheritor': '#FFD93D', # 黄色 - 传承人
|
||||
'Institution': '#6C5CE7' # 紫色 - 机构
|
||||
}
|
||||
|
||||
# 节点类型大小映射
|
||||
NODE_TYPE_SIZES = {
|
||||
'ICH_Project': 300,
|
||||
'Category': 500,
|
||||
'Batch': 350,
|
||||
'Inheritor': 200,
|
||||
'Institution': 250
|
||||
}
|
||||
|
||||
|
||||
def load_graph_data(nodes_csv, rels_csv):
|
||||
"""加载图谱数据"""
|
||||
print("正在加载数据...")
|
||||
|
||||
# 读取节点和关系
|
||||
nodes_df = pd.read_csv(nodes_csv, encoding='utf-8-sig')
|
||||
rels_df = pd.read_csv(rels_csv, encoding='utf-8-sig')
|
||||
|
||||
print(f" - 节点: {len(nodes_df)}")
|
||||
print(f" - 关系: {len(rels_df)}")
|
||||
|
||||
return nodes_df, rels_df
|
||||
|
||||
|
||||
def build_networkx_graph(nodes_df, rels_df):
|
||||
"""构建NetworkX图"""
|
||||
print("正在构建图...")
|
||||
|
||||
G = nx.DiGraph() # 有向图
|
||||
|
||||
# 添加节点
|
||||
for idx, row in nodes_df.iterrows():
|
||||
node_id = row['id']
|
||||
label = row['label']
|
||||
node_type = row['type']
|
||||
|
||||
# 截断过长的标签
|
||||
if len(label) > 10:
|
||||
display_label = label[:10] + '...'
|
||||
else:
|
||||
display_label = label
|
||||
|
||||
G.add_node(
|
||||
node_id,
|
||||
label=display_label,
|
||||
full_label=label,
|
||||
node_type=node_type,
|
||||
color=NODE_TYPE_COLORS.get(node_type, '#CCCCCC'),
|
||||
size=NODE_TYPE_SIZES.get(node_type, 200)
|
||||
)
|
||||
|
||||
# 添加边
|
||||
for idx, row in rels_df.iterrows():
|
||||
source = row['source']
|
||||
target = row['target']
|
||||
rel_type = row['type']
|
||||
|
||||
if source in G.nodes() and target in G.nodes():
|
||||
G.add_edge(source, target, rel_type=rel_type)
|
||||
|
||||
print(f" - 节点数: {G.number_of_nodes()}")
|
||||
print(f" - 边数: {G.number_of_edges()}")
|
||||
|
||||
return G
|
||||
|
||||
|
||||
def filter_graph_by_type(G, include_types=None):
|
||||
"""按节点类型过滤图"""
|
||||
if include_types is None:
|
||||
return G
|
||||
|
||||
nodes_to_keep = [n for n, d in G.nodes(data=True)
|
||||
if d.get('node_type') in include_types]
|
||||
|
||||
return G.subgraph(nodes_to_keep).copy()
|
||||
|
||||
|
||||
def draw_graph(G, output_path, title="知识图谱", layout='spring'):
|
||||
"""绘制知识图谱"""
|
||||
print(f"正在绘制图谱: {title}")
|
||||
|
||||
plt.figure(figsize=(20, 16))
|
||||
|
||||
# 选择布局算法
|
||||
if layout == 'spring':
|
||||
pos = nx.spring_layout(G, k=2, iterations=50, seed=42)
|
||||
elif layout == 'circular':
|
||||
pos = nx.circular_layout(G)
|
||||
elif layout == 'kamada_kawai':
|
||||
pos = nx.kamada_kawai_layout(G)
|
||||
elif layout == 'random':
|
||||
pos = nx.random_layout(G)
|
||||
else:
|
||||
pos = nx.spring_layout(G, k=2, iterations=50, seed=42)
|
||||
|
||||
# 按节点类型分组
|
||||
node_types = {}
|
||||
for node, data in G.nodes(data=True):
|
||||
node_type = data.get('node_type', 'Unknown')
|
||||
if node_type not in node_types:
|
||||
node_types[node_type] = []
|
||||
node_types[node_type].append(node)
|
||||
|
||||
# 绘制边
|
||||
nx.draw_networkx_edges(
|
||||
G, pos,
|
||||
alpha=0.3,
|
||||
width=0.5,
|
||||
edge_color='gray',
|
||||
arrows=True,
|
||||
arrowsize=10,
|
||||
arrowstyle='->,head_width=0.2,head_length=0.3'
|
||||
)
|
||||
|
||||
# 按类型绘制节点
|
||||
for node_type, nodes in node_types.items():
|
||||
color = NODE_TYPE_COLORS.get(node_type, '#CCCCCC')
|
||||
size = NODE_TYPE_SIZES.get(node_type, 200)
|
||||
|
||||
nx.draw_networkx_nodes(
|
||||
G, pos,
|
||||
nodelist=nodes,
|
||||
node_color=color,
|
||||
node_size=size,
|
||||
alpha=0.8,
|
||||
edgecolors='white',
|
||||
linewidths=1
|
||||
)
|
||||
|
||||
# 绘制标签（只对重要节点）
|
||||
important_nodes = []
|
||||
important_labels = {}
|
||||
|
||||
for node, data in G.nodes(data=True):
|
||||
node_type = data.get('node_type')
|
||||
# 只显示类别、批次和部分重要节点的标签
|
||||
if node_type in ['Category', 'Batch'] or (
|
||||
node_type == 'ICH_Project' and data.get('size', 0) > 400
|
||||
):
|
||||
important_nodes.append(node)
|
||||
important_labels[node] = data.get('label', node)
|
||||
|
||||
if len(important_nodes) <= 100: # 节点不多时显示所有标签
|
||||
nx.draw_networkx_labels(
|
||||
G, pos,
|
||||
labels=important_labels,
|
||||
font_size=8,
|
||||
font_weight='bold',
|
||||
font_family='sans-serif'
|
||||
)
|
||||
else:
|
||||
# 节点太多时只显示类别标签
|
||||
category_labels = {n: d['label'] for n, d in G.nodes(data=True)
|
||||
if d.get('node_type') == 'Category'}
|
||||
nx.draw_networkx_labels(
|
||||
G, pos,
|
||||
labels=category_labels,
|
||||
font_size=10,
|
||||
font_weight='bold'
|
||||
)
|
||||
|
||||
# 图例
|
||||
legend_patches = []
|
||||
for node_type, color in NODE_TYPE_COLORS.items():
|
||||
if node_type in node_types:
|
||||
patch = mpatches.Patch(color=color, label=node_type)
|
||||
legend_patches.append(patch)
|
||||
|
||||
plt.legend(
|
||||
handles=legend_patches,
|
||||
loc='upper right',
|
||||
fontsize=12,
|
||||
framealpha=0.9
|
||||
)
|
||||
|
||||
plt.title(title, fontsize=16, fontweight='bold', pad=20)
|
||||
plt.axis('off')
|
||||
plt.tight_layout()
|
||||
|
||||
# 保存图片
|
||||
plt.savefig(output_path, dpi=150, bbox_inches='tight')
|
||||
print(f" - 保存到: {output_path}")
|
||||
plt.close()
|
||||
|
||||
|
||||
def draw_subgraphs(G, output_dir):
|
||||
"""绘制子图（按节点类型分组）"""
|
||||
print("\n正在绘制子图...")
|
||||
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 1. 只显示项目和类别
|
||||
print(" 1. 项目-类别关系图...")
|
||||
G1 = filter_graph_by_type(G, ['ICH_Project', 'Category'])
|
||||
draw_graph(
|
||||
G1,
|
||||
output_dir / 'kg_project_category.png',
|
||||
title='非遗项目与类别关系',
|
||||
layout='spring'
|
||||
)
|
||||
|
||||
# 2. 只显示项目和传承人
|
||||
print(" 2. 项目-传承人关系图...")
|
||||
G2 = filter_graph_by_type(G, ['ICH_Project', 'Inheritor'])
|
||||
if G2.number_of_nodes() > 0:
|
||||
draw_graph(
|
||||
G2,
|
||||
output_dir / 'kg_project_inheritor.png',
|
||||
title='非遗项目与传承人关系',
|
||||
layout='spring'
|
||||
)
|
||||
|
||||
# 3. 只显示项目、类别和批次
|
||||
print(" 3. 项目-类别-批次关系图...")
|
||||
G3 = filter_graph_by_type(G, ['ICH_Project', 'Category', 'Batch'])
|
||||
draw_graph(
|
||||
G3,
|
||||
output_dir / 'kg_project_category_batch.png',
|
||||
title='非遗项目、类别与批次关系',
|
||||
layout='kamada_kawai'
|
||||
)
|
||||
|
||||
# 4. 完整图谱（抽样显示）
|
||||
print(" 4. 完整知识图谱...")
|
||||
if G.number_of_nodes() > 500:
|
||||
# 节点太多时，只显示连接度高的节点
|
||||
degrees = dict(G.degree())
|
||||
high_degree_nodes = [n for n, d in degrees.items() if d >= 3]
|
||||
G_sample = G.subgraph(high_degree_nodes).copy()
|
||||
draw_graph(
|
||||
G_sample,
|
||||
output_dir / 'kg_full_sampled.png',
|
||||
title=f'完整知识图谱（抽样，显示{G_sample.number_of_nodes()}个节点）',
|
||||
layout='spring'
|
||||
)
|
||||
else:
|
||||
draw_graph(
|
||||
G,
|
||||
output_dir / 'kg_full.png',
|
||||
title='完整知识图谱',
|
||||
layout='spring'
|
||||
)
|
||||
|
||||
|
||||
def print_statistics(G):
|
||||
"""打印图统计信息"""
|
||||
print("\n" + "="*60)
|
||||
print("图谱统计信息")
|
||||
print("="*60)
|
||||
|
||||
print(f"\n节点总数: {G.number_of_nodes()}")
|
||||
print(f"边总数: {G.number_of_edges()}")
|
||||
|
||||
# 按类型统计节点
|
||||
print("\n节点类型分布:")
|
||||
node_types = {}
|
||||
for node, data in G.nodes(data=True):
|
||||
node_type = data.get('node_type', 'Unknown')
|
||||
node_types[node_type] = node_types.get(node_type, 0) + 1
|
||||
|
||||
for node_type, count in sorted(node_types.items()):
|
||||
percentage = (count / G.number_of_nodes() * 100)
|
||||
print(f" - {node_type}: {count} ({percentage:.1f}%)")
|
||||
|
||||
# 按类型统计边
|
||||
print("\n关系类型分布:")
|
||||
rel_types = {}
|
||||
for u, v, data in G.edges(data=True):
|
||||
rel_type = data.get('rel_type', 'Unknown')
|
||||
rel_types[rel_type] = rel_types.get(rel_type, 0) + 1
|
||||
|
||||
for rel_type, count in sorted(rel_types.items()):
|
||||
percentage = (count / G.number_of_edges() * 100)
|
||||
print(f" - {rel_type}: {count} ({percentage:.1f}%)")
|
||||
|
||||
# 连接度统计
|
||||
degrees = [d for n, d in G.degree()]
|
||||
print(f"\n连接度统计:")
|
||||
print(f" - 平均连接度: {np.mean(degrees):.2f}")
|
||||
print(f" - 最大连接度: {max(degrees)}")
|
||||
print(f" - 最小连接度: {min(degrees)}")
|
||||
|
||||
# 找出连接度最高的节点
|
||||
top_nodes = sorted(G.degree(), key=lambda x: x[1], reverse=True)[:10]
|
||||
print(f"\n连接度最高的10个节点:")
|
||||
for node, degree in top_nodes:
|
||||
node_data = G.nodes[node]
|
||||
label = node_data.get('full_label', node)
|
||||
node_type = node_data.get('node_type', '')
|
||||
print(f" - [{node_type}] {label}: {degree}个连接")
|
||||
|
||||
print("="*60)
|
||||
|
||||
|
||||
def visualize_kg(nodes_csv, rels_csv, output_dir):
|
||||
"""可视化知识图谱"""
|
||||
print("="*60)
|
||||
print("知识图谱可视化工具")
|
||||
print("="*60)
|
||||
|
||||
# 设置中文字体
|
||||
setup_chinese_font()
|
||||
|
||||
# 加载数据
|
||||
nodes_df, rels_df = load_graph_data(nodes_csv, rels_csv)
|
||||
|
||||
# 构建图
|
||||
G = build_networkx_graph(nodes_df, rels_df)
|
||||
|
||||
# 打印统计信息
|
||||
print_statistics(G)
|
||||
|
||||
# 绘制子图
|
||||
draw_subgraphs(G, output_dir)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("可视化完成！")
|
||||
print("="*60)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 输入文件
|
||||
nodes_csv = r'E:\Project\2026_KG_ICH\dofile\kg_project\output\nodes.csv'
|
||||
rels_csv = r'E:\Project\2026_KG_ICH\dofile\kg_project\output\rels.csv'
|
||||
|
||||
# 输出目录
|
||||
output_dir = r'E:\Project\2026_KG_ICH\dofile\kg_project\output\visualizations'
|
||||
|
||||
# 执行可视化
|
||||
visualize_kg(nodes_csv, rels_csv, output_dir)
|
||||
Reference in New Issue
Block a user