Initial commit: 国土空间规划课程智能体 v1.0

单容器 Docker 架构的国土空间规划课程智能问答系统,集成 FastAPI 后端与 Next.js 前端。

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-05-22 09:40:18 +08:00
commit ddbb79b9f6
167 changed files with 44147 additions and 0 deletions
+311
View File
@@ -0,0 +1,311 @@
#!/usr/bin/env python3
"""
数据备份脚本 - 备份数据库和向量存储
"""
import os
import sys
import shutil
import json
import tarfile
from datetime import datetime
from pathlib import Path
import logging
from typing import Dict, List, Optional
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
class DataBackup:
"""数据备份类"""
def __init__(self, backup_dir: str = "./backups"):
self.backup_dir = Path(backup_dir)
self.backup_dir.mkdir(parents=True, exist_ok=True)
# 备份配置
self.backup_items = {
"database": {
"name": "数据库",
"paths": ["course_agent.db"],
"enabled": True
},
"vector_store": {
"name": "向量存储",
"paths": ["vector_store"],
"enabled": True
},
"knowledge_base": {
"name": "知识库",
"paths": ["data/knowledge_base"],
"enabled": True
},
"uploads": {
"name": "上传文件",
"paths": ["uploads"],
"enabled": True
},
"logs": {
"name": "日志文件",
"paths": ["logs"],
"enabled": False # 默认不备份日志
}
}
def create_backup(self, backup_name: Optional[str] = None) -> str:
"""创建备份"""
if not backup_name:
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
backup_name = f"backup_{timestamp}"
backup_path = self.backup_dir / backup_name
backup_path.mkdir(exist_ok=True)
logger.info(f"开始创建备份: {backup_name}")
backup_info = {
"name": backup_name,
"created_at": datetime.now().isoformat(),
"items": {}
}
# 备份每个项目
for item_id, item_config in self.backup_items.items():
if not item_config["enabled"]:
continue
item_backup_path = backup_path / item_id
item_backup_path.mkdir(exist_ok=True)
item_info = self._backup_item(item_id, item_config, item_backup_path)
backup_info["items"][item_id] = item_info
if item_info["status"] == "success":
logger.info(f"{item_config['name']}: {item_info['file_count']} 个文件")
else:
logger.warning(f"⚠️ {item_config['name']}: {item_info['error']}")
# 保存备份信息
info_file = backup_path / "backup_info.json"
with open(info_file, "w", encoding="utf-8") as f:
json.dump(backup_info, f, ensure_ascii=False, indent=2)
# 创建压缩包
tar_path = self._create_tar_archive(backup_path)
# 清理临时目录
shutil.rmtree(backup_path)
logger.info(f"✅ 备份创建完成: {tar_path}")
return str(tar_path)
def _backup_item(self, item_id: str, item_config: Dict, backup_path: Path) -> Dict:
"""备份单个项目"""
item_info = {
"name": item_config["name"],
"paths": item_config["paths"],
"status": "pending",
"file_count": 0,
"total_size": 0
}
try:
file_count = 0
total_size = 0
for path_str in item_config["paths"]:
source_path = Path(path_str)
if not source_path.exists():
logger.warning(f"路径不存在: {source_path}")
continue
if source_path.is_file():
# 备份单个文件
shutil.copy2(source_path, backup_path / source_path.name)
file_count += 1
total_size += source_path.stat().st_size
else:
# 备份目录
for root, dirs, files in os.walk(source_path):
for file in files:
source_file = Path(root) / file
rel_path = source_file.relative_to(source_path)
target_file = backup_path / rel_path
target_file.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source_file, target_file)
file_count += 1
total_size += source_file.stat().st_size
item_info.update({
"status": "success",
"file_count": file_count,
"total_size": total_size
})
except Exception as e:
item_info.update({
"status": "error",
"error": str(e)
})
return item_info
def _create_tar_archive(self, backup_path: Path) -> Path:
"""创建tar.gz压缩包"""
tar_filename = f"{backup_path.name}.tar.gz"
tar_path = self.backup_dir / tar_filename
with tarfile.open(tar_path, "w:gz") as tar:
# 添加备份信息文件
info_file = backup_path / "backup_info.json"
if info_file.exists():
tar.add(info_file, arcname="backup_info.json")
# 添加其他文件
for item_dir in backup_path.iterdir():
if item_dir.is_dir():
tar.add(item_dir, arcname=item_dir.name)
return tar_path
def list_backups(self) -> List[Dict]:
"""列出所有备份"""
backups = []
for item in self.backup_dir.iterdir():
if item.is_file() and item.suffix == ".gz":
backup_info = self._get_backup_info(item)
if backup_info:
backups.append(backup_info)
# 按创建时间排序
backups.sort(key=lambda x: x.get("created_at", ""), reverse=True)
return backups
def _get_backup_info(self, backup_file: Path) -> Optional[Dict]:
"""从备份文件中获取信息"""
try:
with tarfile.open(backup_file, "r:gz") as tar:
# 查找备份信息文件
info_member = None
for member in tar.getmembers():
if member.name == "backup_info.json":
info_member = member
break
if info_member:
# 提取并读取信息文件
info_data = tar.extractfile(info_member)
if info_data:
info = json.load(info_data)
info["filename"] = backup_file.name
info["size"] = backup_file.stat().st_size
return info
except Exception as e:
logger.error(f"读取备份信息失败 {backup_file}: {e}")
return None
def restore_backup(self, backup_filename: str, restore_dir: str = ".") -> bool:
"""恢复备份"""
backup_file = self.backup_dir / backup_filename
if not backup_file.exists():
logger.error(f"备份文件不存在: {backup_filename}")
return False
restore_path = Path(restore_dir)
restore_path.mkdir(parents=True, exist_ok=True)
logger.info(f"开始恢复备份: {backup_filename}")
try:
with tarfile.open(backup_file, "r:gz") as tar:
# 提取所有文件
tar.extractall(restore_path)
logger.info(f"✅ 备份恢复完成到: {restore_path}")
return True
except Exception as e:
logger.error(f"恢复备份失败: {e}")
return False
def cleanup_old_backups(self, keep_days: int = 30) -> int:
"""清理旧备份"""
cutoff_date = datetime.now() - timedelta(days=keep_days)
deleted_count = 0
for backup_info in self.list_backups():
created_at_str = backup_info.get("created_at")
if not created_at_str:
continue
try:
created_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
if created_at < cutoff_date:
backup_file = self.backup_dir / backup_info["filename"]
backup_file.unlink()
logger.info(f"删除旧备份: {backup_info['filename']}")
deleted_count += 1
except Exception as e:
logger.error(f"解析备份日期失败: {e}")
return deleted_count
def main():
"""主函数"""
import argparse
parser = argparse.ArgumentParser(description="数据备份工具")
parser.add_argument("action", choices=["create", "list", "restore", "cleanup"],
help="执行的操作")
parser.add_argument("--name", help="备份名称(用于create和restore")
parser.add_argument("--backup-dir", default="./backups", help="备份目录")
parser.add_argument("--restore-dir", default=".", help="恢复目录")
parser.add_argument("--keep-days", type=int, default=30, help="保留备份的天数")
args = parser.parse_args()
backup = DataBackup(args.backup_dir)
if args.action == "create":
backup_path = backup.create_backup(args.name)
print(f"备份创建完成: {backup_path}")
elif args.action == "list":
backups = backup.list_backups()
print(f"\n找到 {len(backups)} 个备份:\n")
for b in backups:
print(f"名称: {b['name']}")
print(f"文件: {b['filename']}")
print(f"大小: {b['size'] / 1024 / 1024:.2f} MB")
print(f"时间: {b['created_at']}")
print(f"项目: {', '.join(b['items'].keys())}")
print("-" * 40)
elif args.action == "restore":
if not args.name:
print("错误: 恢复备份需要指定备份文件名")
sys.exit(1)
success = backup.restore_backup(args.name, args.restore_dir)
if success:
print("✅ 备份恢复成功")
else:
print("❌ 备份恢复失败")
sys.exit(1)
elif args.action == "cleanup":
deleted = backup.cleanup_old_backups(args.keep_days)
print(f"清理了 {deleted} 个旧备份")
if __name__ == "__main__":
main()
+304
View File
@@ -0,0 +1,304 @@
#!/usr/bin/env python3
"""
Docker环境初始化脚本
"""
import os
import sys
import time
import logging
from pathlib import Path
from datetime import datetime
# 添加src目录到Python路径
sys.path.append(str(Path(__file__).parent.parent))
from src.core.database import SessionLocal, create_tables, check_database_connection, get_database_info
from src.core.config import get_settings, is_postgresql_database
from src.models.user import User
from src.models.knowledge_base import KnowledgeBase
from src.models.forum import ForumCategory
from src.services.auth_service import AuthService
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
class DockerInitializer:
"""Docker环境初始化"""
def __init__(self):
self.settings = get_settings()
self.session = SessionLocal()
self.auth_service = AuthService(self.session)
def initialize(self):
"""执行初始化"""
logger.info("=" * 60)
logger.info("Docker环境初始化")
logger.info("=" * 60)
# 1. 检查数据库连接
if not self._check_database():
return False
# 2. 创建数据库表
self._create_tables()
# 3. 创建默认管理员用户
self._create_admin_user()
# 4. 创建系统知识库
self._create_system_knowledge_bases()
# 5. 创建论坛分类
self._create_forum_categories()
# 6. 检查数据目录
self._check_data_directories()
logger.info("✅ Docker环境初始化完成")
return True
def _check_database(self):
"""检查数据库连接"""
logger.info("检查数据库连接...")
max_retries = 10
retry_delay = 5
for i in range(max_retries):
try:
if check_database_connection():
db_info = get_database_info()
logger.info(f"✅ 数据库连接成功: {db_info.get('type', 'Unknown')}")
if is_postgresql_database():
logger.info(f" 数据库: {db_info.get('database', 'Unknown')}")
logger.info(f" 版本: {db_info.get('version', 'Unknown')}")
return True
else:
logger.warning(f"数据库连接失败,重试 {i+1}/{max_retries}...")
except Exception as e:
logger.warning(f"数据库连接异常: {e}")
if i < max_retries - 1:
time.sleep(retry_delay)
logger.error("❌ 数据库连接失败,请检查数据库服务")
return False
def _create_tables(self):
"""创建数据库表"""
logger.info("创建数据库表...")
try:
create_tables()
logger.info("✅ 数据库表创建完成")
except Exception as e:
logger.error(f"❌ 创建数据库表失败: {e}")
raise
def _create_admin_user(self):
"""创建默认管理员用户"""
logger.info("创建默认管理员用户...")
try:
# 检查是否已存在管理员用户
admin_user = self.session.query(User).filter(
User.username == "admin"
).first()
if admin_user:
logger.info("管理员用户已存在")
return
# 创建管理员用户
admin_data = {
"username": "admin",
"email": "admin@course-agent.local",
"password": "admin123", # 默认密码,首次登录后应修改
"full_name": "系统管理员",
"is_superuser": True
}
admin_user = self.auth_service.create_user(**admin_data)
if admin_user:
logger.info("✅ 管理员用户创建成功")
logger.info(f" 用户名: {admin_user.username}")
logger.info(f" 邮箱: {admin_user.email}")
logger.info("⚠️ 请尽快修改默认密码")
else:
logger.warning("管理员用户创建失败")
except Exception as e:
logger.error(f"创建管理员用户失败: {e}")
def _create_system_knowledge_bases(self):
"""创建系统知识库"""
logger.info("创建系统知识库...")
system_kbs = [
{
"name": "国土空间法律法规集",
"description": "国土空间规划相关法律法规、政策文件",
"is_system": True,
"is_active": True
},
{
"name": "国土空间规划原理集",
"description": "国土空间规划理论、原理、方法",
"is_system": True,
"is_active": True
},
{
"name": "国土空间规划案例集",
"description": "各级国土空间规划案例",
"is_system": True,
"is_active": True
}
]
created_count = 0
for kb_data in system_kbs:
try:
# 检查是否已存在
existing = self.session.query(KnowledgeBase).filter(
KnowledgeBase.name == kb_data["name"],
KnowledgeBase.is_system == True
).first()
if existing:
continue
# 创建知识库
kb = KnowledgeBase(
user_id=1, # 管理员用户ID
**kb_data
)
self.session.add(kb)
created_count += 1
except Exception as e:
logger.error(f"创建知识库失败 {kb_data['name']}: {e}")
if created_count > 0:
self.session.commit()
logger.info(f"✅ 创建了 {created_count} 个系统知识库")
else:
logger.info("系统知识库已存在")
def _create_forum_categories(self):
"""创建论坛分类"""
logger.info("创建论坛分类...")
categories = [
{
"name": "系统使用优化建议",
"description": "分享系统使用经验,提出改进建议"
},
{
"name": "课程学习反馈",
"description": "交流学习心得,讨论课程内容"
}
]
created_count = 0
for cat_data in categories:
try:
# 检查是否已存在
existing = self.session.query(ForumCategory).filter(
ForumCategory.name == cat_data["name"]
).first()
if existing:
continue
# 创建分类
category = ForumCategory(**cat_data)
self.session.add(category)
created_count += 1
except Exception as e:
logger.error(f"创建论坛分类失败 {cat_data['name']}: {e}")
if created_count > 0:
self.session.commit()
logger.info(f"✅ 创建了 {created_count} 个论坛分类")
else:
logger.info("论坛分类已存在")
def _check_data_directories(self):
"""检查数据目录"""
logger.info("检查数据目录...")
directories = [
self.settings.vector_store_path,
self.settings.upload_dir,
self.settings.knowledge_base_dir,
self.settings.generated_images_dir,
os.path.dirname(self.settings.log_file),
]
for dir_path in directories:
try:
path = Path(dir_path)
path.mkdir(parents=True, exist_ok=True)
# 检查权限
test_file = path / ".test_write"
test_file.touch()
test_file.unlink()
logger.info(f"✅ 目录可访问: {dir_path}")
except Exception as e:
logger.error(f"❌ 目录访问失败 {dir_path}: {e}")
def close(self):
"""关闭资源"""
if self.session:
self.session.close()
def main():
"""主函数"""
initializer = None
try:
initializer = DockerInitializer()
success = initializer.initialize()
if success:
print("\n" + "=" * 60)
print("初始化完成!")
print("=" * 60)
print("\n访问信息:")
print(f" 前端: http://localhost:8001")
print(f" 后端API: http://localhost:8000")
print(f" API文档: http://localhost:8000/docs")
print("\n默认管理员账号:")
print(" 用户名: admin")
print(" 密码: admin123")
print(" ⚠️ 请尽快修改默认密码")
print("\n" + "=" * 60)
else:
print("❌ 初始化失败")
sys.exit(1)
except Exception as e:
print(f"❌ 初始化过程中发生错误: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
finally:
if initializer:
initializer.close()
if __name__ == "__main__":
main()
+162
View File
@@ -0,0 +1,162 @@
"""
导入书籍结构到数据库
解析LaTeX文件并导入书籍、章节、节、小节(知识点)
层级结构:书籍 -> Chapter -> Section -> Subsection(知识点)
"""
import sys
import re
from pathlib import Path
# 添加项目根目录到路径
sys.path.insert(0, str(Path(__file__).parent.parent))
from sqlalchemy.orm import Session
from src.core.database import SessionLocal, create_tables
from src.core.config import get_settings
from src.models.book_structure import Book, Chapter, Section, Subsection
from src.services.latex_parser import LaTeXParser
def import_book_structure():
"""导入书籍结构"""
settings = get_settings()
book_dir = Path(settings.book_dir)
# 如果相对路径不存在,尝试从项目根目录查找
if not book_dir.exists():
# 脚本位置: dofile/backend/scripts/import_book_structure.py
# 项目根目录: 向上3级
script_dir = Path(__file__).parent.parent.parent.parent # 项目根目录
book_dir = script_dir / "data" / "book"
if not book_dir.exists():
print(f"错误: 书籍目录不存在")
print(f"尝试的路径1: {settings.book_dir}")
print(f"尝试的路径2: {book_dir}")
return
print(f"使用书籍目录: {book_dir}")
# 创建数据库表
print("创建数据库表...")
create_tables()
# 创建数据库会话
db: Session = SessionLocal()
try:
# 清空旧数据(如果存在)
db.query(Subsection).delete()
db.query(Section).delete()
db.query(Chapter).delete()
db.query(Book).delete()
db.commit()
print("已清空旧的书籍结构数据。")
# 创建书籍
book_title = "国土空间规划:概念、原理、方法"
book_description = "基于LaTeX书籍内容构建的知识图谱系统"
book = Book(title=book_title, description=book_description)
db.add(book)
db.commit()
db.refresh(book)
print(f"已创建书籍: {book.title}")
parser = LaTeXParser(book_dir)
# 解析主文件获取章节顺序
main_tex_path = book_dir / "main.tex"
if not main_tex_path.exists():
print(f"错误: main.tex 文件不存在于 {book_dir}")
return
main_content = main_tex_path.read_text(encoding='utf-8')
chapter_input_re = re.compile(r'^\\input{(chapter\d+)}', re.MULTILINE)
chapter_files_in_order = []
for match in chapter_input_re.finditer(main_content):
chapter_files_in_order.append(f"{match.group(1)}.tex")
if not chapter_files_in_order:
print("未在 main.tex 中找到章节文件引用。")
return
total_chapters = 0
total_sections = 0
total_subsections = 0
# 遍历章节文件并导入
for chapter_idx, chapter_filename in enumerate(chapter_files_in_order):
chapter_file_path = book_dir / chapter_filename
if not chapter_file_path.exists():
print(f"警告: 章节文件 {chapter_file_path} 不存在,跳过。")
continue
print(f"解析LaTeX文件: {chapter_file_path.name}...")
chapter_structure = parser.parse_chapter_file(chapter_file_path)
if not chapter_structure:
print(f"警告: 文件 {chapter_file_path.name} 未解析出任何结构。")
continue
# 每个文件只包含一个 \chapter
parsed_chapter_data = chapter_structure[0]
chapter_obj = Chapter(
book_id=book.id,
chapter_number=parsed_chapter_data["chapter_number"],
title=parsed_chapter_data["title"],
file_path=chapter_file_path.name,
start_line=parsed_chapter_data["start_line"],
end_line=parsed_chapter_data["end_line"],
display_order=parsed_chapter_data["chapter_number"]
)
db.add(chapter_obj)
db.flush() # Flush to get chapter_obj.id
total_chapters += 1
# 导入节(Section
for section_data in parsed_chapter_data.get("sections", []):
section_obj = Section(
chapter_id=chapter_obj.id,
section_number=section_data["section_number"],
title=section_data["title"],
file_path=chapter_file_path.name,
start_line=section_data["start_line"],
end_line=section_data["end_line"],
display_order=section_data["section_number"]
)
db.add(section_obj)
db.flush() # Flush to get section_obj.id
total_sections += 1
# 导入小节(Subsection,作为知识点)
for subsection_data in section_data.get("subsections", []):
subsection_obj = Subsection(
section_id=section_obj.id,
subsection_number=subsection_data["subsection_number"],
title=subsection_data["title"],
file_path=chapter_file_path.name,
start_line=subsection_data["start_line"],
end_line=subsection_data["end_line"],
display_order=subsection_data["subsection_number"]
)
db.add(subsection_obj)
total_subsections += 1
db.commit()
print(f"\n[SUCCESS] 书籍结构导入成功!")
print(f" - 书籍: {book.title}")
print(f" - 章节: {total_chapters}")
print(f" - 节: {total_sections}")
print(f" - 小节(知识点): {total_subsections}")
except Exception as e:
db.rollback()
print(f"[ERROR] 导入失败: {e}")
import traceback
traceback.print_exc()
raise
finally:
db.close()
if __name__ == "__main__":
import_book_structure()
+372
View File
@@ -0,0 +1,372 @@
#!/usr/bin/env python3
"""
服务监控脚本 - 监控多服务架构的健康状态
"""
import os
import sys
import time
import json
import logging
import requests
from datetime import datetime, timedelta
from typing import Dict, List, Optional
from pathlib import Path
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler('service_monitor.log'),
logging.StreamHandler()
]
)
logger = logging.getLogger(__name__)
class ServiceMonitor:
"""服务监控器"""
def __init__(self):
self.services = {
"backend": {
"name": "FastAPI后端服务",
"url": "http://localhost:8000/health",
"timeout": 5,
"required": True
},
"frontend": {
"name": "Next.js前端服务",
"url": "http://localhost:8001",
"timeout": 5,
"required": True
},
"database": {
"name": "PostgreSQL数据库",
"type": "internal", # 内部服务,通过后端检查
"required": True
},
"vectorizer": {
"name": "向量化服务",
"type": "process", # 进程检查
"process_name": "vectorizer_service.py",
"required": False
}
}
self.check_interval = 30 # 检查间隔(秒)
self.alert_threshold = 3 # 报警阈值(连续失败次数)
self.status_history = []
self.max_history = 100
def check_service(self, service_id: str, service_config: Dict) -> Dict:
"""检查单个服务"""
service_status = {
"service_id": service_id,
"name": service_config["name"],
"timestamp": datetime.now().isoformat(),
"status": "unknown",
"response_time": None,
"error": None
}
try:
start_time = time.time()
if service_config.get("type") == "internal":
# 内部服务,通过后端检查
backend_status = self._check_backend_service(service_id)
service_status.update(backend_status)
elif service_config.get("type") == "process":
# 进程检查
process_status = self._check_process_service(service_config)
service_status.update(process_status)
else:
# HTTP服务检查
response = requests.get(
service_config["url"],
timeout=service_config.get("timeout", 5)
)
response_time = (time.time() - start_time) * 1000 # 毫秒
if response.status_code == 200:
service_status["status"] = "healthy"
service_status["response_time"] = response_time
# 解析响应内容
try:
data = response.json()
service_status["details"] = data
except:
pass
else:
service_status["status"] = "unhealthy"
service_status["error"] = f"HTTP {response.status_code}"
except requests.exceptions.Timeout:
service_status["status"] = "timeout"
service_status["error"] = "请求超时"
except requests.exceptions.ConnectionError:
service_status["status"] = "unreachable"
service_status["error"] = "连接失败"
except Exception as e:
service_status["status"] = "error"
service_status["error"] = str(e)
return service_status
def _check_backend_service(self, service_id: str) -> Dict:
"""检查后端内部服务"""
try:
# 通过后端API检查数据库状态
response = requests.get(
"http://localhost:8000/system/status",
timeout=5
)
if response.status_code == 200:
data = response.json()
if service_id == "database":
db_status = data.get("database", {})
db_type = db_status.get("type", "unknown")
return {
"status": "healthy" if db_type != "unknown" else "unhealthy",
"details": db_status
}
return {"status": "unhealthy", "error": "后端检查失败"}
except Exception as e:
return {"status": "error", "error": str(e)}
def _check_process_service(self, service_config: Dict) -> Dict:
"""检查进程服务"""
process_name = service_config.get("process_name")
if not process_name:
return {"status": "unknown", "error": "未配置进程名"}
try:
# 检查进程是否存在(Linux/Mac)
if sys.platform != "win32":
import subprocess
result = subprocess.run(
["pgrep", "-f", process_name],
capture_output=True,
text=True
)
if result.returncode == 0:
return {"status": "healthy", "details": {"processes": result.stdout.strip().split()}}
else:
return {"status": "stopped", "error": "进程未运行"}
else:
# Windows平台检查
import psutil
process_count = 0
for proc in psutil.process_iter(['pid', 'name', 'cmdline']):
try:
cmdline = proc.info.get('cmdline')
if cmdline and process_name in ' '.join(cmdline):
process_count += 1
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
if process_count > 0:
return {"status": "healthy", "details": {"process_count": process_count}}
else:
return {"status": "stopped", "error": "进程未运行"}
except Exception as e:
return {"status": "error", "error": str(e)}
def check_all_services(self) -> Dict:
"""检查所有服务"""
logger.info("开始检查所有服务...")
results = {}
overall_status = "healthy"
failed_services = []
for service_id, service_config in self.services.items():
status = self.check_service(service_id, service_config)
results[service_id] = status
if status["status"] != "healthy":
if service_config.get("required", False):
overall_status = "unhealthy"
failed_services.append(service_id)
logger.warning(f"{service_config['name']}: {status['status']} - {status.get('error', '')}")
else:
logger.info(f"{service_config['name']}: 健康")
# 保存到历史记录
check_result = {
"timestamp": datetime.now().isoformat(),
"overall_status": overall_status,
"services": results,
"failed_services": failed_services
}
self.status_history.append(check_result)
if len(self.status_history) > self.max_history:
self.status_history = self.status_history[-self.max_history:]
# 保存到文件
self._save_status_history()
logger.info(f"检查完成: 总体状态 - {overall_status}")
return check_result
def _save_status_history(self):
"""保存状态历史"""
try:
history_file = Path("service_status_history.json")
# 只保存最近24小时的数据
cutoff_time = datetime.now() - timedelta(hours=24)
recent_history = [
h for h in self.status_history
if datetime.fromisoformat(h["timestamp"].replace("Z", "+00:00")) > cutoff_time
]
with open(history_file, "w", encoding="utf-8") as f:
json.dump(recent_history, f, ensure_ascii=False, indent=2)
except Exception as e:
logger.error(f"保存状态历史失败: {e}")
def load_status_history(self) -> List[Dict]:
"""加载状态历史"""
try:
history_file = Path("service_status_history.json")
if history_file.exists():
with open(history_file, "r", encoding="utf-8") as f:
return json.load(f)
except Exception as e:
logger.error(f"加载状态历史失败: {e}")
return []
def get_service_stats(self, hours: int = 24) -> Dict:
"""获取服务统计信息"""
cutoff_time = datetime.now() - timedelta(hours=hours)
relevant_history = [
h for h in self.status_history
if datetime.fromisoformat(h["timestamp"].replace("Z", "+00:00")) > cutoff_time
]
if not relevant_history:
return {}
stats = {
"total_checks": len(relevant_history),
"healthy_checks": sum(1 for h in relevant_history if h["overall_status"] == "healthy"),
"uptime_percentage": 0,
"service_stats": {}
}
if stats["total_checks"] > 0:
stats["uptime_percentage"] = (stats["healthy_checks"] / stats["total_checks"]) * 100
# 计算每个服务的统计
for service_id in self.services.keys():
service_checks = []
for check in relevant_history:
if service_id in check["services"]:
service_checks.append(check["services"][service_id])
if service_checks:
healthy_count = sum(1 for s in service_checks if s["status"] == "healthy")
total_count = len(service_checks)
stats["service_stats"][service_id] = {
"name": self.services[service_id]["name"],
"total_checks": total_count,
"healthy_checks": healthy_count,
"availability": (healthy_count / total_count * 100) if total_count > 0 else 0,
"last_status": service_checks[-1]["status"] if service_checks else "unknown"
}
return stats
def run_monitor(self, duration_minutes: Optional[int] = None):
"""运行监控器"""
logger.info("=" * 60)
logger.info("服务监控器启动")
logger.info("=" * 60)
start_time = time.time()
try:
while True:
# 检查是否达到运行时长限制
if duration_minutes:
elapsed_minutes = (time.time() - start_time) / 60
if elapsed_minutes >= duration_minutes:
logger.info(f"达到运行时长限制 ({duration_minutes} 分钟),停止监控")
break
# 执行检查
self.check_all_services()
# 显示统计信息
stats = self.get_service_stats(hours=1)
if stats:
logger.info(f"最近1小时可用性: {stats.get('uptime_percentage', 0):.1f}%")
# 等待下一次检查
logger.info(f"等待 {self.check_interval} 秒...")
time.sleep(self.check_interval)
except KeyboardInterrupt:
logger.info("收到中断信号,停止监控")
except Exception as e:
logger.error(f"监控器运行异常: {e}")
finally:
logger.info("服务监控器停止")
def main():
"""主函数"""
import argparse
parser = argparse.ArgumentParser(description="服务监控工具")
parser.add_argument("action", choices=["check", "monitor", "stats"],
help="执行的操作")
parser.add_argument("--duration", type=int, default=60,
help="监控持续时间(分钟),仅用于monitor模式")
parser.add_argument("--hours", type=int, default=24,
help="统计小时数,用于stats模式")
args = parser.parse_args()
monitor = ServiceMonitor()
if args.action == "check":
# 单次检查
result = monitor.check_all_services()
print(json.dumps(result, ensure_ascii=False, indent=2))
elif args.action == "monitor":
# 持续监控
monitor.run_monitor(args.duration)
elif args.action == "stats":
# 显示统计信息
stats = monitor.get_service_stats(args.hours)
print(json.dumps(stats, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+170
View File
@@ -0,0 +1,170 @@
#!/usr/bin/env python3
"""
向量化服务 - 独立处理文档向量化
"""
import os
import sys
import time
import logging
from pathlib import Path
from datetime import datetime, timedelta
# 添加src目录到Python路径
sys.path.append(str(Path(__file__).parent.parent))
from src.core.database import SessionLocal, get_database_info
from src.models.document import Document
from src.services.document_service import DocumentService
from src.core.config import get_settings
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler('vectorizer_service.log'),
logging.StreamHandler()
]
)
logger = logging.getLogger(__name__)
# 减少不必要的日志
logging.getLogger("pypdf").setLevel(logging.ERROR)
logging.getLogger("PIL").setLevel(logging.ERROR)
logging.getLogger("chromadb").setLevel(logging.WARNING)
class VectorizerService:
"""向量化服务"""
def __init__(self):
self.settings = get_settings()
self.batch_size = 5 # 每次处理的文档数量
self.check_interval = 60 # 检查间隔(秒)
self.max_retries = 3 # 最大重试次数
self.retry_delay = 30 # 重试延迟(秒)
def get_unprocessed_documents(self, session):
"""获取未处理的文档"""
try:
# 查找未处理的文档
documents = session.query(Document).filter(
Document.is_processed == False,
Document.source_type == "knowledge_base"
).order_by(Document.id).limit(self.batch_size).all()
# 过滤掉文件不存在的文档
valid_documents = []
for doc in documents:
if Path(doc.file_path).exists():
valid_documents.append(doc)
else:
logger.warning(f"文档文件不存在: {doc.file_path}")
# 标记为已处理以避免重复检查
doc.is_processed = True
session.commit()
return valid_documents
except Exception as e:
logger.error(f"获取未处理文档失败: {e}")
return []
def process_document(self, session, document_service, document):
"""处理单个文档"""
logger.info(f"开始处理文档: {document.file_path} (ID: {document.id})")
for retry in range(self.max_retries):
try:
# 使用异步处理
import asyncio
# 创建新的事件循环
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
# 处理文档
success = loop.run_until_complete(
document_service.process_document(document.id)
)
loop.close()
if success:
logger.info(f"✅ 文档处理成功: {document.file_path}")
return True
else:
logger.warning(f"❌ 文档处理失败: {document.file_path}")
if retry < self.max_retries - 1:
logger.info(f"等待 {self.retry_delay} 秒后重试...")
time.sleep(self.retry_delay)
except Exception as e:
logger.error(f"文档处理异常 (尝试 {retry + 1}/{self.max_retries}): {e}")
if retry < self.max_retries - 1:
logger.info(f"等待 {self.retry_delay} 秒后重试...")
time.sleep(self.retry_delay)
logger.error(f"文档处理最终失败: {document.file_path}")
return False
def run(self):
"""运行向量化服务"""
logger.info("=" * 60)
logger.info("向量化服务启动")
logger.info("=" * 60)
# 显示数据库信息
db_info = get_database_info()
logger.info(f"数据库: {db_info.get('type', 'Unknown')}")
logger.info(f"批量大小: {self.batch_size}")
logger.info(f"检查间隔: {self.check_interval}")
session = SessionLocal()
document_service = DocumentService(session)
try:
while True:
try:
# 获取未处理的文档
documents = self.get_unprocessed_documents(session)
if documents:
logger.info(f"找到 {len(documents)} 个待处理文档")
success_count = 0
for doc in documents:
if self.process_document(session, document_service, doc):
success_count += 1
logger.info(f"批量处理完成: {success_count}/{len(documents)} 成功")
else:
logger.info("没有待处理的文档,等待中...")
# 等待下一次检查
time.sleep(self.check_interval)
except KeyboardInterrupt:
logger.info("收到中断信号,停止服务")
break
except Exception as e:
logger.error(f"服务运行异常: {e}")
logger.info(f"等待 {self.check_interval} 秒后继续...")
time.sleep(self.check_interval)
finally:
session.close()
logger.info("向量化服务停止")
def main():
"""主函数"""
# 检查环境
if not os.path.exists("vectorizer_service.log"):
with open("vectorizer_service.log", "w") as f:
f.write(f"向量化服务日志 - 启动时间: {datetime.now()}\n")
# 运行服务
service = VectorizerService()
service.run()
if __name__ == "__main__":
main()