From 52e980ba2e9cafdcc2424b48e57a7f72894ac8ff Mon Sep 17 00:00:00 2001 From: admin Date: Mon, 1 Jun 2026 16:47:35 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0SQLite=E7=9B=91?= =?UTF-8?q?=E6=8E=A7=E8=A1=A8=E8=87=AA=E5=8A=A8=E5=88=9D=E5=A7=8B=E5=8C=96?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 新增ensure_sqlite_monitor_tables函数检查并创建SQLite监控表 2. 在lifespan启动流程中加入SQLite表检查步骤 3. 重构监控表初始化脚本，支持本地执行模式 4. 优化模型差异检测，支持SQL脚本自动建表 --- apps/data_opt/mds/migrations/model_diff.py | 195 +++++++++++++- core/database.py | 79 ++++++ core/lifespan.py | 16 +- .../migrate/monitor/setup_monitor_tables.sh | 98 ++++--- .../migrate/staging/setup_staging_tables.sh | 250 ++++++++++++++---- 5 files changed, 533 insertions(+), 105 deletions(-) diff --git a/apps/data_opt/mds/migrations/model_diff.py b/apps/data_opt/mds/migrations/model_diff.py index f0f26c1..c82df32 100644 --- a/apps/data_opt/mds/migrations/model_diff.py +++ b/apps/data_opt/mds/migrations/model_diff.py @@ -208,18 +208,207 @@ class ModelDiffer: default_value=default_value if has_default else None ) - async def diff(self, model_class) -> List[AlterStmt]: + async def table_exists(self, table_name: str) -> bool: """ - 对比单个模型与数据库表的差异 + 检查表是否存在于数据库中 + + Args: + table_name: 表名 + + Returns: + 表是否存在 + """ + conn = await get_db_connection_safely(self.db_name) + + query = """ + SELECT EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_name = $1 AND table_schema = 'public' + ) + """ + + result = await conn.execute_query(query, (table_name,)) + + if result and result[1]: + return result[1][0].get('exists', False) + return False + + def _generate_create_table_sql(self, model_class) -> str: + """ + 生成 CREATE TABLE 语句 Args: model_class: Tortoise ORM 模型类 Returns: - 差异列表（需要新增的字段） + CREATE TABLE SQL 语句 + """ + table_name = model_class._meta.db_table + model_fields = self.get_model_fields(model_class) + + columns = [] + primary_key = None + + for field_name, field in model_fields.items(): + db_field_name = getattr(field, 'source_field', None) or field_name + _, sql_type_def = self._map_field_type_to_sql(field) + + is_nullable = getattr(field, 'null', True) + default_value = getattr(field, 'default', None) + has_default = default_value is not None and str(default_value) != 'PydanticUndefined' + + # 检查主键 + if getattr(field, 'pk', False) or field_name == 'id': + primary_key = db_field_name + # 处理主键字段 + if isinstance(field, IntField) and getattr(field, 'generated', False): + columns.append(f'"{db_field_name}" SERIAL PRIMARY KEY') + else: + columns.append(f'"{db_field_name}" {sql_type_def} PRIMARY KEY') + continue + + parts = [f'"{db_field_name}" {sql_type_def}'] + + if not is_nullable: + parts.append('NOT NULL') + + if has_default: + if isinstance(default_value, str): + parts.append(f"DEFAULT '{default_value}'") + elif isinstance(default_value, bool): + parts.append(f"DEFAULT {'TRUE' if default_value else 'FALSE'}") + else: + parts.append(f'DEFAULT {default_value}') + + columns.append(' '.join(parts)) + + # 添加主键约束（如果没有自动主键） + if primary_key is None and '_staging_id' in [getattr(f, 'source_field', None) or fn for fn, f in model_fields.items()]: + columns.append(f'PRIMARY KEY ("_staging_id")') + + sql_statement = f'CREATE TABLE IF NOT EXISTS "{table_name}" (\n ' + ',\n '.join(columns) + '\n)' + + return sql_statement + + def _load_staging_sql_script(self, table_name: str) -> Optional[str]: + """ + 从 SQL 文件加载指定表的创建脚本 + + Args: + table_name: 表名 + + Returns: + SQL 脚本内容，如果找不到返回 None + """ + import os + from pathlib import Path + + # 定位 SQL 文件 + sql_file = Path(__file__).resolve().parent.parent.parent.parent.parent / "scripts" / "migrate" / "staging" / "staging_tables.sql" + + if not sql_file.exists(): + logger.warning(f"SQL迁移脚本不存在: {sql_file}") + return None + + try: + with open(sql_file, 'r', encoding='utf-8') as f: + content = f.read() + + # 解析 SQL 文件，提取指定表的创建脚本 + # SQL 文件结构: 以 "-- =====================================================" 分隔不同表的定义 + sections = content.split('-- =====================================================') + + for section in sections: + # 查找包含 CREATE TABLE 且表名匹配的部分 + if f'CREATE TABLE IF NOT EXISTS {table_name}' in section or f'CREATE TABLE IF NOT EXISTS "{table_name}"' in section: + # 提取从 CREATE TABLE 到下一个分隔符或文件结束的内容 + lines = [] + in_comment_block = False + + for line in section.split('\n'): + # 跳过空行和注释行，但保留 DO $$ ... END $$ 块内的注释 + stripped = line.strip() + + # 处理 DO $$ 块 + if stripped.startswith('DO $$'): + in_comment_block = True + lines.append(line) + continue + if stripped.startswith('END $$;') and in_comment_block: + lines.append(line) + in_comment_block = False + continue + if in_comment_block: + lines.append(line) + continue + + # 跳过普通注释行 + if stripped.startswith('--'): + continue + if not stripped: + continue + + lines.append(line) + + return '\n'.join(lines) + + logger.warning(f"在SQL文件中未找到表 {table_name} 的定义") + return None + + except Exception as e: + logger.error(f"读取SQL文件失败: {str(e)}") + return None + + async def diff(self, model_class) -> List[AlterStmt]: + """ + 对比单个模型与数据库表的差异 + + 优化策略： + - 表不存在时：使用 SQL 文件创建（包含索引、注释、触发器等完整功能） + - 表已存在时：进行字段差异检测，动态生成 ALTER TABLE 语句 + + Args: + model_class: Tortoise ORM 模型类 + + Returns: + 差异列表（需要新增的字段或表） """ table_name = model_class._meta.db_table + # 检查表是否存在 + if not await self.table_exists(table_name): + # 如果表不存在，优先使用 SQL 文件 + sql_script = self._load_staging_sql_script(table_name) + + if sql_script: + logger.info(f"检测到表不存在，将使用SQL文件创建: {table_name}") + create_stmt = AlterStmt( + table_name=table_name, + field_name='__table__', + db_field_name='__table__', + sql_type='TABLE', + sql_statement=sql_script, + is_nullable=False, + default_value=None + ) + return [create_stmt] + else: + # SQL 文件不可用，降级到动态生成 + logger.warning(f"SQL文件不可用，将动态生成创建语句: {table_name}") + create_sql = self._generate_create_table_sql(model_class) + create_stmt = AlterStmt( + table_name=table_name, + field_name='__table__', + db_field_name='__table__', + sql_type='TABLE', + sql_statement=create_sql, + is_nullable=False, + default_value=None + ) + return [create_stmt] + + # 表已存在，检测字段差异 db_columns = await self.get_db_columns(table_name) model_fields = self.get_model_fields(model_class) diff --git a/core/database.py b/core/database.py index 3e6344c..f381ca4 100644 --- a/core/database.py +++ b/core/database.py @@ -642,6 +642,85 @@ def register_database(app): log_config.info("✅ 系统监控服务已集成") +REQUIRED_SQLITE_TABLES = [ + "api_requests", + "outbound_api_requests", + "system_logs", + "binlog_positions", + "processed_events", + "failed_operations", + "schema_version", +] + + +async def ensure_sqlite_monitor_tables() -> bool: + """ + 确保 SQLite 监控模块表存在 + + 在启动时检查必需的表是否存在于 SQLite 数据库中， + 如果不存在则执行迁移脚本创建表。 + + Returns: + bool: 所有表都存在或创建成功返回 True + """ + import sqlite3 + from pathlib import Path + + db_path = BASE_DIR / "storage" / f"{SQLITE_FILE}.sqlite3" + + try: + conn = sqlite3.connect(str(db_path)) + cursor = conn.cursor() + + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;") + existing_tables = {row[0] for row in cursor.fetchall()} + + missing_tables = [table for table in REQUIRED_SQLITE_TABLES if table not in existing_tables] + + if not missing_tables: + log_config.debug("✅ SQLite 监控表检查通过，所有表已存在") + conn.close() + return True + + log_config.warning(f"⚠️ SQLite 监控表缺失: {missing_tables}") + + migration_script = BASE_DIR / "scripts" / "migrate" / "monitor" / "monitor_tables.sql" + if migration_script.exists(): + log_config.info(f"📦 正在执行数据库迁移脚本: {migration_script}") + + with open(migration_script, 'r', encoding='utf-8') as f: + sql_content = f.read() + + cursor.executescript(sql_content) + conn.commit() + + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;") + created_tables = {row[0] for row in cursor.fetchall()} + + still_missing = [table for table in REQUIRED_SQLITE_TABLES if table not in created_tables] + if still_missing: + log_config.error(f"❌ 迁移后仍有表缺失: {still_missing}") + conn.close() + return False + + log_config.info("✅ SQLite 监控表迁移完成") + conn.close() + return True + else: + log_config.error(f"❌ 迁移脚本不存在: {migration_script}") + conn.close() + return False + + except Exception as e: + log_config.error(f"❌ SQLite 监控表初始化失败: {e}") + try: + if 'conn' in locals(): + conn.close() + except Exception: + pass + return False + + async def warmup_connections(): """ 预热数据库连接，增强容错处理 diff --git a/core/lifespan.py b/core/lifespan.py index 63f2f64..2860665 100644 --- a/core/lifespan.py +++ b/core/lifespan.py @@ -17,7 +17,7 @@ from apps.common.monitor import ( from apps.common.monitor.log_stream_service import start_log_stream, stop_log_stream from globalobjects import EVENT_AGGREGATOR from core.settings import TURNON_BINLOG_LISTENER, TRUNON_SCHEDULER, MAX_EVENTS_BATCH_SIZE -from core.database import check_db_connections, warmup_connections, start_pool_monitoring, db_init_manager +from core.database import check_db_connections, warmup_connections, start_pool_monitoring, db_init_manager, ensure_sqlite_monitor_tables from core.task_manager import get_task_manager @@ -72,7 +72,19 @@ async def lifespan(app): log_config.warning("⚠️ 数据库连接预热超时，继续启动其他服务") except Exception as e: log_config.error(f"❌ 数据库连接预热失败: {e}") - + + log_config.info("检查 SQLite 监控表...") + try: + tables_ready = await asyncio.wait_for(ensure_sqlite_monitor_tables(), timeout=30) + if tables_ready: + log_config.info("✅ SQLite 监控表检查完成") + else: + log_config.warning("⚠️ SQLite 监控表检查未通过，部分功能可能受影响") + except asyncio.TimeoutError: + log_config.warning("⚠️ SQLite 监控表检查超时") + except Exception as e: + log_config.error(f"❌ SQLite 监控表检查异常: {e}") + log_config.info("开始启动资源监控...") resource_monitor.start_monitoring(interval=30) log_config.info("系统资源监控已启动") diff --git a/scripts/migrate/monitor/setup_monitor_tables.sh b/scripts/migrate/monitor/setup_monitor_tables.sh index ef317de..00d6c4c 100755 --- a/scripts/migrate/monitor/setup_monitor_tables.sh +++ b/scripts/migrate/monitor/setup_monitor_tables.sh @@ -147,50 +147,64 @@ else # 2. 检查容器是否存在 echo -e "\n${YELLOW}🔍 检查MyAPS API容器状态...${NC}" if ! docker inspect "$CONTAINER_NAME" &>/dev/null; then - echo -e "${RED}❌ 错误: 容器 $CONTAINER_NAME 不存在${NC}" - echo -e "${RED} 请先启动MyAPS API容器${NC}" - exit 1 - fi - - # 检查容器是否运行 - CONTAINER_STATUS=$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME") - if [ "$CONTAINER_STATUS" != "running" ]; then - if $DRY_RUN; then - echo -e "${YELLOW}⚠️ [模拟] 容器未运行，将启动...${NC}" - else - echo -e "${YELLOW}⚠️ 容器未运行，正在启动...${NC}" - docker start "$CONTAINER_NAME" - sleep 5 - fi - fi - echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}" - - # 3. 执行建表 - echo -e "\n${YELLOW}⚙️ 执行建表脚本...${NC}" - echo -e "${BLUE} 数据库文件: ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3${NC}" - - if $DRY_RUN; then - echo -e "${YELLOW} [模拟] docker cp ${SQL_FILE} ${CONTAINER_NAME}:/tmp/monitor_tables.sql${NC}" - echo -e "${YELLOW} [模拟] docker exec ${CONTAINER_NAME} sqlite3 ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3 < /tmp/monitor_tables.sql${NC}" + echo -e "${YELLOW}⚠️ 容器 $CONTAINER_NAME 不存在，自动切换到本地模式${NC}" + LOCAL_MODE=true else - # 复制SQL文件到容器 - echo -e "${YELLOW} 复制SQL文件到容器...${NC}" - docker cp "$SQL_FILE" "$CONTAINER_NAME":/tmp/monitor_tables.sql - - # 执行建表脚本 - echo -e "${YELLOW} 执行SQL脚本...${NC}" - docker exec "$CONTAINER_NAME" sqlite3 "${STORAGE_DIR}/${SQLITE_FILE}.sqlite3" < /tmp/monitor_tables.sql - - EXIT_CODE=$? - if [ $EXIT_CODE -eq 0 ]; then - echo -e "${GREEN}✅ 建表成功${NC}" - else - echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}" - exit 1 + CONTAINER_STATUS=$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME") + if [ "$CONTAINER_STATUS" != "running" ]; then + echo -e "${YELLOW}⚠️ 容器 $CONTAINER_NAME 未运行，自动切换到本地模式${NC}" + LOCAL_MODE=true + fi + fi + + if [ "$LOCAL_MODE" = true ]; then + echo -e "\n${YELLOW}⚙️ 本地模式执行...${NC}" + + DB_PATH="${SCRIPT_DIR}/../../../storage/${SQLITE_FILE}.sqlite3" + echo -e "${BLUE} 数据库文件: ${DB_PATH}${NC}" + + if $DRY_RUN; then + echo -e "${YELLOW} [模拟] sqlite3 ${DB_PATH} < ${SQL_FILE}${NC}" + else + mkdir -p "$(dirname "$DB_PATH")" + echo -e "${YELLOW} 执行SQL脚本...${NC}" + sqlite3 "$DB_PATH" < "$SQL_FILE" + + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo -e "${GREEN}✅ 建表成功${NC}" + else + echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}" + exit 1 + fi + fi + else + echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}" + + # 3. 执行建表 + echo -e "\n${YELLOW}⚙️ 执行建表脚本...${NC}" + echo -e "${BLUE} 数据库文件: ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3${NC}" + + if $DRY_RUN; then + echo -e "${YELLOW} [模拟] docker cp ${SQL_FILE} ${CONTAINER_NAME}:/tmp/monitor_tables.sql${NC}" + echo -e "${YELLOW} [模拟] docker exec ${CONTAINER_NAME} sqlite3 ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3 < /tmp/monitor_tables.sql${NC}" + else + echo -e "${YELLOW} 复制SQL文件到容器...${NC}" + docker cp "$SQL_FILE" "$CONTAINER_NAME":/tmp/monitor_tables.sql + + echo -e "${YELLOW} 执行SQL脚本...${NC}" + docker exec "$CONTAINER_NAME" sqlite3 "${STORAGE_DIR}/${SQLITE_FILE}.sqlite3" < /tmp/monitor_tables.sql + + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo -e "${GREEN}✅ 建表成功${NC}" + else + echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}" + exit 1 + fi + + docker exec "$CONTAINER_NAME" rm -f /tmp/monitor_tables.sql fi - - # 清理容器内的临时文件 - docker exec "$CONTAINER_NAME" rm -f /tmp/monitor_tables.sql fi fi diff --git a/scripts/migrate/staging/setup_staging_tables.sh b/scripts/migrate/staging/setup_staging_tables.sh index 25b193a..5e339a9 100755 --- a/scripts/migrate/staging/setup_staging_tables.sh +++ b/scripts/migrate/staging/setup_staging_tables.sh @@ -10,7 +10,7 @@ set -e # 配置 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) SQL_FILE="${SCRIPT_DIR}/staging_tables.sql" -ENV_FILE="${SCRIPT_DIR}/../../.env" +ENV_FILE="${SCRIPT_DIR}/../../../.env" CONTAINER_NAME="myaps_postgres" # 默认数据库名 @@ -47,6 +47,7 @@ show_help() { echo " --db, -d 指定数据库名称（必填）" echo " --container, -c 指定PostgreSQL容器名称 (默认: $CONTAINER_NAME)" echo " --dry-run, -n 仅显示将要执行的操作，不实际执行" + echo " --local, -l 在本地直接执行（不通过容器）" echo "" echo "示例:" echo " # 使用默认配置" @@ -55,12 +56,16 @@ show_help() { echo " # 指定容器名称" echo " ./setup_staging_tables.sh -d myaps_db -c my_postgres" echo "" + echo " # 本地执行（开发环境）" + echo " ./setup_staging_tables.sh -d myaps_db -l" + echo "" echo " # 模拟执行" echo " ./setup_staging_tables.sh -d myaps_db -n" } # 解析参数 DRY_RUN=false +LOCAL_MODE=false while [[ $# -gt 0 ]]; do case "$1" in @@ -79,6 +84,9 @@ while [[ $# -gt 0 ]]; do --dry-run|-n) DRY_RUN=true ;; + --local|-l) + LOCAL_MODE=true + ;; *) echo "未知选项: $1" show_help @@ -95,6 +103,9 @@ YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' +# 读取环境变量（优先使用命令行参数，env文件作兜底） +read_env_config + # 检查数据库名称是否指定 if [ -z "$DB_NAME" ]; then echo -e "${RED}❌ 错误: 必须使用 -d 指定数据库名称${NC}" @@ -102,9 +113,6 @@ if [ -z "$DB_NAME" ]; then exit 1 fi -# 读取环境变量 -read_env_config - echo -e "${BLUE}==============================================${NC}" echo -e "${BLUE} PostgreSQL 缓冲表一键建表脚本${NC}" echo -e "${BLUE}==============================================${NC}" @@ -117,53 +125,165 @@ if [ ! -f "$SQL_FILE" ]; then fi echo -e "${GREEN}✅ SQL文件存在: $SQL_FILE${NC}" -# 2. 检查容器是否存在 -echo -e "\n${YELLOW}🔍 检查PostgreSQL容器状态...${NC}" -if ! docker inspect "$CONTAINER_NAME" &>/dev/null; then - echo -e "${RED}❌ 错误: 容器 $CONTAINER_NAME 不存在${NC}" - echo -e "${RED} 请先启动PostgreSQL容器${NC}" - exit 1 -fi - -# 检查容器是否运行 -CONTAINER_STATUS=$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME") -if [ "$CONTAINER_STATUS" != "running" ]; then - if $DRY_RUN; then - echo -e "${YELLOW}⚠️ [模拟] 容器未运行，将启动...${NC}" - else - echo -e "${YELLOW}⚠️ 容器未运行，正在启动...${NC}" - docker start "$CONTAINER_NAME" - sleep 5 - fi -fi -echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}" - -# 3. 执行建表 -echo -e "\n${YELLOW}⚙️ 执行建表脚本...${NC}" -echo -e "${BLUE} 数据库: ${DB_NAME}${NC}" -echo -e "${BLUE} 用户: ${DB_USER}${NC}" -echo -e "${BLUE} 端口: ${DB_PORT}${NC}" - -if $DRY_RUN; then - echo -e "${YELLOW} [模拟] docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME -f - < $SQL_FILE${NC}" -else - # 检查数据库是否存在，不存在则创建 - echo -e "${YELLOW} 检查数据库是否存在...${NC}" - if ! docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" | grep -q 1; then +exec_sql_container() { + local db="$1" + if ! docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='$db'" | grep -q 1; then echo -e "${YELLOW} 数据库不存在，创建数据库...${NC}" - docker exec "$CONTAINER_NAME" createdb -U "$DB_USER" "$DB_NAME" + docker exec "$CONTAINER_NAME" createdb -U "$DB_USER" "$db" + fi + echo -e "${YELLOW} 执行SQL脚本...${NC}" + cat "$SQL_FILE" | docker exec -i "$CONTAINER_NAME" psql -U "$DB_USER" -d "$db" + return $? +} + +exec_sql_local() { + local db="$1" + export PGPASSWORD="$DB_PASSWORD" + if ! command -v psql &>/dev/null; then + echo -e "${RED}❌ 错误: 未找到psql客户端${NC}" + echo -e "${YELLOW} 请安装postgresql-client或使用Docker模式${NC}" + return 1 fi - # 执行建表脚本 - echo -e "${YELLOW} 执行SQL脚本...${NC}" - cat "$SQL_FILE" | docker exec -i "$CONTAINER_NAME" psql -U "$DB_USER" -d "$DB_NAME" + echo -e "${YELLOW} 尝试连接PostgreSQL...${NC}" + if ! psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d postgres -tAc "SELECT 1" 2>/dev/null | grep -q 1; then + echo -e "${RED}❌ 错误: 无法连接到PostgreSQL服务器${NC}" + echo -e "${YELLOW} 服务器: ${DB_HOST}:${DB_PORT}, 用户: ${DB_USER}${NC}" + return 1 + fi - EXIT_CODE=$? - if [ $EXIT_CODE -eq 0 ]; then - echo -e "${GREEN}✅ 建表成功${NC}" + if ! psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='$db'" 2>/dev/null | grep -q 1; then + echo -e "${YELLOW} 数据库不存在，创建数据库...${NC}" + psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -c "CREATE DATABASE $db" 2>/dev/null + fi + echo -e "${YELLOW} 执行SQL脚本...${NC}" + cat "$SQL_FILE" | psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$db" + return $? +} + +verify_tables_container() { + local db="$1" + TABLES=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$db" -tAc "SELECT tablename FROM pg_tables WHERE tablename LIKE '%staging' OR tablename IN ('t_validation_error', 't_transform_rule', 't_schema_version') ORDER BY tablename;") + VERSION=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$db" -tAc "SELECT version FROM t_schema_version ORDER BY applied_at DESC LIMIT 1;") +} + +verify_tables_local() { + local db="$1" + export PGPASSWORD="$DB_PASSWORD" + TABLES=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$db" -tAc "SELECT tablename FROM pg_tables WHERE tablename LIKE '%staging' OR tablename IN ('t_validation_error', 't_transform_rule', 't_schema_version') ORDER BY tablename;" 2>/dev/null) + VERSION=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$db" -tAc "SELECT version FROM t_schema_version ORDER BY applied_at DESC LIMIT 1;" 2>/dev/null) +} + +# 检查Docker权限 +check_docker_access() { + if docker info &>/dev/null; then + return 0 else - echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}" - exit 1 + return 1 + fi +} + +# 自动检测运行中的PostgreSQL容器 +find_postgres_container() { + # 首先检查Docker权限 + if ! check_docker_access; then + return 1 + fi + + # 查找运行中的postgres容器 + local pg_container=$(docker ps --filter "ancestor=postgres" --format "{{.Names}}" 2>/dev/null | head -1) + if [ -n "$pg_container" ]; then + echo "$pg_container" + return 0 + fi + # 尝试其他方式查找 + pg_container=$(docker ps --filter "name=postgres" --format "{{.Names}}" 2>/dev/null | head -1) + if [ -n "$pg_container" ]; then + echo "$pg_container" + return 0 + fi + # 查找包含postgres的容器 + pg_container=$(docker ps --filter "name=*postgres*" --format "{{.Names}}" 2>/dev/null | head -1) + if [ -n "$pg_container" ]; then + echo "$pg_container" + return 0 + fi + return 1 +} + +# 2. 检查容器是否存在并决定执行模式 +echo -e "\n${YELLOW}🔍 检查PostgreSQL容器状态...${NC}" +AUTO_LOCAL=false + +# 先检查Docker权限 +if ! check_docker_access; then + echo -e "${YELLOW}⚠️ Docker权限不足${NC}" + echo -e "${YELLOW} 请使用 sudo 运行脚本或检查Docker配置${NC}" + echo -e "${YELLOW} 尝试本地模式...${NC}" + AUTO_LOCAL=true + LOCAL_MODE=true +else + if ! docker inspect "$CONTAINER_NAME" &>/dev/null; then + echo -e "${YELLOW}⚠️ 配置的容器 $CONTAINER_NAME 不存在${NC}" + # 尝试自动检测PostgreSQL容器 + DETECTED_CONTAINER=$(find_postgres_container) + if [ -n "$DETECTED_CONTAINER" ]; then + echo -e "${GREEN}✅ 自动检测到运行中的PostgreSQL容器: $DETECTED_CONTAINER${NC}" + CONTAINER_NAME="$DETECTED_CONTAINER" + else + echo -e "${YELLOW} 未检测到运行中的PostgreSQL容器，尝试本地模式${NC}" + AUTO_LOCAL=true + LOCAL_MODE=true + fi + elif [ "$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME")" != "running" ]; then + echo -e "${YELLOW}⚠️ 容器 $CONTAINER_NAME 未运行${NC}" + DETECTED_CONTAINER=$(find_postgres_container) + if [ -n "$DETECTED_CONTAINER" ]; then + echo -e "${GREEN}✅ 自动检测到运行中的PostgreSQL容器: $DETECTED_CONTAINER${NC}" + CONTAINER_NAME="$DETECTED_CONTAINER" + else + echo -e "${YELLOW} 未检测到运行中的PostgreSQL容器，尝试本地模式${NC}" + AUTO_LOCAL=true + LOCAL_MODE=true + fi + fi +fi + +if [ "$LOCAL_MODE" = true ]; then + echo -e "${GREEN}✅ 使用本地PostgreSQL执行${NC}" + echo -e "${BLUE} 主机: ${DB_HOST}:${DB_PORT}${NC}" + echo -e "${BLUE} 用户: ${DB_USER}${NC}" + echo -e "${BLUE} 数据库: ${DB_NAME}${NC}" + + if $DRY_RUN; then + echo -e "${YELLOW} [模拟] psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -f $SQL_FILE${NC}" + else + exec_sql_local "$DB_NAME" + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo -e "${GREEN}✅ 建表成功${NC}" + else + echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}" + exit 1 + fi + fi +else + echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}" + echo -e "${BLUE} 数据库: ${DB_NAME}${NC}" + echo -e "${BLUE} 用户: ${DB_USER}${NC}" + echo -e "${BLUE} 端口: ${DB_PORT}${NC}" + + if $DRY_RUN; then + echo -e "${YELLOW} [模拟] docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME -f - < $SQL_FILE${NC}" + else + exec_sql_container "$DB_NAME" + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo -e "${GREEN}✅ 建表成功${NC}" + else + echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}" + exit 1 + fi fi fi @@ -172,18 +292,28 @@ echo -e "\n${YELLOW}📊 验证建表结果...${NC}" if $DRY_RUN; then echo -e "${YELLOW} [模拟] 验证缓冲表是否创建成功${NC}" else - TABLES=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$DB_NAME" -tAc "SELECT tablename FROM pg_tables WHERE tablename LIKE '%staging' OR tablename IN ('t_validation_error', 't_transform_rule', 't_schema_version') ORDER BY tablename;") - echo -e "${GREEN}✅ 创建的表:${NC}" - echo "$TABLES" | while read -r table; do - if [ -n "$table" ]; then - echo -e " - $table" + if [ "$LOCAL_MODE" = true ]; then + verify_tables_local "$DB_NAME" + echo -e "${GREEN}✅ 创建的表:${NC}" + echo "$TABLES" | while read -r table; do + if [ -n "$table" ]; then + echo -e " - $table" + fi + done + if [ -n "$VERSION" ]; then + echo -e "\n${GREEN}✅ 版本记录: $VERSION${NC}" + fi + else + verify_tables_container "$DB_NAME" + echo -e "${GREEN}✅ 创建的表:${NC}" + echo "$TABLES" | while read -r table; do + if [ -n "$table" ]; then + echo -e " - $table" + fi + done + if [ -n "$VERSION" ]; then + echo -e "\n${GREEN}✅ 版本记录: $VERSION${NC}" fi - done - - # 检查版本记录 - VERSION=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$DB_NAME" -tAc "SELECT version FROM t_schema_version ORDER BY applied_at DESC LIMIT 1;") - if [ -n "$VERSION" ]; then - echo -e "\n${GREEN}✅ 版本记录: $VERSION${NC}" fi fi @@ -195,5 +325,9 @@ else fi echo -e "${BLUE}==============================================${NC}" echo -e "\n${YELLOW}💡 验证命令:${NC}" -echo -e "${YELLOW} docker exec -it $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME${NC}" +if [ "$LOCAL_MODE" = true ]; then + echo -e "${YELLOW} PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME${NC}" +else + echo -e "${YELLOW} docker exec -it $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME${NC}" +fi echo -e "${YELLOW} SELECT * FROM t_schema_version;${NC}" \ No newline at end of file