feat: 添加SQLite监控表自动初始化功能

1. 新增ensure_sqlite_monitor_tables函数检查并创建SQLite监控表
2. 在lifespan启动流程中加入SQLite表检查步骤
3. 重构监控表初始化脚本,支持本地执行模式
4. 优化模型差异检测,支持SQL脚本自动建表
This commit is contained in:
admin
2026-06-01 16:47:35 +08:00
parent f7e79269f5
commit 52e980ba2e
5 changed files with 533 additions and 105 deletions
+192 -3
View File
@@ -208,18 +208,207 @@ class ModelDiffer:
default_value=default_value if has_default else None default_value=default_value if has_default else None
) )
async def diff(self, model_class) -> List[AlterStmt]: async def table_exists(self, table_name: str) -> bool:
""" """
对比单个模型与数据库表的差异 检查表是否存在于数据库中
Args:
table_name: 表名
Returns:
表是否存在
"""
conn = await get_db_connection_safely(self.db_name)
query = """
SELECT EXISTS (
SELECT 1
FROM information_schema.tables
WHERE table_name = $1 AND table_schema = 'public'
)
"""
result = await conn.execute_query(query, (table_name,))
if result and result[1]:
return result[1][0].get('exists', False)
return False
def _generate_create_table_sql(self, model_class) -> str:
"""
生成 CREATE TABLE 语句
Args: Args:
model_class: Tortoise ORM 模型类 model_class: Tortoise ORM 模型类
Returns: Returns:
差异列表(需要新增的字段) CREATE TABLE SQL 语句
"""
table_name = model_class._meta.db_table
model_fields = self.get_model_fields(model_class)
columns = []
primary_key = None
for field_name, field in model_fields.items():
db_field_name = getattr(field, 'source_field', None) or field_name
_, sql_type_def = self._map_field_type_to_sql(field)
is_nullable = getattr(field, 'null', True)
default_value = getattr(field, 'default', None)
has_default = default_value is not None and str(default_value) != 'PydanticUndefined'
# 检查主键
if getattr(field, 'pk', False) or field_name == 'id':
primary_key = db_field_name
# 处理主键字段
if isinstance(field, IntField) and getattr(field, 'generated', False):
columns.append(f'"{db_field_name}" SERIAL PRIMARY KEY')
else:
columns.append(f'"{db_field_name}" {sql_type_def} PRIMARY KEY')
continue
parts = [f'"{db_field_name}" {sql_type_def}']
if not is_nullable:
parts.append('NOT NULL')
if has_default:
if isinstance(default_value, str):
parts.append(f"DEFAULT '{default_value}'")
elif isinstance(default_value, bool):
parts.append(f"DEFAULT {'TRUE' if default_value else 'FALSE'}")
else:
parts.append(f'DEFAULT {default_value}')
columns.append(' '.join(parts))
# 添加主键约束(如果没有自动主键)
if primary_key is None and '_staging_id' in [getattr(f, 'source_field', None) or fn for fn, f in model_fields.items()]:
columns.append(f'PRIMARY KEY ("_staging_id")')
sql_statement = f'CREATE TABLE IF NOT EXISTS "{table_name}" (\n ' + ',\n '.join(columns) + '\n)'
return sql_statement
def _load_staging_sql_script(self, table_name: str) -> Optional[str]:
"""
从 SQL 文件加载指定表的创建脚本
Args:
table_name: 表名
Returns:
SQL 脚本内容,如果找不到返回 None
"""
import os
from pathlib import Path
# 定位 SQL 文件
sql_file = Path(__file__).resolve().parent.parent.parent.parent.parent / "scripts" / "migrate" / "staging" / "staging_tables.sql"
if not sql_file.exists():
logger.warning(f"SQL迁移脚本不存在: {sql_file}")
return None
try:
with open(sql_file, 'r', encoding='utf-8') as f:
content = f.read()
# 解析 SQL 文件,提取指定表的创建脚本
# SQL 文件结构: 以 "-- =====================================================" 分隔不同表的定义
sections = content.split('-- =====================================================')
for section in sections:
# 查找包含 CREATE TABLE 且表名匹配的部分
if f'CREATE TABLE IF NOT EXISTS {table_name}' in section or f'CREATE TABLE IF NOT EXISTS "{table_name}"' in section:
# 提取从 CREATE TABLE 到下一个分隔符或文件结束的内容
lines = []
in_comment_block = False
for line in section.split('\n'):
# 跳过空行和注释行,但保留 DO $$ ... END $$ 块内的注释
stripped = line.strip()
# 处理 DO $$ 块
if stripped.startswith('DO $$'):
in_comment_block = True
lines.append(line)
continue
if stripped.startswith('END $$;') and in_comment_block:
lines.append(line)
in_comment_block = False
continue
if in_comment_block:
lines.append(line)
continue
# 跳过普通注释行
if stripped.startswith('--'):
continue
if not stripped:
continue
lines.append(line)
return '\n'.join(lines)
logger.warning(f"在SQL文件中未找到表 {table_name} 的定义")
return None
except Exception as e:
logger.error(f"读取SQL文件失败: {str(e)}")
return None
async def diff(self, model_class) -> List[AlterStmt]:
"""
对比单个模型与数据库表的差异
优化策略:
- 表不存在时:使用 SQL 文件创建(包含索引、注释、触发器等完整功能)
- 表已存在时:进行字段差异检测,动态生成 ALTER TABLE 语句
Args:
model_class: Tortoise ORM 模型类
Returns:
差异列表(需要新增的字段或表)
""" """
table_name = model_class._meta.db_table table_name = model_class._meta.db_table
# 检查表是否存在
if not await self.table_exists(table_name):
# 如果表不存在,优先使用 SQL 文件
sql_script = self._load_staging_sql_script(table_name)
if sql_script:
logger.info(f"检测到表不存在,将使用SQL文件创建: {table_name}")
create_stmt = AlterStmt(
table_name=table_name,
field_name='__table__',
db_field_name='__table__',
sql_type='TABLE',
sql_statement=sql_script,
is_nullable=False,
default_value=None
)
return [create_stmt]
else:
# SQL 文件不可用,降级到动态生成
logger.warning(f"SQL文件不可用,将动态生成创建语句: {table_name}")
create_sql = self._generate_create_table_sql(model_class)
create_stmt = AlterStmt(
table_name=table_name,
field_name='__table__',
db_field_name='__table__',
sql_type='TABLE',
sql_statement=create_sql,
is_nullable=False,
default_value=None
)
return [create_stmt]
# 表已存在,检测字段差异
db_columns = await self.get_db_columns(table_name) db_columns = await self.get_db_columns(table_name)
model_fields = self.get_model_fields(model_class) model_fields = self.get_model_fields(model_class)
+79
View File
@@ -642,6 +642,85 @@ def register_database(app):
log_config.info("✅ 系统监控服务已集成") log_config.info("✅ 系统监控服务已集成")
REQUIRED_SQLITE_TABLES = [
"api_requests",
"outbound_api_requests",
"system_logs",
"binlog_positions",
"processed_events",
"failed_operations",
"schema_version",
]
async def ensure_sqlite_monitor_tables() -> bool:
"""
确保 SQLite 监控模块表存在
在启动时检查必需的表是否存在于 SQLite 数据库中,
如果不存在则执行迁移脚本创建表。
Returns:
bool: 所有表都存在或创建成功返回 True
"""
import sqlite3
from pathlib import Path
db_path = BASE_DIR / "storage" / f"{SQLITE_FILE}.sqlite3"
try:
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;")
existing_tables = {row[0] for row in cursor.fetchall()}
missing_tables = [table for table in REQUIRED_SQLITE_TABLES if table not in existing_tables]
if not missing_tables:
log_config.debug("✅ SQLite 监控表检查通过,所有表已存在")
conn.close()
return True
log_config.warning(f"⚠️ SQLite 监控表缺失: {missing_tables}")
migration_script = BASE_DIR / "scripts" / "migrate" / "monitor" / "monitor_tables.sql"
if migration_script.exists():
log_config.info(f"📦 正在执行数据库迁移脚本: {migration_script}")
with open(migration_script, 'r', encoding='utf-8') as f:
sql_content = f.read()
cursor.executescript(sql_content)
conn.commit()
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;")
created_tables = {row[0] for row in cursor.fetchall()}
still_missing = [table for table in REQUIRED_SQLITE_TABLES if table not in created_tables]
if still_missing:
log_config.error(f"❌ 迁移后仍有表缺失: {still_missing}")
conn.close()
return False
log_config.info("✅ SQLite 监控表迁移完成")
conn.close()
return True
else:
log_config.error(f"❌ 迁移脚本不存在: {migration_script}")
conn.close()
return False
except Exception as e:
log_config.error(f"❌ SQLite 监控表初始化失败: {e}")
try:
if 'conn' in locals():
conn.close()
except Exception:
pass
return False
async def warmup_connections(): async def warmup_connections():
""" """
预热数据库连接,增强容错处理 预热数据库连接,增强容错处理
+14 -2
View File
@@ -17,7 +17,7 @@ from apps.common.monitor import (
from apps.common.monitor.log_stream_service import start_log_stream, stop_log_stream from apps.common.monitor.log_stream_service import start_log_stream, stop_log_stream
from globalobjects import EVENT_AGGREGATOR from globalobjects import EVENT_AGGREGATOR
from core.settings import TURNON_BINLOG_LISTENER, TRUNON_SCHEDULER, MAX_EVENTS_BATCH_SIZE from core.settings import TURNON_BINLOG_LISTENER, TRUNON_SCHEDULER, MAX_EVENTS_BATCH_SIZE
from core.database import check_db_connections, warmup_connections, start_pool_monitoring, db_init_manager from core.database import check_db_connections, warmup_connections, start_pool_monitoring, db_init_manager, ensure_sqlite_monitor_tables
from core.task_manager import get_task_manager from core.task_manager import get_task_manager
@@ -72,7 +72,19 @@ async def lifespan(app):
log_config.warning("⚠️ 数据库连接预热超时,继续启动其他服务") log_config.warning("⚠️ 数据库连接预热超时,继续启动其他服务")
except Exception as e: except Exception as e:
log_config.error(f"❌ 数据库连接预热失败: {e}") log_config.error(f"❌ 数据库连接预热失败: {e}")
log_config.info("检查 SQLite 监控表...")
try:
tables_ready = await asyncio.wait_for(ensure_sqlite_monitor_tables(), timeout=30)
if tables_ready:
log_config.info("✅ SQLite 监控表检查完成")
else:
log_config.warning("⚠️ SQLite 监控表检查未通过,部分功能可能受影响")
except asyncio.TimeoutError:
log_config.warning("⚠️ SQLite 监控表检查超时")
except Exception as e:
log_config.error(f"❌ SQLite 监控表检查异常: {e}")
log_config.info("开始启动资源监控...") log_config.info("开始启动资源监控...")
resource_monitor.start_monitoring(interval=30) resource_monitor.start_monitoring(interval=30)
log_config.info("系统资源监控已启动") log_config.info("系统资源监控已启动")
+56 -42
View File
@@ -147,50 +147,64 @@ else
# 2. 检查容器是否存在 # 2. 检查容器是否存在
echo -e "\n${YELLOW}🔍 检查MyAPS API容器状态...${NC}" echo -e "\n${YELLOW}🔍 检查MyAPS API容器状态...${NC}"
if ! docker inspect "$CONTAINER_NAME" &>/dev/null; then if ! docker inspect "$CONTAINER_NAME" &>/dev/null; then
echo -e "${RED}❌ 错误: 容器 $CONTAINER_NAME 不存在${NC}" echo -e "${YELLOW}⚠️ 容器 $CONTAINER_NAME 不存在,自动切换到本地模式${NC}"
echo -e "${RED} 请先启动MyAPS API容器${NC}" LOCAL_MODE=true
exit 1
fi
# 检查容器是否运行
CONTAINER_STATUS=$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME")
if [ "$CONTAINER_STATUS" != "running" ]; then
if $DRY_RUN; then
echo -e "${YELLOW}⚠️ [模拟] 容器未运行,将启动...${NC}"
else
echo -e "${YELLOW}⚠️ 容器未运行,正在启动...${NC}"
docker start "$CONTAINER_NAME"
sleep 5
fi
fi
echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}"
# 3. 执行建表
echo -e "\n${YELLOW}⚙️ 执行建表脚本...${NC}"
echo -e "${BLUE} 数据库文件: ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3${NC}"
if $DRY_RUN; then
echo -e "${YELLOW} [模拟] docker cp ${SQL_FILE} ${CONTAINER_NAME}:/tmp/monitor_tables.sql${NC}"
echo -e "${YELLOW} [模拟] docker exec ${CONTAINER_NAME} sqlite3 ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3 < /tmp/monitor_tables.sql${NC}"
else else
# 复制SQL文件到容器 CONTAINER_STATUS=$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME")
echo -e "${YELLOW} 复制SQL文件到容器...${NC}" if [ "$CONTAINER_STATUS" != "running" ]; then
docker cp "$SQL_FILE" "$CONTAINER_NAME":/tmp/monitor_tables.sql echo -e "${YELLOW}⚠️ 容器 $CONTAINER_NAME 未运行,自动切换到本地模式${NC}"
LOCAL_MODE=true
# 执行建表脚本 fi
echo -e "${YELLOW} 执行SQL脚本...${NC}" fi
docker exec "$CONTAINER_NAME" sqlite3 "${STORAGE_DIR}/${SQLITE_FILE}.sqlite3" < /tmp/monitor_tables.sql
if [ "$LOCAL_MODE" = true ]; then
EXIT_CODE=$? echo -e "\n${YELLOW}⚙️ 本地模式执行...${NC}"
if [ $EXIT_CODE -eq 0 ]; then
echo -e "${GREEN}✅ 建表成功${NC}" DB_PATH="${SCRIPT_DIR}/../../../storage/${SQLITE_FILE}.sqlite3"
else echo -e "${BLUE} 数据库文件: ${DB_PATH}${NC}"
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}"
exit 1 if $DRY_RUN; then
echo -e "${YELLOW} [模拟] sqlite3 ${DB_PATH} < ${SQL_FILE}${NC}"
else
mkdir -p "$(dirname "$DB_PATH")"
echo -e "${YELLOW} 执行SQL脚本...${NC}"
sqlite3 "$DB_PATH" < "$SQL_FILE"
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
echo -e "${GREEN}✅ 建表成功${NC}"
else
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}"
exit 1
fi
fi
else
echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}"
# 3. 执行建表
echo -e "\n${YELLOW}⚙️ 执行建表脚本...${NC}"
echo -e "${BLUE} 数据库文件: ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3${NC}"
if $DRY_RUN; then
echo -e "${YELLOW} [模拟] docker cp ${SQL_FILE} ${CONTAINER_NAME}:/tmp/monitor_tables.sql${NC}"
echo -e "${YELLOW} [模拟] docker exec ${CONTAINER_NAME} sqlite3 ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3 < /tmp/monitor_tables.sql${NC}"
else
echo -e "${YELLOW} 复制SQL文件到容器...${NC}"
docker cp "$SQL_FILE" "$CONTAINER_NAME":/tmp/monitor_tables.sql
echo -e "${YELLOW} 执行SQL脚本...${NC}"
docker exec "$CONTAINER_NAME" sqlite3 "${STORAGE_DIR}/${SQLITE_FILE}.sqlite3" < /tmp/monitor_tables.sql
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
echo -e "${GREEN}✅ 建表成功${NC}"
else
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}"
exit 1
fi
docker exec "$CONTAINER_NAME" rm -f /tmp/monitor_tables.sql
fi fi
# 清理容器内的临时文件
docker exec "$CONTAINER_NAME" rm -f /tmp/monitor_tables.sql
fi fi
fi fi
+192 -58
View File
@@ -10,7 +10,7 @@ set -e
# 配置 # 配置
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
SQL_FILE="${SCRIPT_DIR}/staging_tables.sql" SQL_FILE="${SCRIPT_DIR}/staging_tables.sql"
ENV_FILE="${SCRIPT_DIR}/../../.env" ENV_FILE="${SCRIPT_DIR}/../../../.env"
CONTAINER_NAME="myaps_postgres" CONTAINER_NAME="myaps_postgres"
# 默认数据库名 # 默认数据库名
@@ -47,6 +47,7 @@ show_help() {
echo " --db, -d 指定数据库名称(必填)" echo " --db, -d 指定数据库名称(必填)"
echo " --container, -c 指定PostgreSQL容器名称 (默认: $CONTAINER_NAME)" echo " --container, -c 指定PostgreSQL容器名称 (默认: $CONTAINER_NAME)"
echo " --dry-run, -n 仅显示将要执行的操作,不实际执行" echo " --dry-run, -n 仅显示将要执行的操作,不实际执行"
echo " --local, -l 在本地直接执行(不通过容器)"
echo "" echo ""
echo "示例:" echo "示例:"
echo " # 使用默认配置" echo " # 使用默认配置"
@@ -55,12 +56,16 @@ show_help() {
echo " # 指定容器名称" echo " # 指定容器名称"
echo " ./setup_staging_tables.sh -d myaps_db -c my_postgres" echo " ./setup_staging_tables.sh -d myaps_db -c my_postgres"
echo "" echo ""
echo " # 本地执行(开发环境)"
echo " ./setup_staging_tables.sh -d myaps_db -l"
echo ""
echo " # 模拟执行" echo " # 模拟执行"
echo " ./setup_staging_tables.sh -d myaps_db -n" echo " ./setup_staging_tables.sh -d myaps_db -n"
} }
# 解析参数 # 解析参数
DRY_RUN=false DRY_RUN=false
LOCAL_MODE=false
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
case "$1" in case "$1" in
@@ -79,6 +84,9 @@ while [[ $# -gt 0 ]]; do
--dry-run|-n) --dry-run|-n)
DRY_RUN=true DRY_RUN=true
;; ;;
--local|-l)
LOCAL_MODE=true
;;
*) *)
echo "未知选项: $1" echo "未知选项: $1"
show_help show_help
@@ -95,6 +103,9 @@ YELLOW='\033[1;33m'
BLUE='\033[0;34m' BLUE='\033[0;34m'
NC='\033[0m' NC='\033[0m'
# 读取环境变量(优先使用命令行参数,env文件作兜底)
read_env_config
# 检查数据库名称是否指定 # 检查数据库名称是否指定
if [ -z "$DB_NAME" ]; then if [ -z "$DB_NAME" ]; then
echo -e "${RED}❌ 错误: 必须使用 -d 指定数据库名称${NC}" echo -e "${RED}❌ 错误: 必须使用 -d 指定数据库名称${NC}"
@@ -102,9 +113,6 @@ if [ -z "$DB_NAME" ]; then
exit 1 exit 1
fi fi
# 读取环境变量
read_env_config
echo -e "${BLUE}==============================================${NC}" echo -e "${BLUE}==============================================${NC}"
echo -e "${BLUE} PostgreSQL 缓冲表一键建表脚本${NC}" echo -e "${BLUE} PostgreSQL 缓冲表一键建表脚本${NC}"
echo -e "${BLUE}==============================================${NC}" echo -e "${BLUE}==============================================${NC}"
@@ -117,53 +125,165 @@ if [ ! -f "$SQL_FILE" ]; then
fi fi
echo -e "${GREEN}✅ SQL文件存在: $SQL_FILE${NC}" echo -e "${GREEN}✅ SQL文件存在: $SQL_FILE${NC}"
# 2. 检查容器是否存在 exec_sql_container() {
echo -e "\n${YELLOW}🔍 检查PostgreSQL容器状态...${NC}" local db="$1"
if ! docker inspect "$CONTAINER_NAME" &>/dev/null; then if ! docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='$db'" | grep -q 1; then
echo -e "${RED}❌ 错误: 容器 $CONTAINER_NAME 不存在${NC}"
echo -e "${RED} 请先启动PostgreSQL容器${NC}"
exit 1
fi
# 检查容器是否运行
CONTAINER_STATUS=$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME")
if [ "$CONTAINER_STATUS" != "running" ]; then
if $DRY_RUN; then
echo -e "${YELLOW}⚠️ [模拟] 容器未运行,将启动...${NC}"
else
echo -e "${YELLOW}⚠️ 容器未运行,正在启动...${NC}"
docker start "$CONTAINER_NAME"
sleep 5
fi
fi
echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}"
# 3. 执行建表
echo -e "\n${YELLOW}⚙️ 执行建表脚本...${NC}"
echo -e "${BLUE} 数据库: ${DB_NAME}${NC}"
echo -e "${BLUE} 用户: ${DB_USER}${NC}"
echo -e "${BLUE} 端口: ${DB_PORT}${NC}"
if $DRY_RUN; then
echo -e "${YELLOW} [模拟] docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME -f - < $SQL_FILE${NC}"
else
# 检查数据库是否存在,不存在则创建
echo -e "${YELLOW} 检查数据库是否存在...${NC}"
if ! docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" | grep -q 1; then
echo -e "${YELLOW} 数据库不存在,创建数据库...${NC}" echo -e "${YELLOW} 数据库不存在,创建数据库...${NC}"
docker exec "$CONTAINER_NAME" createdb -U "$DB_USER" "$DB_NAME" docker exec "$CONTAINER_NAME" createdb -U "$DB_USER" "$db"
fi
echo -e "${YELLOW} 执行SQL脚本...${NC}"
cat "$SQL_FILE" | docker exec -i "$CONTAINER_NAME" psql -U "$DB_USER" -d "$db"
return $?
}
exec_sql_local() {
local db="$1"
export PGPASSWORD="$DB_PASSWORD"
if ! command -v psql &>/dev/null; then
echo -e "${RED}❌ 错误: 未找到psql客户端${NC}"
echo -e "${YELLOW} 请安装postgresql-client或使用Docker模式${NC}"
return 1
fi fi
# 执行建表脚本 echo -e "${YELLOW} 尝试连接PostgreSQL...${NC}"
echo -e "${YELLOW} 执行SQL脚本...${NC}" if ! psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d postgres -tAc "SELECT 1" 2>/dev/null | grep -q 1; then
cat "$SQL_FILE" | docker exec -i "$CONTAINER_NAME" psql -U "$DB_USER" -d "$DB_NAME" echo -e "${RED}❌ 错误: 无法连接到PostgreSQL服务器${NC}"
echo -e "${YELLOW} 服务器: ${DB_HOST}:${DB_PORT}, 用户: ${DB_USER}${NC}"
return 1
fi
EXIT_CODE=$? if ! psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='$db'" 2>/dev/null | grep -q 1; then
if [ $EXIT_CODE -eq 0 ]; then echo -e "${YELLOW} 数据库不存在,创建数据库...${NC}"
echo -e "${GREEN}✅ 建表成功${NC}" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -c "CREATE DATABASE $db" 2>/dev/null
fi
echo -e "${YELLOW} 执行SQL脚本...${NC}"
cat "$SQL_FILE" | psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$db"
return $?
}
verify_tables_container() {
local db="$1"
TABLES=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$db" -tAc "SELECT tablename FROM pg_tables WHERE tablename LIKE '%staging' OR tablename IN ('t_validation_error', 't_transform_rule', 't_schema_version') ORDER BY tablename;")
VERSION=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$db" -tAc "SELECT version FROM t_schema_version ORDER BY applied_at DESC LIMIT 1;")
}
verify_tables_local() {
local db="$1"
export PGPASSWORD="$DB_PASSWORD"
TABLES=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$db" -tAc "SELECT tablename FROM pg_tables WHERE tablename LIKE '%staging' OR tablename IN ('t_validation_error', 't_transform_rule', 't_schema_version') ORDER BY tablename;" 2>/dev/null)
VERSION=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$db" -tAc "SELECT version FROM t_schema_version ORDER BY applied_at DESC LIMIT 1;" 2>/dev/null)
}
# 检查Docker权限
check_docker_access() {
if docker info &>/dev/null; then
return 0
else else
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}" return 1
exit 1 fi
}
# 自动检测运行中的PostgreSQL容器
find_postgres_container() {
# 首先检查Docker权限
if ! check_docker_access; then
return 1
fi
# 查找运行中的postgres容器
local pg_container=$(docker ps --filter "ancestor=postgres" --format "{{.Names}}" 2>/dev/null | head -1)
if [ -n "$pg_container" ]; then
echo "$pg_container"
return 0
fi
# 尝试其他方式查找
pg_container=$(docker ps --filter "name=postgres" --format "{{.Names}}" 2>/dev/null | head -1)
if [ -n "$pg_container" ]; then
echo "$pg_container"
return 0
fi
# 查找包含postgres的容器
pg_container=$(docker ps --filter "name=*postgres*" --format "{{.Names}}" 2>/dev/null | head -1)
if [ -n "$pg_container" ]; then
echo "$pg_container"
return 0
fi
return 1
}
# 2. 检查容器是否存在并决定执行模式
echo -e "\n${YELLOW}🔍 检查PostgreSQL容器状态...${NC}"
AUTO_LOCAL=false
# 先检查Docker权限
if ! check_docker_access; then
echo -e "${YELLOW}⚠️ Docker权限不足${NC}"
echo -e "${YELLOW} 请使用 sudo 运行脚本或检查Docker配置${NC}"
echo -e "${YELLOW} 尝试本地模式...${NC}"
AUTO_LOCAL=true
LOCAL_MODE=true
else
if ! docker inspect "$CONTAINER_NAME" &>/dev/null; then
echo -e "${YELLOW}⚠️ 配置的容器 $CONTAINER_NAME 不存在${NC}"
# 尝试自动检测PostgreSQL容器
DETECTED_CONTAINER=$(find_postgres_container)
if [ -n "$DETECTED_CONTAINER" ]; then
echo -e "${GREEN}✅ 自动检测到运行中的PostgreSQL容器: $DETECTED_CONTAINER${NC}"
CONTAINER_NAME="$DETECTED_CONTAINER"
else
echo -e "${YELLOW} 未检测到运行中的PostgreSQL容器,尝试本地模式${NC}"
AUTO_LOCAL=true
LOCAL_MODE=true
fi
elif [ "$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME")" != "running" ]; then
echo -e "${YELLOW}⚠️ 容器 $CONTAINER_NAME 未运行${NC}"
DETECTED_CONTAINER=$(find_postgres_container)
if [ -n "$DETECTED_CONTAINER" ]; then
echo -e "${GREEN}✅ 自动检测到运行中的PostgreSQL容器: $DETECTED_CONTAINER${NC}"
CONTAINER_NAME="$DETECTED_CONTAINER"
else
echo -e "${YELLOW} 未检测到运行中的PostgreSQL容器,尝试本地模式${NC}"
AUTO_LOCAL=true
LOCAL_MODE=true
fi
fi
fi
if [ "$LOCAL_MODE" = true ]; then
echo -e "${GREEN}✅ 使用本地PostgreSQL执行${NC}"
echo -e "${BLUE} 主机: ${DB_HOST}:${DB_PORT}${NC}"
echo -e "${BLUE} 用户: ${DB_USER}${NC}"
echo -e "${BLUE} 数据库: ${DB_NAME}${NC}"
if $DRY_RUN; then
echo -e "${YELLOW} [模拟] psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -f $SQL_FILE${NC}"
else
exec_sql_local "$DB_NAME"
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
echo -e "${GREEN}✅ 建表成功${NC}"
else
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}"
exit 1
fi
fi
else
echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}"
echo -e "${BLUE} 数据库: ${DB_NAME}${NC}"
echo -e "${BLUE} 用户: ${DB_USER}${NC}"
echo -e "${BLUE} 端口: ${DB_PORT}${NC}"
if $DRY_RUN; then
echo -e "${YELLOW} [模拟] docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME -f - < $SQL_FILE${NC}"
else
exec_sql_container "$DB_NAME"
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
echo -e "${GREEN}✅ 建表成功${NC}"
else
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}"
exit 1
fi
fi fi
fi fi
@@ -172,18 +292,28 @@ echo -e "\n${YELLOW}📊 验证建表结果...${NC}"
if $DRY_RUN; then if $DRY_RUN; then
echo -e "${YELLOW} [模拟] 验证缓冲表是否创建成功${NC}" echo -e "${YELLOW} [模拟] 验证缓冲表是否创建成功${NC}"
else else
TABLES=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$DB_NAME" -tAc "SELECT tablename FROM pg_tables WHERE tablename LIKE '%staging' OR tablename IN ('t_validation_error', 't_transform_rule', 't_schema_version') ORDER BY tablename;") if [ "$LOCAL_MODE" = true ]; then
echo -e "${GREEN}✅ 创建的表:${NC}" verify_tables_local "$DB_NAME"
echo "$TABLES" | while read -r table; do echo -e "${GREEN}✅ 创建的表:${NC}"
if [ -n "$table" ]; then echo "$TABLES" | while read -r table; do
echo -e " - $table" if [ -n "$table" ]; then
echo -e " - $table"
fi
done
if [ -n "$VERSION" ]; then
echo -e "\n${GREEN}✅ 版本记录: $VERSION${NC}"
fi
else
verify_tables_container "$DB_NAME"
echo -e "${GREEN}✅ 创建的表:${NC}"
echo "$TABLES" | while read -r table; do
if [ -n "$table" ]; then
echo -e " - $table"
fi
done
if [ -n "$VERSION" ]; then
echo -e "\n${GREEN}✅ 版本记录: $VERSION${NC}"
fi fi
done
# 检查版本记录
VERSION=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$DB_NAME" -tAc "SELECT version FROM t_schema_version ORDER BY applied_at DESC LIMIT 1;")
if [ -n "$VERSION" ]; then
echo -e "\n${GREEN}✅ 版本记录: $VERSION${NC}"
fi fi
fi fi
@@ -195,5 +325,9 @@ else
fi fi
echo -e "${BLUE}==============================================${NC}" echo -e "${BLUE}==============================================${NC}"
echo -e "\n${YELLOW}💡 验证命令:${NC}" echo -e "\n${YELLOW}💡 验证命令:${NC}"
echo -e "${YELLOW} docker exec -it $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME${NC}" if [ "$LOCAL_MODE" = true ]; then
echo -e "${YELLOW} PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME${NC}"
else
echo -e "${YELLOW} docker exec -it $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME${NC}"
fi
echo -e "${YELLOW} SELECT * FROM t_schema_version;${NC}" echo -e "${YELLOW} SELECT * FROM t_schema_version;${NC}"