mirror of
https://github.com/rnvm9wjdtj-bot/myaps_api.git
synced 2026-06-02 05:54:40 +00:00
feat: 添加SQLite监控表自动初始化功能
1. 新增ensure_sqlite_monitor_tables函数检查并创建SQLite监控表 2. 在lifespan启动流程中加入SQLite表检查步骤 3. 重构监控表初始化脚本，支持本地执行模式 4. 优化模型差异检测，支持SQL脚本自动建表
This commit is contained in:
admin
@@ -208,18 +208,207 @@ class ModelDiffer:
|
||||
default_value=default_value if has_default else None
|
||||
)
|
||||
|
||||
async def diff(self, model_class) -> List[AlterStmt]:
|
||||
async def table_exists(self, table_name: str) -> bool:
|
||||
"""
|
||||
对比单个模型与数据库表的差异
|
||||
检查表是否存在于数据库中
|
||||
|
||||
Args:
|
||||
table_name: 表名
|
||||
|
||||
Returns:
|
||||
表是否存在
|
||||
"""
|
||||
conn = await get_db_connection_safely(self.db_name)
|
||||
|
||||
query = """
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.tables
|
||||
WHERE table_name = $1 AND table_schema = 'public'
|
||||
)
|
||||
"""
|
||||
|
||||
result = await conn.execute_query(query, (table_name,))
|
||||
|
||||
if result and result[1]:
|
||||
return result[1][0].get('exists', False)
|
||||
return False
|
||||
|
||||
def _generate_create_table_sql(self, model_class) -> str:
|
||||
"""
|
||||
生成 CREATE TABLE 语句
|
||||
|
||||
Args:
|
||||
model_class: Tortoise ORM 模型类
|
||||
|
||||
Returns:
|
||||
差异列表（需要新增的字段）
|
||||
CREATE TABLE SQL 语句
|
||||
"""
|
||||
table_name = model_class._meta.db_table
|
||||
model_fields = self.get_model_fields(model_class)
|
||||
|
||||
columns = []
|
||||
primary_key = None
|
||||
|
||||
for field_name, field in model_fields.items():
|
||||
db_field_name = getattr(field, 'source_field', None) or field_name
|
||||
_, sql_type_def = self._map_field_type_to_sql(field)
|
||||
|
||||
is_nullable = getattr(field, 'null', True)
|
||||
default_value = getattr(field, 'default', None)
|
||||
has_default = default_value is not None and str(default_value) != 'PydanticUndefined'
|
||||
|
||||
# 检查主键
|
||||
if getattr(field, 'pk', False) or field_name == 'id':
|
||||
primary_key = db_field_name
|
||||
# 处理主键字段
|
||||
if isinstance(field, IntField) and getattr(field, 'generated', False):
|
||||
columns.append(f'"{db_field_name}" SERIAL PRIMARY KEY')
|
||||
else:
|
||||
columns.append(f'"{db_field_name}" {sql_type_def} PRIMARY KEY')
|
||||
continue
|
||||
|
||||
parts = [f'"{db_field_name}" {sql_type_def}']
|
||||
|
||||
if not is_nullable:
|
||||
parts.append('NOT NULL')
|
||||
|
||||
if has_default:
|
||||
if isinstance(default_value, str):
|
||||
parts.append(f"DEFAULT '{default_value}'")
|
||||
elif isinstance(default_value, bool):
|
||||
parts.append(f"DEFAULT {'TRUE' if default_value else 'FALSE'}")
|
||||
else:
|
||||
parts.append(f'DEFAULT {default_value}')
|
||||
|
||||
columns.append(' '.join(parts))
|
||||
|
||||
# 添加主键约束（如果没有自动主键）
|
||||
if primary_key is None and '_staging_id' in [getattr(f, 'source_field', None) or fn for fn, f in model_fields.items()]:
|
||||
columns.append(f'PRIMARY KEY ("_staging_id")')
|
||||
|
||||
sql_statement = f'CREATE TABLE IF NOT EXISTS "{table_name}" (\n ' + ',\n '.join(columns) + '\n)'
|
||||
|
||||
return sql_statement
|
||||
|
||||
def _load_staging_sql_script(self, table_name: str) -> Optional[str]:
|
||||
"""
|
||||
从 SQL 文件加载指定表的创建脚本
|
||||
|
||||
Args:
|
||||
table_name: 表名
|
||||
|
||||
Returns:
|
||||
SQL 脚本内容，如果找不到返回 None
|
||||
"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# 定位 SQL 文件
|
||||
sql_file = Path(__file__).resolve().parent.parent.parent.parent.parent / "scripts" / "migrate" / "staging" / "staging_tables.sql"
|
||||
|
||||
if not sql_file.exists():
|
||||
logger.warning(f"SQL迁移脚本不存在: {sql_file}")
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(sql_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# 解析 SQL 文件，提取指定表的创建脚本
|
||||
# SQL 文件结构: 以 "-- =====================================================" 分隔不同表的定义
|
||||
sections = content.split('-- =====================================================')
|
||||
|
||||
for section in sections:
|
||||
# 查找包含 CREATE TABLE 且表名匹配的部分
|
||||
if f'CREATE TABLE IF NOT EXISTS {table_name}' in section or f'CREATE TABLE IF NOT EXISTS "{table_name}"' in section:
|
||||
# 提取从 CREATE TABLE 到下一个分隔符或文件结束的内容
|
||||
lines = []
|
||||
in_comment_block = False
|
||||
|
||||
for line in section.split('\n'):
|
||||
# 跳过空行和注释行，但保留 DO $$ ... END $$ 块内的注释
|
||||
stripped = line.strip()
|
||||
|
||||
# 处理 DO $$ 块
|
||||
if stripped.startswith('DO $$'):
|
||||
in_comment_block = True
|
||||
lines.append(line)
|
||||
continue
|
||||
if stripped.startswith('END $$;') and in_comment_block:
|
||||
lines.append(line)
|
||||
in_comment_block = False
|
||||
continue
|
||||
if in_comment_block:
|
||||
lines.append(line)
|
||||
continue
|
||||
|
||||
# 跳过普通注释行
|
||||
if stripped.startswith('--'):
|
||||
continue
|
||||
if not stripped:
|
||||
continue
|
||||
|
||||
lines.append(line)
|
||||
|
||||
return '\n'.join(lines)
|
||||
|
||||
logger.warning(f"在SQL文件中未找到表 {table_name} 的定义")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"读取SQL文件失败: {str(e)}")
|
||||
return None
|
||||
|
||||
async def diff(self, model_class) -> List[AlterStmt]:
|
||||
"""
|
||||
对比单个模型与数据库表的差异
|
||||
|
||||
优化策略：
|
||||
- 表不存在时：使用 SQL 文件创建（包含索引、注释、触发器等完整功能）
|
||||
- 表已存在时：进行字段差异检测，动态生成 ALTER TABLE 语句
|
||||
|
||||
Args:
|
||||
model_class: Tortoise ORM 模型类
|
||||
|
||||
Returns:
|
||||
差异列表（需要新增的字段或表）
|
||||
"""
|
||||
table_name = model_class._meta.db_table
|
||||
|
||||
# 检查表是否存在
|
||||
if not await self.table_exists(table_name):
|
||||
# 如果表不存在，优先使用 SQL 文件
|
||||
sql_script = self._load_staging_sql_script(table_name)
|
||||
|
||||
if sql_script:
|
||||
logger.info(f"检测到表不存在，将使用SQL文件创建: {table_name}")
|
||||
create_stmt = AlterStmt(
|
||||
table_name=table_name,
|
||||
field_name='__table__',
|
||||
db_field_name='__table__',
|
||||
sql_type='TABLE',
|
||||
sql_statement=sql_script,
|
||||
is_nullable=False,
|
||||
default_value=None
|
||||
)
|
||||
return [create_stmt]
|
||||
else:
|
||||
# SQL 文件不可用，降级到动态生成
|
||||
logger.warning(f"SQL文件不可用，将动态生成创建语句: {table_name}")
|
||||
create_sql = self._generate_create_table_sql(model_class)
|
||||
create_stmt = AlterStmt(
|
||||
table_name=table_name,
|
||||
field_name='__table__',
|
||||
db_field_name='__table__',
|
||||
sql_type='TABLE',
|
||||
sql_statement=create_sql,
|
||||
is_nullable=False,
|
||||
default_value=None
|
||||
)
|
||||
return [create_stmt]
|
||||
|
||||
# 表已存在，检测字段差异
|
||||
db_columns = await self.get_db_columns(table_name)
|
||||
model_fields = self.get_model_fields(model_class)
|
||||
|
||||
|
||||
@@ -642,6 +642,85 @@ def register_database(app):
|
||||
log_config.info("✅ 系统监控服务已集成")
|
||||
|
||||
|
||||
REQUIRED_SQLITE_TABLES = [
|
||||
"api_requests",
|
||||
"outbound_api_requests",
|
||||
"system_logs",
|
||||
"binlog_positions",
|
||||
"processed_events",
|
||||
"failed_operations",
|
||||
"schema_version",
|
||||
]
|
||||
|
||||
|
||||
async def ensure_sqlite_monitor_tables() -> bool:
|
||||
"""
|
||||
确保 SQLite 监控模块表存在
|
||||
|
||||
在启动时检查必需的表是否存在于 SQLite 数据库中，
|
||||
如果不存在则执行迁移脚本创建表。
|
||||
|
||||
Returns:
|
||||
bool: 所有表都存在或创建成功返回 True
|
||||
"""
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
db_path = BASE_DIR / "storage" / f"{SQLITE_FILE}.sqlite3"
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;")
|
||||
existing_tables = {row[0] for row in cursor.fetchall()}
|
||||
|
||||
missing_tables = [table for table in REQUIRED_SQLITE_TABLES if table not in existing_tables]
|
||||
|
||||
if not missing_tables:
|
||||
log_config.debug("✅ SQLite 监控表检查通过，所有表已存在")
|
||||
conn.close()
|
||||
return True
|
||||
|
||||
log_config.warning(f"⚠️ SQLite 监控表缺失: {missing_tables}")
|
||||
|
||||
migration_script = BASE_DIR / "scripts" / "migrate" / "monitor" / "monitor_tables.sql"
|
||||
if migration_script.exists():
|
||||
log_config.info(f"📦 正在执行数据库迁移脚本: {migration_script}")
|
||||
|
||||
with open(migration_script, 'r', encoding='utf-8') as f:
|
||||
sql_content = f.read()
|
||||
|
||||
cursor.executescript(sql_content)
|
||||
conn.commit()
|
||||
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;")
|
||||
created_tables = {row[0] for row in cursor.fetchall()}
|
||||
|
||||
still_missing = [table for table in REQUIRED_SQLITE_TABLES if table not in created_tables]
|
||||
if still_missing:
|
||||
log_config.error(f"❌ 迁移后仍有表缺失: {still_missing}")
|
||||
conn.close()
|
||||
return False
|
||||
|
||||
log_config.info("✅ SQLite 监控表迁移完成")
|
||||
conn.close()
|
||||
return True
|
||||
else:
|
||||
log_config.error(f"❌ 迁移脚本不存在: {migration_script}")
|
||||
conn.close()
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
log_config.error(f"❌ SQLite 监控表初始化失败: {e}")
|
||||
try:
|
||||
if 'conn' in locals():
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
async def warmup_connections():
|
||||
"""
|
||||
预热数据库连接，增强容错处理
|
||||
|
||||
+13
-1
@@ -17,7 +17,7 @@ from apps.common.monitor import (
|
||||
from apps.common.monitor.log_stream_service import start_log_stream, stop_log_stream
|
||||
from globalobjects import EVENT_AGGREGATOR
|
||||
from core.settings import TURNON_BINLOG_LISTENER, TRUNON_SCHEDULER, MAX_EVENTS_BATCH_SIZE
|
||||
from core.database import check_db_connections, warmup_connections, start_pool_monitoring, db_init_manager
|
||||
from core.database import check_db_connections, warmup_connections, start_pool_monitoring, db_init_manager, ensure_sqlite_monitor_tables
|
||||
from core.task_manager import get_task_manager
|
||||
|
||||
|
||||
@@ -73,6 +73,18 @@ async def lifespan(app):
|
||||
except Exception as e:
|
||||
log_config.error(f"❌ 数据库连接预热失败: {e}")
|
||||
|
||||
log_config.info("检查 SQLite 监控表...")
|
||||
try:
|
||||
tables_ready = await asyncio.wait_for(ensure_sqlite_monitor_tables(), timeout=30)
|
||||
if tables_ready:
|
||||
log_config.info("✅ SQLite 监控表检查完成")
|
||||
else:
|
||||
log_config.warning("⚠️ SQLite 监控表检查未通过，部分功能可能受影响")
|
||||
except asyncio.TimeoutError:
|
||||
log_config.warning("⚠️ SQLite 监控表检查超时")
|
||||
except Exception as e:
|
||||
log_config.error(f"❌ SQLite 监控表检查异常: {e}")
|
||||
|
||||
log_config.info("开始启动资源监控...")
|
||||
resource_monitor.start_monitoring(interval=30)
|
||||
log_config.info("系统资源监控已启动")
|
||||
|
||||
@@ -147,50 +147,64 @@ else
|
||||
# 2. 检查容器是否存在
|
||||
echo -e "\n${YELLOW}🔍 检查MyAPS API容器状态...${NC}"
|
||||
if ! docker inspect "$CONTAINER_NAME" &>/dev/null; then
|
||||
echo -e "${RED}❌ 错误: 容器 $CONTAINER_NAME 不存在${NC}"
|
||||
echo -e "${RED} 请先启动MyAPS API容器${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查容器是否运行
|
||||
CONTAINER_STATUS=$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME")
|
||||
if [ "$CONTAINER_STATUS" != "running" ]; then
|
||||
if $DRY_RUN; then
|
||||
echo -e "${YELLOW}⚠️ [模拟] 容器未运行，将启动...${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠️ 容器未运行，正在启动...${NC}"
|
||||
docker start "$CONTAINER_NAME"
|
||||
sleep 5
|
||||
fi
|
||||
fi
|
||||
echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}"
|
||||
|
||||
# 3. 执行建表
|
||||
echo -e "\n${YELLOW}⚙️ 执行建表脚本...${NC}"
|
||||
echo -e "${BLUE} 数据库文件: ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3${NC}"
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo -e "${YELLOW} [模拟] docker cp ${SQL_FILE} ${CONTAINER_NAME}:/tmp/monitor_tables.sql${NC}"
|
||||
echo -e "${YELLOW} [模拟] docker exec ${CONTAINER_NAME} sqlite3 ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3 < /tmp/monitor_tables.sql${NC}"
|
||||
echo -e "${YELLOW}⚠️ 容器 $CONTAINER_NAME 不存在，自动切换到本地模式${NC}"
|
||||
LOCAL_MODE=true
|
||||
else
|
||||
# 复制SQL文件到容器
|
||||
echo -e "${YELLOW} 复制SQL文件到容器...${NC}"
|
||||
docker cp "$SQL_FILE" "$CONTAINER_NAME":/tmp/monitor_tables.sql
|
||||
|
||||
# 执行建表脚本
|
||||
echo -e "${YELLOW} 执行SQL脚本...${NC}"
|
||||
docker exec "$CONTAINER_NAME" sqlite3 "${STORAGE_DIR}/${SQLITE_FILE}.sqlite3" < /tmp/monitor_tables.sql
|
||||
|
||||
EXIT_CODE=$?
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✅ 建表成功${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}"
|
||||
exit 1
|
||||
CONTAINER_STATUS=$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME")
|
||||
if [ "$CONTAINER_STATUS" != "running" ]; then
|
||||
echo -e "${YELLOW}⚠️ 容器 $CONTAINER_NAME 未运行，自动切换到本地模式${NC}"
|
||||
LOCAL_MODE=true
|
||||
fi
|
||||
fi
|
||||
|
||||
# 清理容器内的临时文件
|
||||
docker exec "$CONTAINER_NAME" rm -f /tmp/monitor_tables.sql
|
||||
if [ "$LOCAL_MODE" = true ]; then
|
||||
echo -e "\n${YELLOW}⚙️ 本地模式执行...${NC}"
|
||||
|
||||
DB_PATH="${SCRIPT_DIR}/../../../storage/${SQLITE_FILE}.sqlite3"
|
||||
echo -e "${BLUE} 数据库文件: ${DB_PATH}${NC}"
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo -e "${YELLOW} [模拟] sqlite3 ${DB_PATH} < ${SQL_FILE}${NC}"
|
||||
else
|
||||
mkdir -p "$(dirname "$DB_PATH")"
|
||||
echo -e "${YELLOW} 执行SQL脚本...${NC}"
|
||||
sqlite3 "$DB_PATH" < "$SQL_FILE"
|
||||
|
||||
EXIT_CODE=$?
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✅ 建表成功${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}"
|
||||
|
||||
# 3. 执行建表
|
||||
echo -e "\n${YELLOW}⚙️ 执行建表脚本...${NC}"
|
||||
echo -e "${BLUE} 数据库文件: ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3${NC}"
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo -e "${YELLOW} [模拟] docker cp ${SQL_FILE} ${CONTAINER_NAME}:/tmp/monitor_tables.sql${NC}"
|
||||
echo -e "${YELLOW} [模拟] docker exec ${CONTAINER_NAME} sqlite3 ${STORAGE_DIR}/${SQLITE_FILE}.sqlite3 < /tmp/monitor_tables.sql${NC}"
|
||||
else
|
||||
echo -e "${YELLOW} 复制SQL文件到容器...${NC}"
|
||||
docker cp "$SQL_FILE" "$CONTAINER_NAME":/tmp/monitor_tables.sql
|
||||
|
||||
echo -e "${YELLOW} 执行SQL脚本...${NC}"
|
||||
docker exec "$CONTAINER_NAME" sqlite3 "${STORAGE_DIR}/${SQLITE_FILE}.sqlite3" < /tmp/monitor_tables.sql
|
||||
|
||||
EXIT_CODE=$?
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✅ 建表成功${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker exec "$CONTAINER_NAME" rm -f /tmp/monitor_tables.sql
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ set -e
|
||||
# 配置
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
SQL_FILE="${SCRIPT_DIR}/staging_tables.sql"
|
||||
ENV_FILE="${SCRIPT_DIR}/../../.env"
|
||||
ENV_FILE="${SCRIPT_DIR}/../../../.env"
|
||||
CONTAINER_NAME="myaps_postgres"
|
||||
|
||||
# 默认数据库名
|
||||
@@ -47,6 +47,7 @@ show_help() {
|
||||
echo " --db, -d 指定数据库名称（必填）"
|
||||
echo " --container, -c 指定PostgreSQL容器名称 (默认: $CONTAINER_NAME)"
|
||||
echo " --dry-run, -n 仅显示将要执行的操作，不实际执行"
|
||||
echo " --local, -l 在本地直接执行（不通过容器）"
|
||||
echo ""
|
||||
echo "示例:"
|
||||
echo " # 使用默认配置"
|
||||
@@ -55,12 +56,16 @@ show_help() {
|
||||
echo " # 指定容器名称"
|
||||
echo " ./setup_staging_tables.sh -d myaps_db -c my_postgres"
|
||||
echo ""
|
||||
echo " # 本地执行（开发环境）"
|
||||
echo " ./setup_staging_tables.sh -d myaps_db -l"
|
||||
echo ""
|
||||
echo " # 模拟执行"
|
||||
echo " ./setup_staging_tables.sh -d myaps_db -n"
|
||||
}
|
||||
|
||||
# 解析参数
|
||||
DRY_RUN=false
|
||||
LOCAL_MODE=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -79,6 +84,9 @@ while [[ $# -gt 0 ]]; do
|
||||
--dry-run|-n)
|
||||
DRY_RUN=true
|
||||
;;
|
||||
--local|-l)
|
||||
LOCAL_MODE=true
|
||||
;;
|
||||
*)
|
||||
echo "未知选项: $1"
|
||||
show_help
|
||||
@@ -95,6 +103,9 @@ YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
# 读取环境变量（优先使用命令行参数，env文件作兜底）
|
||||
read_env_config
|
||||
|
||||
# 检查数据库名称是否指定
|
||||
if [ -z "$DB_NAME" ]; then
|
||||
echo -e "${RED}❌ 错误: 必须使用 -d 指定数据库名称${NC}"
|
||||
@@ -102,9 +113,6 @@ if [ -z "$DB_NAME" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 读取环境变量
|
||||
read_env_config
|
||||
|
||||
echo -e "${BLUE}==============================================${NC}"
|
||||
echo -e "${BLUE} PostgreSQL 缓冲表一键建表脚本${NC}"
|
||||
echo -e "${BLUE}==============================================${NC}"
|
||||
@@ -117,53 +125,165 @@ if [ ! -f "$SQL_FILE" ]; then
|
||||
fi
|
||||
echo -e "${GREEN}✅ SQL文件存在: $SQL_FILE${NC}"
|
||||
|
||||
# 2. 检查容器是否存在
|
||||
echo -e "\n${YELLOW}🔍 检查PostgreSQL容器状态...${NC}"
|
||||
if ! docker inspect "$CONTAINER_NAME" &>/dev/null; then
|
||||
echo -e "${RED}❌ 错误: 容器 $CONTAINER_NAME 不存在${NC}"
|
||||
echo -e "${RED} 请先启动PostgreSQL容器${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查容器是否运行
|
||||
CONTAINER_STATUS=$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME")
|
||||
if [ "$CONTAINER_STATUS" != "running" ]; then
|
||||
if $DRY_RUN; then
|
||||
echo -e "${YELLOW}⚠️ [模拟] 容器未运行，将启动...${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠️ 容器未运行，正在启动...${NC}"
|
||||
docker start "$CONTAINER_NAME"
|
||||
sleep 5
|
||||
fi
|
||||
fi
|
||||
echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}"
|
||||
|
||||
# 3. 执行建表
|
||||
echo -e "\n${YELLOW}⚙️ 执行建表脚本...${NC}"
|
||||
echo -e "${BLUE} 数据库: ${DB_NAME}${NC}"
|
||||
echo -e "${BLUE} 用户: ${DB_USER}${NC}"
|
||||
echo -e "${BLUE} 端口: ${DB_PORT}${NC}"
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo -e "${YELLOW} [模拟] docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME -f - < $SQL_FILE${NC}"
|
||||
else
|
||||
# 检查数据库是否存在，不存在则创建
|
||||
echo -e "${YELLOW} 检查数据库是否存在...${NC}"
|
||||
if ! docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" | grep -q 1; then
|
||||
exec_sql_container() {
|
||||
local db="$1"
|
||||
if ! docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='$db'" | grep -q 1; then
|
||||
echo -e "${YELLOW} 数据库不存在，创建数据库...${NC}"
|
||||
docker exec "$CONTAINER_NAME" createdb -U "$DB_USER" "$DB_NAME"
|
||||
docker exec "$CONTAINER_NAME" createdb -U "$DB_USER" "$db"
|
||||
fi
|
||||
echo -e "${YELLOW} 执行SQL脚本...${NC}"
|
||||
cat "$SQL_FILE" | docker exec -i "$CONTAINER_NAME" psql -U "$DB_USER" -d "$db"
|
||||
return $?
|
||||
}
|
||||
|
||||
exec_sql_local() {
|
||||
local db="$1"
|
||||
export PGPASSWORD="$DB_PASSWORD"
|
||||
if ! command -v psql &>/dev/null; then
|
||||
echo -e "${RED}❌ 错误: 未找到psql客户端${NC}"
|
||||
echo -e "${YELLOW} 请安装postgresql-client或使用Docker模式${NC}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 执行建表脚本
|
||||
echo -e "${YELLOW} 执行SQL脚本...${NC}"
|
||||
cat "$SQL_FILE" | docker exec -i "$CONTAINER_NAME" psql -U "$DB_USER" -d "$DB_NAME"
|
||||
echo -e "${YELLOW} 尝试连接PostgreSQL...${NC}"
|
||||
if ! psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d postgres -tAc "SELECT 1" 2>/dev/null | grep -q 1; then
|
||||
echo -e "${RED}❌ 错误: 无法连接到PostgreSQL服务器${NC}"
|
||||
echo -e "${YELLOW} 服务器: ${DB_HOST}:${DB_PORT}, 用户: ${DB_USER}${NC}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
EXIT_CODE=$?
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✅ 建表成功${NC}"
|
||||
if ! psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -tAc "SELECT 1 FROM pg_database WHERE datname='$db'" 2>/dev/null | grep -q 1; then
|
||||
echo -e "${YELLOW} 数据库不存在，创建数据库...${NC}"
|
||||
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -c "CREATE DATABASE $db" 2>/dev/null
|
||||
fi
|
||||
echo -e "${YELLOW} 执行SQL脚本...${NC}"
|
||||
cat "$SQL_FILE" | psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$db"
|
||||
return $?
|
||||
}
|
||||
|
||||
verify_tables_container() {
|
||||
local db="$1"
|
||||
TABLES=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$db" -tAc "SELECT tablename FROM pg_tables WHERE tablename LIKE '%staging' OR tablename IN ('t_validation_error', 't_transform_rule', 't_schema_version') ORDER BY tablename;")
|
||||
VERSION=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$db" -tAc "SELECT version FROM t_schema_version ORDER BY applied_at DESC LIMIT 1;")
|
||||
}
|
||||
|
||||
verify_tables_local() {
|
||||
local db="$1"
|
||||
export PGPASSWORD="$DB_PASSWORD"
|
||||
TABLES=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$db" -tAc "SELECT tablename FROM pg_tables WHERE tablename LIKE '%staging' OR tablename IN ('t_validation_error', 't_transform_rule', 't_schema_version') ORDER BY tablename;" 2>/dev/null)
|
||||
VERSION=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$db" -tAc "SELECT version FROM t_schema_version ORDER BY applied_at DESC LIMIT 1;" 2>/dev/null)
|
||||
}
|
||||
|
||||
# 检查Docker权限
|
||||
check_docker_access() {
|
||||
if docker info &>/dev/null; then
|
||||
return 0
|
||||
else
|
||||
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 自动检测运行中的PostgreSQL容器
|
||||
find_postgres_container() {
|
||||
# 首先检查Docker权限
|
||||
if ! check_docker_access; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 查找运行中的postgres容器
|
||||
local pg_container=$(docker ps --filter "ancestor=postgres" --format "{{.Names}}" 2>/dev/null | head -1)
|
||||
if [ -n "$pg_container" ]; then
|
||||
echo "$pg_container"
|
||||
return 0
|
||||
fi
|
||||
# 尝试其他方式查找
|
||||
pg_container=$(docker ps --filter "name=postgres" --format "{{.Names}}" 2>/dev/null | head -1)
|
||||
if [ -n "$pg_container" ]; then
|
||||
echo "$pg_container"
|
||||
return 0
|
||||
fi
|
||||
# 查找包含postgres的容器
|
||||
pg_container=$(docker ps --filter "name=*postgres*" --format "{{.Names}}" 2>/dev/null | head -1)
|
||||
if [ -n "$pg_container" ]; then
|
||||
echo "$pg_container"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# 2. 检查容器是否存在并决定执行模式
|
||||
echo -e "\n${YELLOW}🔍 检查PostgreSQL容器状态...${NC}"
|
||||
AUTO_LOCAL=false
|
||||
|
||||
# 先检查Docker权限
|
||||
if ! check_docker_access; then
|
||||
echo -e "${YELLOW}⚠️ Docker权限不足${NC}"
|
||||
echo -e "${YELLOW} 请使用 sudo 运行脚本或检查Docker配置${NC}"
|
||||
echo -e "${YELLOW} 尝试本地模式...${NC}"
|
||||
AUTO_LOCAL=true
|
||||
LOCAL_MODE=true
|
||||
else
|
||||
if ! docker inspect "$CONTAINER_NAME" &>/dev/null; then
|
||||
echo -e "${YELLOW}⚠️ 配置的容器 $CONTAINER_NAME 不存在${NC}"
|
||||
# 尝试自动检测PostgreSQL容器
|
||||
DETECTED_CONTAINER=$(find_postgres_container)
|
||||
if [ -n "$DETECTED_CONTAINER" ]; then
|
||||
echo -e "${GREEN}✅ 自动检测到运行中的PostgreSQL容器: $DETECTED_CONTAINER${NC}"
|
||||
CONTAINER_NAME="$DETECTED_CONTAINER"
|
||||
else
|
||||
echo -e "${YELLOW} 未检测到运行中的PostgreSQL容器，尝试本地模式${NC}"
|
||||
AUTO_LOCAL=true
|
||||
LOCAL_MODE=true
|
||||
fi
|
||||
elif [ "$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME")" != "running" ]; then
|
||||
echo -e "${YELLOW}⚠️ 容器 $CONTAINER_NAME 未运行${NC}"
|
||||
DETECTED_CONTAINER=$(find_postgres_container)
|
||||
if [ -n "$DETECTED_CONTAINER" ]; then
|
||||
echo -e "${GREEN}✅ 自动检测到运行中的PostgreSQL容器: $DETECTED_CONTAINER${NC}"
|
||||
CONTAINER_NAME="$DETECTED_CONTAINER"
|
||||
else
|
||||
echo -e "${YELLOW} 未检测到运行中的PostgreSQL容器，尝试本地模式${NC}"
|
||||
AUTO_LOCAL=true
|
||||
LOCAL_MODE=true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$LOCAL_MODE" = true ]; then
|
||||
echo -e "${GREEN}✅ 使用本地PostgreSQL执行${NC}"
|
||||
echo -e "${BLUE} 主机: ${DB_HOST}:${DB_PORT}${NC}"
|
||||
echo -e "${BLUE} 用户: ${DB_USER}${NC}"
|
||||
echo -e "${BLUE} 数据库: ${DB_NAME}${NC}"
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo -e "${YELLOW} [模拟] psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -f $SQL_FILE${NC}"
|
||||
else
|
||||
exec_sql_local "$DB_NAME"
|
||||
EXIT_CODE=$?
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✅ 建表成功${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo -e "${GREEN}✅ 容器 $CONTAINER_NAME 运行正常${NC}"
|
||||
echo -e "${BLUE} 数据库: ${DB_NAME}${NC}"
|
||||
echo -e "${BLUE} 用户: ${DB_USER}${NC}"
|
||||
echo -e "${BLUE} 端口: ${DB_PORT}${NC}"
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo -e "${YELLOW} [模拟] docker exec -i $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME -f - < $SQL_FILE${NC}"
|
||||
else
|
||||
exec_sql_container "$DB_NAME"
|
||||
EXIT_CODE=$?
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✅ 建表成功${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ 建表失败 (退出码: $EXIT_CODE)${NC}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -172,18 +292,28 @@ echo -e "\n${YELLOW}📊 验证建表结果...${NC}"
|
||||
if $DRY_RUN; then
|
||||
echo -e "${YELLOW} [模拟] 验证缓冲表是否创建成功${NC}"
|
||||
else
|
||||
TABLES=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$DB_NAME" -tAc "SELECT tablename FROM pg_tables WHERE tablename LIKE '%staging' OR tablename IN ('t_validation_error', 't_transform_rule', 't_schema_version') ORDER BY tablename;")
|
||||
echo -e "${GREEN}✅ 创建的表:${NC}"
|
||||
echo "$TABLES" | while read -r table; do
|
||||
if [ -n "$table" ]; then
|
||||
echo -e " - $table"
|
||||
if [ "$LOCAL_MODE" = true ]; then
|
||||
verify_tables_local "$DB_NAME"
|
||||
echo -e "${GREEN}✅ 创建的表:${NC}"
|
||||
echo "$TABLES" | while read -r table; do
|
||||
if [ -n "$table" ]; then
|
||||
echo -e " - $table"
|
||||
fi
|
||||
done
|
||||
if [ -n "$VERSION" ]; then
|
||||
echo -e "\n${GREEN}✅ 版本记录: $VERSION${NC}"
|
||||
fi
|
||||
else
|
||||
verify_tables_container "$DB_NAME"
|
||||
echo -e "${GREEN}✅ 创建的表:${NC}"
|
||||
echo "$TABLES" | while read -r table; do
|
||||
if [ -n "$table" ]; then
|
||||
echo -e " - $table"
|
||||
fi
|
||||
done
|
||||
if [ -n "$VERSION" ]; then
|
||||
echo -e "\n${GREEN}✅ 版本记录: $VERSION${NC}"
|
||||
fi
|
||||
done
|
||||
|
||||
# 检查版本记录
|
||||
VERSION=$(docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$DB_NAME" -tAc "SELECT version FROM t_schema_version ORDER BY applied_at DESC LIMIT 1;")
|
||||
if [ -n "$VERSION" ]; then
|
||||
echo -e "\n${GREEN}✅ 版本记录: $VERSION${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -195,5 +325,9 @@ else
|
||||
fi
|
||||
echo -e "${BLUE}==============================================${NC}"
|
||||
echo -e "\n${YELLOW}💡 验证命令:${NC}"
|
||||
echo -e "${YELLOW} docker exec -it $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME${NC}"
|
||||
if [ "$LOCAL_MODE" = true ]; then
|
||||
echo -e "${YELLOW} PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME${NC}"
|
||||
else
|
||||
echo -e "${YELLOW} docker exec -it $CONTAINER_NAME psql -U $DB_USER -d $DB_NAME${NC}"
|
||||
fi
|
||||
echo -e "${YELLOW} SELECT * FROM t_schema_version;${NC}"
|
||||
Reference in New Issue
Block a user