228 lines
6.9 KiB
Python
228 lines
6.9 KiB
Python
"""
|
|
定时维护：清理过期构建任务（含本地产物）、清理残留临时文件、在线备份 SQLite。
|
|
|
|
被 management 命令（容器内常驻守护进程）与后台「维护设置」页面共用：
|
|
run_maintenance(force_backup=False) -> dict # 执行一轮
|
|
maintenance_status() -> dict # 页面展示的状态摘要
|
|
所有设置经 app_settings 统一读取（后台配置 → 环境变量 → 默认值）。
|
|
"""
|
|
import os
|
|
import shutil
|
|
import sqlite3
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
from django.conf import settings
|
|
from django.utils import timezone
|
|
|
|
from . import app_settings
|
|
from .models import GithubRun
|
|
|
|
# 与 GithubRun.is_finished 保持一致的终态集合
|
|
FINISHED_STATUSES = ('success', 'failure', 'cancelled', 'timed_out', 'skipped')
|
|
|
|
# 临时配置包/JSON 的最长保留时间（天），失败任务遗留时兜底清理
|
|
TEMP_FILE_DAYS = 7
|
|
|
|
BACKUP_NAME_PREFIX = 'db-'
|
|
BACKUP_NAME_SUFFIX = '.sqlite3'
|
|
# 自动备份最小间隔（秒）：守护进程每小时醒一次，但一天只备份一次
|
|
BACKUP_INTERVAL = 24 * 3600
|
|
|
|
|
|
def _setting_int(key, default):
|
|
try:
|
|
return int(str(app_settings.get_value(key)).strip())
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def _setting_bool(key, default=True):
|
|
return str(app_settings.get_value(key)).strip().lower() in ('1', 'true', 'yes', 'on', '是')
|
|
|
|
|
|
def retention_days():
|
|
"""构建任务保留天数；0 表示永不自动清理。"""
|
|
return max(0, _setting_int('BUILD_RETENTION_DAYS', 30))
|
|
|
|
|
|
def backup_enabled():
|
|
return _setting_bool('DB_BACKUP_ENABLED', True)
|
|
|
|
|
|
def backup_keep():
|
|
return max(1, min(100, _setting_int('DB_BACKUP_KEEP', 7)))
|
|
|
|
|
|
def db_path():
|
|
return Path(settings.DATABASES['default']['NAME'])
|
|
|
|
|
|
def backup_dir():
|
|
d = db_path().parent / 'backups'
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
return d
|
|
|
|
|
|
def list_backups():
|
|
"""返回 [(Path, 大小字节, mtime), ...]，按时间倒序。"""
|
|
items = []
|
|
for p in backup_dir().glob(f'{BACKUP_NAME_PREFIX}*{BACKUP_NAME_SUFFIX}'):
|
|
if p.is_file():
|
|
st = p.stat()
|
|
items.append((p, st.st_size, st.st_mtime))
|
|
items.sort(key=lambda x: x[2], reverse=True)
|
|
return items
|
|
|
|
|
|
def cleanup_expired_builds(days=None):
|
|
"""删除超过保留天数的终态任务，并同步清理 exe/png 产物。返回删除条数。"""
|
|
if days is None:
|
|
days = retention_days()
|
|
if days <= 0:
|
|
return 0
|
|
|
|
cutoff = timezone.now() - timedelta(days=days)
|
|
qs = GithubRun.objects.filter(created_at__lt=cutoff, status__in=FINISHED_STATUSES)
|
|
count = 0
|
|
for run in qs.iterator():
|
|
if run.uuid:
|
|
shutil.rmtree(Path('exe') / run.uuid, ignore_errors=True)
|
|
shutil.rmtree(Path('png') / run.uuid, ignore_errors=True)
|
|
run.delete()
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def cleanup_temp_files(days=TEMP_FILE_DAYS):
|
|
"""清理失败任务遗留的加密配置包与临时 JSON。返回删除文件数。"""
|
|
cutoff = time.time() - days * 86400
|
|
count = 0
|
|
targets = []
|
|
zip_dir = Path('temp_zips')
|
|
if zip_dir.is_dir():
|
|
targets += list(zip_dir.glob('secrets_*.zip'))
|
|
# 早期版本 data_*.json 写在工作目录根下
|
|
targets += list(Path('.').glob('data_*.json'))
|
|
for p in targets:
|
|
try:
|
|
if p.is_file() and p.stat().st_mtime < cutoff:
|
|
p.unlink()
|
|
count += 1
|
|
except OSError:
|
|
pass
|
|
return count
|
|
|
|
|
|
def backup_database():
|
|
"""用 sqlite3 在线备份 API 生成一致性副本，然后按保留份数轮转。
|
|
|
|
返回备份文件 Path；非 SQLite 数据库或源文件不存在时返回 None。
|
|
"""
|
|
src = db_path()
|
|
if not src.is_file() or settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3':
|
|
return None
|
|
|
|
dest = backup_dir() / f"{BACKUP_NAME_PREFIX}{time.strftime('%Y%m%d-%H%M%S')}{BACKUP_NAME_SUFFIX}"
|
|
src_conn = sqlite3.connect(str(src))
|
|
try:
|
|
dst_conn = sqlite3.connect(str(dest))
|
|
try:
|
|
src_conn.backup(dst_conn)
|
|
finally:
|
|
dst_conn.close()
|
|
finally:
|
|
src_conn.close()
|
|
|
|
rotate_backups()
|
|
return dest
|
|
|
|
|
|
def rotate_backups():
|
|
"""按设置的保留份数删除最旧备份（每轮维护都执行，含容器启动前的预迁移备份）。"""
|
|
deleted = 0
|
|
for old_path, _size, _mtime in list_backups()[backup_keep():]:
|
|
try:
|
|
old_path.unlink()
|
|
deleted += 1
|
|
except OSError:
|
|
pass
|
|
return deleted
|
|
|
|
|
|
def _latest_backup_age():
|
|
backups = list_backups()
|
|
if not backups:
|
|
return None
|
|
return time.time() - backups[0][2]
|
|
|
|
|
|
def run_maintenance(force_backup=False):
|
|
"""执行一轮维护。返回各项动作的计数/结果（供命令行与页面反馈）。"""
|
|
result = {
|
|
'deleted_builds': cleanup_expired_builds(),
|
|
'deleted_temp_files': cleanup_temp_files(),
|
|
'rotated_backups': rotate_backups(),
|
|
'backup_created': None,
|
|
'backup_skipped': False,
|
|
}
|
|
if backup_enabled() and (force_backup or (_latest_backup_age() or BACKUP_INTERVAL) >= BACKUP_INTERVAL):
|
|
dest = backup_database()
|
|
result['backup_created'] = str(dest) if dest else None
|
|
if dest is None:
|
|
result['backup_skipped'] = True
|
|
else:
|
|
result['backup_skipped'] = True
|
|
return result
|
|
|
|
|
|
def _dir_size(path):
|
|
total = 0
|
|
base = Path(path)
|
|
if not base.is_dir():
|
|
return 0
|
|
for root, _dirs, files in os.walk(base):
|
|
for name in files:
|
|
try:
|
|
total += (Path(root) / name).stat().st_size
|
|
except OSError:
|
|
pass
|
|
return total
|
|
|
|
|
|
def maintenance_status():
|
|
"""后台维护页展示用摘要。"""
|
|
days = retention_days()
|
|
total = GithubRun.objects.count()
|
|
in_progress = GithubRun.objects.exclude(status__in=FINISHED_STATUSES).count()
|
|
expired = 0
|
|
if days > 0:
|
|
cutoff = timezone.now() - timedelta(days=days)
|
|
expired = GithubRun.objects.filter(
|
|
created_at__lt=cutoff, status__in=FINISHED_STATUSES
|
|
).count()
|
|
|
|
backups = [
|
|
{
|
|
'name': p.name,
|
|
'size': size,
|
|
'mtime': datetime.fromtimestamp(mtime, tz=timezone.get_current_timezone()),
|
|
}
|
|
for p, size, mtime in list_backups()
|
|
]
|
|
return {
|
|
'retention_days': days,
|
|
'backup_enabled': backup_enabled(),
|
|
'backup_keep': backup_keep(),
|
|
'db_path': str(db_path()),
|
|
'db_size': db_path().stat().st_size if db_path().is_file() else 0,
|
|
'total_builds': total,
|
|
'in_progress_builds': in_progress,
|
|
'expired_builds': expired,
|
|
'artifacts_size': _dir_size('exe') + _dir_size('png'),
|
|
'temp_size': _dir_size('temp_zips'),
|
|
'backups': backups,
|
|
'backup_interval_hours': BACKUP_INTERVAL // 3600,
|
|
}
|