Initial commit: rdgen with GitHub/Gitea dual build backend

2026-09-29 19:56:02 +08:00
commit a3686a49d4
103 changed files with 20007 additions and 0 deletions
@@ -0,0 +1,227 @@
+"""
+定时维护：清理过期构建任务（含本地产物）、清理残留临时文件、在线备份 SQLite。
+
+被 management 命令（容器内常驻守护进程）与后台「维护设置」页面共用：
+    run_maintenance(force_backup=False) -> dict  # 执行一轮
+    maintenance_status() -> dict                # 页面展示的状态摘要
+所有设置经 app_settings 统一读取（后台配置 → 环境变量 → 默认值）。
+"""
+import os
+import shutil
+import sqlite3
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from django.conf import settings
+from django.utils import timezone
+
+from . import app_settings
+from .models import GithubRun
+
+# 与 GithubRun.is_finished 保持一致的终态集合
+FINISHED_STATUSES = ('success', 'failure', 'cancelled', 'timed_out', 'skipped')
+
+# 临时配置包/JSON 的最长保留时间（天），失败任务遗留时兜底清理
+TEMP_FILE_DAYS = 7
+
+BACKUP_NAME_PREFIX = 'db-'
+BACKUP_NAME_SUFFIX = '.sqlite3'
+# 自动备份最小间隔（秒）：守护进程每小时醒一次，但一天只备份一次
+BACKUP_INTERVAL = 24 * 3600
+
+
+def _setting_int(key, default):
+    try:
+        return int(str(app_settings.get_value(key)).strip())
+    except (TypeError, ValueError):
+        return default
+
+
+def _setting_bool(key, default=True):
+    return str(app_settings.get_value(key)).strip().lower() in ('1', 'true', 'yes', 'on', '是')
+
+
+def retention_days():
+    """构建任务保留天数；0 表示永不自动清理。"""
+    return max(0, _setting_int('BUILD_RETENTION_DAYS', 30))
+
+
+def backup_enabled():
+    return _setting_bool('DB_BACKUP_ENABLED', True)
+
+
+def backup_keep():
+    return max(1, min(100, _setting_int('DB_BACKUP_KEEP', 7)))
+
+
+def db_path():
+    return Path(settings.DATABASES['default']['NAME'])
+
+
+def backup_dir():
+    d = db_path().parent / 'backups'
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def list_backups():
+    """返回 [(Path, 大小字节, mtime), ...]，按时间倒序。"""
+    items = []
+    for p in backup_dir().glob(f'{BACKUP_NAME_PREFIX}*{BACKUP_NAME_SUFFIX}'):
+        if p.is_file():
+            st = p.stat()
+            items.append((p, st.st_size, st.st_mtime))
+    items.sort(key=lambda x: x[2], reverse=True)
+    return items
+
+
+def cleanup_expired_builds(days=None):
+    """删除超过保留天数的终态任务，并同步清理 exe/png 产物。返回删除条数。"""
+    if days is None:
+        days = retention_days()
+    if days <= 0:
+        return 0
+
+    cutoff = timezone.now() - timedelta(days=days)
+    qs = GithubRun.objects.filter(created_at__lt=cutoff, status__in=FINISHED_STATUSES)
+    count = 0
+    for run in qs.iterator():
+        if run.uuid:
+            shutil.rmtree(Path('exe') / run.uuid, ignore_errors=True)
+            shutil.rmtree(Path('png') / run.uuid, ignore_errors=True)
+        run.delete()
+        count += 1
+    return count
+
+
+def cleanup_temp_files(days=TEMP_FILE_DAYS):
+    """清理失败任务遗留的加密配置包与临时 JSON。返回删除文件数。"""
+    cutoff = time.time() - days * 86400
+    count = 0
+    targets = []
+    zip_dir = Path('temp_zips')
+    if zip_dir.is_dir():
+        targets += list(zip_dir.glob('secrets_*.zip'))
+    # 早期版本 data_*.json 写在工作目录根下
+    targets += list(Path('.').glob('data_*.json'))
+    for p in targets:
+        try:
+            if p.is_file() and p.stat().st_mtime < cutoff:
+                p.unlink()
+                count += 1
+        except OSError:
+            pass
+    return count
+
+
+def backup_database():
+    """用 sqlite3 在线备份 API 生成一致性副本，然后按保留份数轮转。
+
+    返回备份文件 Path；非 SQLite 数据库或源文件不存在时返回 None。
+    """
+    src = db_path()
+    if not src.is_file() or settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3':
+        return None
+
+    dest = backup_dir() / f"{BACKUP_NAME_PREFIX}{time.strftime('%Y%m%d-%H%M%S')}{BACKUP_NAME_SUFFIX}"
+    src_conn = sqlite3.connect(str(src))
+    try:
+        dst_conn = sqlite3.connect(str(dest))
+        try:
+            src_conn.backup(dst_conn)
+        finally:
+            dst_conn.close()
+    finally:
+        src_conn.close()
+
+    rotate_backups()
+    return dest
+
+
+def rotate_backups():
+    """按设置的保留份数删除最旧备份（每轮维护都执行，含容器启动前的预迁移备份）。"""
+    deleted = 0
+    for old_path, _size, _mtime in list_backups()[backup_keep():]:
+        try:
+            old_path.unlink()
+            deleted += 1
+        except OSError:
+            pass
+    return deleted
+
+
+def _latest_backup_age():
+    backups = list_backups()
+    if not backups:
+        return None
+    return time.time() - backups[0][2]
+
+
+def run_maintenance(force_backup=False):
+    """执行一轮维护。返回各项动作的计数/结果（供命令行与页面反馈）。"""
+    result = {
+        'deleted_builds': cleanup_expired_builds(),
+        'deleted_temp_files': cleanup_temp_files(),
+        'rotated_backups': rotate_backups(),
+        'backup_created': None,
+        'backup_skipped': False,
+    }
+    if backup_enabled() and (force_backup or (_latest_backup_age() or BACKUP_INTERVAL) >= BACKUP_INTERVAL):
+        dest = backup_database()
+        result['backup_created'] = str(dest) if dest else None
+        if dest is None:
+            result['backup_skipped'] = True
+    else:
+        result['backup_skipped'] = True
+    return result
+
+
+def _dir_size(path):
+    total = 0
+    base = Path(path)
+    if not base.is_dir():
+        return 0
+    for root, _dirs, files in os.walk(base):
+        for name in files:
+            try:
+                total += (Path(root) / name).stat().st_size
+            except OSError:
+                pass
+    return total
+
+
+def maintenance_status():
+    """后台维护页展示用摘要。"""
+    days = retention_days()
+    total = GithubRun.objects.count()
+    in_progress = GithubRun.objects.exclude(status__in=FINISHED_STATUSES).count()
+    expired = 0
+    if days > 0:
+        cutoff = timezone.now() - timedelta(days=days)
+        expired = GithubRun.objects.filter(
+            created_at__lt=cutoff, status__in=FINISHED_STATUSES
+        ).count()
+
+    backups = [
+        {
+            'name': p.name,
+            'size': size,
+            'mtime': datetime.fromtimestamp(mtime, tz=timezone.get_current_timezone()),
+        }
+        for p, size, mtime in list_backups()
+    ]
+    return {
+        'retention_days': days,
+        'backup_enabled': backup_enabled(),
+        'backup_keep': backup_keep(),
+        'db_path': str(db_path()),
+        'db_size': db_path().stat().st_size if db_path().is_file() else 0,
+        'total_builds': total,
+        'in_progress_builds': in_progress,
+        'expired_builds': expired,
+        'artifacts_size': _dir_size('exe') + _dir_size('png'),
+        'temp_size': _dir_size('temp_zips'),
+        'backups': backups,
+        'backup_interval_hours': BACKUP_INTERVAL // 3600,
+    }