构建状态增加平台终态只读核对,修复回调丢失后任务长期卡在构建中
工作流在脚本收尾前失败(拉取/编译/runner 异常)或 GENURL 不可达时, 结束回调会永久丢失,而等待页 30 秒轮询与列表页此前只读本地库, 任务只能等 6 小时本地超时,平台侧已失败也无法及时反映。 - GitHub/Gitea 后端新增只读 get_run(),仅在平台报告终态时返回结论, 在途、非 200、网络异常、坏 JSON 一律返回 None;不取消、不写平台 - 等待页 30 秒轮询同步核对一次真实状态:20 秒节流、6 秒超时、异常静默 - 我的构建/后台仪表盘改为后台守护线程核对,页面渲染零等待 - 仅做非终态到终态的条件更新,回调抢先到达时不会被覆盖 - 停止/删除纯本地语义、6 小时本地超时兜底等现有逻辑保持不变
This commit is contained in:
@@ -18,6 +18,14 @@ class DispatchResult:
|
||||
html_url: str = '' # 平台上的运行详情页地址
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunInfo:
|
||||
"""平台单次运行的状态快照。"""
|
||||
finished: bool = False # 是否已结束(拿到终态结论)
|
||||
status: str = '' # finished=True 时为 rdgen 内部终态码
|
||||
html_url: str = '' # 平台上的运行详情页地址
|
||||
|
||||
|
||||
# rdgen 目标平台 -> 工作流文件名(两平台同名)
|
||||
WORKFLOW_FILES = {
|
||||
'windows': 'generator-windows.yml',
|
||||
@@ -53,6 +61,11 @@ class BuildBackend:
|
||||
def log_url(self, run_id) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_run(self, run_id, *, timeout=8):
|
||||
"""只读查询单次运行:已结束返回 RunInfo(finished=True, status=内部终态码);
|
||||
仍在途、查询失败或不支持时返回 None(调用方按「仍在进行」处理)。"""
|
||||
return None
|
||||
|
||||
# ---- 后台连接测试 ----
|
||||
def test_connection(self) -> dict:
|
||||
"""返回 {'ok': bool, 'message': 中文说明},不抛异常。"""
|
||||
|
||||
@@ -6,11 +6,25 @@ API 与 GitHub Actions 几乎同形(已对照 Gitea 官方 Swagger 核实)
|
||||
{workflow_run_id, html_url, run_url},不传该参数为 204;
|
||||
- 鉴权:Authorization: token <PAT>。
|
||||
|
||||
注意:本平台不主动查询/取消 Gitea 上的运行。构建状态完全以工作流结束时的
|
||||
回调(update_github_run)为准,停止/删除只作用于本平台本地记录。
|
||||
注意:本平台不主动取消 Gitea 上的运行,停止/删除只作用于本平台本地记录。
|
||||
构建状态以工作流结束时的回调(update_github_run)为第一来源;get_run 仅做
|
||||
只读终态核对,兜底回调丢失(脚本未跑到收尾或 GENURL 不可达)的情况。
|
||||
"""
|
||||
from .. import app_settings, ghnet
|
||||
from .base import BackendError, DispatchResult, WORKFLOW_FILES, BuildBackend
|
||||
from .base import BackendError, DispatchResult, RunInfo, WORKFLOW_FILES, BuildBackend
|
||||
|
||||
# 部分 Gitea 版本 run.status 直接返回终态(无 completed+conclusion 包裹)
|
||||
_GITEA_TERMINAL = {'success', 'failure', 'cancelled', 'skipped', 'timed_out', 'stopped', 'dead'}
|
||||
# Gitea 结论 -> rdgen 内部终态
|
||||
_GITEA_CONCLUSION_MAP = {
|
||||
'success': 'success',
|
||||
'failure': 'failure',
|
||||
'cancelled': 'cancelled',
|
||||
'skipped': 'skipped',
|
||||
'timed_out': 'timed_out',
|
||||
'stopped': 'cancelled',
|
||||
'dead': 'failure',
|
||||
}
|
||||
|
||||
|
||||
class GiteaBackend(BuildBackend):
|
||||
@@ -130,6 +144,42 @@ class GiteaBackend(BuildBackend):
|
||||
print(f'查询 Gitea 最新运行失败:{e}')
|
||||
return None
|
||||
|
||||
# ---- 只读状态查询(回调丢失时的终态兜底) ----
|
||||
def get_run(self, run_id, *, timeout=8):
|
||||
url = self._api(f'/repos/{self._owner()}/{self._repo()}/actions/runs/{run_id}')
|
||||
try:
|
||||
resp = ghnet.get(
|
||||
url, token=app_settings.get_value('GITEA_TOKEN'),
|
||||
timeout=timeout, flavor='gitea', proxy=self._proxy(),
|
||||
)
|
||||
except Exception as e:
|
||||
print(f'查询 Gitea 状态出错:{e}')
|
||||
return None
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
try:
|
||||
data = resp.json()
|
||||
except ValueError:
|
||||
return None
|
||||
html_url = data.get('html_url') or self.log_url(run_id)
|
||||
status = (data.get('status') or '').lower()
|
||||
conclusion = (data.get('conclusion') or '').lower()
|
||||
if status == 'completed':
|
||||
return RunInfo(
|
||||
finished=True,
|
||||
status=_GITEA_CONCLUSION_MAP.get(conclusion, 'failure'),
|
||||
html_url=html_url,
|
||||
)
|
||||
# 部分 Gitea 版本直接在 status 返回终态(无 completed 包裹)
|
||||
if status in _GITEA_TERMINAL:
|
||||
return RunInfo(
|
||||
finished=True,
|
||||
status=_GITEA_CONCLUSION_MAP.get(status, 'failure'),
|
||||
html_url=html_url,
|
||||
)
|
||||
# queued/in_progress/waiting/blocked 等在途状态返回 None,交给回调与下次核对
|
||||
return None
|
||||
|
||||
# ---- 日志链接 ----
|
||||
def log_url(self, run_id):
|
||||
if not run_id:
|
||||
|
||||
@@ -1,13 +1,24 @@
|
||||
"""GitHub Actions 构建后端(迁自 views.py / web_views.py 的原有逻辑,行为保持一致)。
|
||||
|
||||
注意:本平台不主动查询/取消 GitHub 上的运行。构建状态完全以工作流结束时的
|
||||
回调(update_github_run)为准,停止/删除只作用于本平台本地记录。
|
||||
注意:本平台不主动取消 GitHub 上的运行,停止/删除只作用于本平台本地记录。
|
||||
构建状态以工作流结束时的回调(update_github_run)为第一来源;get_run 仅做
|
||||
只读终态核对,兜底回调丢失(脚本未跑到收尾或 GENURL 不可达)的情况。
|
||||
"""
|
||||
from .. import app_settings, ghnet
|
||||
from .base import BackendError, DispatchResult, WORKFLOW_FILES, BuildBackend
|
||||
from .base import BackendError, DispatchResult, RunInfo, WORKFLOW_FILES, BuildBackend
|
||||
|
||||
API = 'https://api.github.com'
|
||||
|
||||
# GitHub conclusion -> rdgen 内部终态;无人值守构建不会有人处理审批,
|
||||
# action_required/neutral/stale/startup_failure 等异常结论一律按失败呈现
|
||||
_GH_CONCLUSION_MAP = {
|
||||
'success': 'success',
|
||||
'failure': 'failure',
|
||||
'cancelled': 'cancelled',
|
||||
'skipped': 'skipped',
|
||||
'timed_out': 'timed_out',
|
||||
}
|
||||
|
||||
|
||||
class GitHubBackend(BuildBackend):
|
||||
name = 'github'
|
||||
@@ -57,6 +68,32 @@ class GitHubBackend(BuildBackend):
|
||||
pass
|
||||
return DispatchResult(run_id=data.get('workflow_run_id'), html_url=data.get('html_url') or '')
|
||||
|
||||
# ---- 只读状态查询(回调丢失时的终态兜底) ----
|
||||
def get_run(self, run_id, *, timeout=8):
|
||||
user = app_settings.get_value('GHUSER')
|
||||
repo = app_settings.get_value('REPONAME')
|
||||
url = f'{API}/repos/{user}/{repo}/actions/runs/{run_id}'
|
||||
try:
|
||||
resp = ghnet.get(url, token=app_settings.get_value('GHBEARER'), timeout=timeout)
|
||||
except Exception as e:
|
||||
print(f'查询 GitHub 状态出错:{e}')
|
||||
return None
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
try:
|
||||
data = resp.json()
|
||||
except ValueError:
|
||||
return None
|
||||
# 只关心终态:在途(queued/in_progress/waiting)返回 None,交给回调与下次核对
|
||||
if data.get('status') != 'completed':
|
||||
return None
|
||||
conclusion = (data.get('conclusion') or 'failure').lower()
|
||||
return RunInfo(
|
||||
finished=True,
|
||||
status=_GH_CONCLUSION_MAP.get(conclusion, 'failure'),
|
||||
html_url=data.get('html_url') or self.log_url(run_id),
|
||||
)
|
||||
|
||||
# ---- 日志链接 ----
|
||||
def log_url(self, run_id):
|
||||
if not run_id:
|
||||
|
||||
+91
-2
@@ -2,6 +2,8 @@ import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
@@ -33,8 +35,93 @@ from .models import GithubRun, STATUS_LABELS, STATUS_BADGE
|
||||
# 终态状态集合
|
||||
TERMINAL_STATUSES = ('success', 'failure', 'cancelled', 'timed_out', 'skipped')
|
||||
# 正常构建约 30~45 分钟;超过该时长仍在进行中的,视为 runner 离线/作业僵死。
|
||||
# 状态以构建脚本结束回调为准,页面不再向构建平台主动查询;超时兜底只做本地判断。
|
||||
# 状态以构建脚本结束回调为第一来源;回调丢失时由 get_run 只读核对平台终态兜底,
|
||||
# 超时判断只做本地兜底。
|
||||
STALE_BUILD_HOURS = 6
|
||||
# 主动核对节流:同一在途任务两次平台查询的最小间隔,避免轮询/刷页面打满 API
|
||||
REFRESH_THROTTLE_SECONDS = 20
|
||||
# 平台查询短超时,平台抖动时等待页轮询最多多等这几秒,不拖垮页面
|
||||
REFRESH_TIMEOUT = 6
|
||||
_refresh_stamps = {}
|
||||
_refresh_lock = threading.Lock()
|
||||
|
||||
|
||||
def _claim_refresh_slot(run_pk):
|
||||
"""节流:到间隔才占位成功;先占位再请求,避免并发页面/线程重复打平台。"""
|
||||
now = time.monotonic()
|
||||
with _refresh_lock:
|
||||
if now - _refresh_stamps.get(run_pk, 0) >= REFRESH_THROTTLE_SECONDS:
|
||||
_refresh_stamps[run_pk] = now
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _apply_remote_terminal(gh_run, info):
|
||||
"""平台明确终态且本地仍在途时落库(条件更新,避免覆盖刚到达的回调)。返回是否变化。"""
|
||||
if info is None or not info.finished:
|
||||
return False
|
||||
new_status = info.status
|
||||
if new_status not in TERMINAL_STATUSES or new_status == gh_run.status:
|
||||
return False
|
||||
updated = GithubRun.objects.filter(pk=gh_run.pk) \
|
||||
.exclude(status__in=TERMINAL_STATUSES) \
|
||||
.update(status=new_status, updated_at=timezone.now())
|
||||
if updated:
|
||||
gh_run.status = new_status
|
||||
return bool(updated)
|
||||
|
||||
|
||||
def refresh_active_run(gh_run):
|
||||
"""同步向构建平台核对单个在途任务(等待页 30 秒轮询 / 状态接口用)。
|
||||
|
||||
20 秒节流、6 秒超时;任何网络异常静默。仅当平台报告终态时落库,
|
||||
工作流结束回调仍是第一状态来源,本核对只兜底回调丢失。
|
||||
"""
|
||||
if gh_run.is_finished() or not gh_run.github_run_id:
|
||||
return False
|
||||
if not _claim_refresh_slot(gh_run.pk):
|
||||
return False
|
||||
try:
|
||||
backend = build_backends.get_backend(gh_run.backend or 'github')
|
||||
info = backend.get_run(gh_run.github_run_id, timeout=REFRESH_TIMEOUT)
|
||||
except Exception as e:
|
||||
print(f"核对构建状态出错:{e}")
|
||||
return False
|
||||
return _apply_remote_terminal(gh_run, info)
|
||||
|
||||
|
||||
def _refresh_runs_in_background(run_pks):
|
||||
"""后台线程体:逐个核对终态;与请求/渲染完全解耦,异常不影响任何页面。"""
|
||||
try:
|
||||
for pk in run_pks:
|
||||
gh_run = GithubRun.objects.filter(pk=pk).first()
|
||||
if gh_run is None or gh_run.is_finished() or not gh_run.github_run_id:
|
||||
continue
|
||||
try:
|
||||
backend = build_backends.get_backend(gh_run.backend or 'github')
|
||||
info = backend.get_run(gh_run.github_run_id, timeout=REFRESH_TIMEOUT)
|
||||
_apply_remote_terminal(gh_run, info)
|
||||
except Exception as e:
|
||||
print(f"后台核对构建状态出错:{e}")
|
||||
finally:
|
||||
# 子线程使用独立数据库连接,结束时必须关闭
|
||||
from django.db import connections
|
||||
connections.close_all()
|
||||
|
||||
|
||||
def schedule_refresh_for_runs(gh_runs):
|
||||
"""列表/仪表盘用:为在途任务安排后台核对,页面渲染零等待。
|
||||
|
||||
与等待页共用 20 秒节流;无在途任务或刚核对过时不创建线程。
|
||||
"""
|
||||
due_pks = [
|
||||
r.pk for r in gh_runs
|
||||
if not r.is_finished() and r.github_run_id and _claim_refresh_slot(r.pk)
|
||||
]
|
||||
if not due_pks:
|
||||
return
|
||||
threading.Thread(
|
||||
target=_refresh_runs_in_background, args=(due_pks,), daemon=True).start()
|
||||
|
||||
|
||||
def mark_stale_builds(gh_runs=None):
|
||||
@@ -422,7 +509,9 @@ def _get_run_status(uuid_val):
|
||||
backend = build_backends.get_backend(gh_run.backend or 'github')
|
||||
github_log_url = backend.log_url(gh_run.github_run_id) if gh_run.github_run_id else ''
|
||||
|
||||
# 状态以构建脚本结束回调为准;这里只做纯本地的 6 小时超时兜底,不请求构建平台
|
||||
# 状态以构建脚本结束回调为准;此处先做平台终态只读核对(20 秒节流、6 秒超时,
|
||||
# 回调丢失时能及时发现平台侧已失败/取消),再做纯本地的 6 小时超时兜底
|
||||
refresh_active_run(gh_run)
|
||||
mark_run_stale_if_needed(gh_run)
|
||||
|
||||
return {
|
||||
|
||||
@@ -32,9 +32,10 @@ def staff_required(view_func):
|
||||
@login_required
|
||||
def my_builds(request):
|
||||
builds = list(GithubRun.objects.filter(created_by=request.user))
|
||||
# 仅做纯本地的 6 小时超时兜底,不向构建平台发起任何请求,页面秒开
|
||||
from .views import mark_stale_builds
|
||||
# 回调为第一状态来源;本地 6 小时超时兜底 + 后台线程只读核对平台终态(不阻塞页面)
|
||||
from .views import mark_stale_builds, schedule_refresh_for_runs
|
||||
mark_stale_builds(builds)
|
||||
schedule_refresh_for_runs(builds)
|
||||
return render(request, 'my_builds.html', {'builds': builds})
|
||||
|
||||
|
||||
@@ -149,9 +150,10 @@ def delete_token(request, key):
|
||||
@staff_required
|
||||
def dashboard(request):
|
||||
builds = list(GithubRun.objects.select_related('created_by').all()[:100])
|
||||
# 仅做纯本地的 6 小时超时兜底,不向构建平台发起任何请求,页面秒开
|
||||
from .views import mark_stale_builds
|
||||
# 回调为第一状态来源;本地 6 小时超时兜底 + 后台线程只读核对平台终态(不阻塞页面)
|
||||
from .views import mark_stale_builds, schedule_refresh_for_runs
|
||||
mark_stale_builds(builds)
|
||||
schedule_refresh_for_runs(builds)
|
||||
stats = {
|
||||
'total': GithubRun.objects.count(),
|
||||
'in_progress': GithubRun.objects.exclude(
|
||||
|
||||
Reference in New Issue
Block a user