构建状态增加平台终态只读核对,修复回调丢失后任务长期卡在构建中
工作流在脚本收尾前失败(拉取/编译/runner 异常)或 GENURL 不可达时, 结束回调会永久丢失,而等待页 30 秒轮询与列表页此前只读本地库, 任务只能等 6 小时本地超时,平台侧已失败也无法及时反映。 - GitHub/Gitea 后端新增只读 get_run(),仅在平台报告终态时返回结论, 在途、非 200、网络异常、坏 JSON 一律返回 None;不取消、不写平台 - 等待页 30 秒轮询同步核对一次真实状态:20 秒节流、6 秒超时、异常静默 - 我的构建/后台仪表盘改为后台守护线程核对,页面渲染零等待 - 仅做非终态到终态的条件更新,回调抢先到达时不会被覆盖 - 停止/删除纯本地语义、6 小时本地超时兜底等现有逻辑保持不变
This commit is contained in:
@@ -18,6 +18,14 @@ class DispatchResult:
|
|||||||
html_url: str = '' # 平台上的运行详情页地址
|
html_url: str = '' # 平台上的运行详情页地址
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class RunInfo:
|
||||||
|
"""平台单次运行的状态快照。"""
|
||||||
|
finished: bool = False # 是否已结束(拿到终态结论)
|
||||||
|
status: str = '' # finished=True 时为 rdgen 内部终态码
|
||||||
|
html_url: str = '' # 平台上的运行详情页地址
|
||||||
|
|
||||||
|
|
||||||
# rdgen 目标平台 -> 工作流文件名(两平台同名)
|
# rdgen 目标平台 -> 工作流文件名(两平台同名)
|
||||||
WORKFLOW_FILES = {
|
WORKFLOW_FILES = {
|
||||||
'windows': 'generator-windows.yml',
|
'windows': 'generator-windows.yml',
|
||||||
@@ -53,6 +61,11 @@ class BuildBackend:
|
|||||||
def log_url(self, run_id) -> str:
|
def log_url(self, run_id) -> str:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def get_run(self, run_id, *, timeout=8):
|
||||||
|
"""只读查询单次运行:已结束返回 RunInfo(finished=True, status=内部终态码);
|
||||||
|
仍在途、查询失败或不支持时返回 None(调用方按「仍在进行」处理)。"""
|
||||||
|
return None
|
||||||
|
|
||||||
# ---- 后台连接测试 ----
|
# ---- 后台连接测试 ----
|
||||||
def test_connection(self) -> dict:
|
def test_connection(self) -> dict:
|
||||||
"""返回 {'ok': bool, 'message': 中文说明},不抛异常。"""
|
"""返回 {'ok': bool, 'message': 中文说明},不抛异常。"""
|
||||||
|
|||||||
@@ -6,11 +6,25 @@ API 与 GitHub Actions 几乎同形(已对照 Gitea 官方 Swagger 核实)
|
|||||||
{workflow_run_id, html_url, run_url},不传该参数为 204;
|
{workflow_run_id, html_url, run_url},不传该参数为 204;
|
||||||
- 鉴权:Authorization: token <PAT>。
|
- 鉴权:Authorization: token <PAT>。
|
||||||
|
|
||||||
注意:本平台不主动查询/取消 Gitea 上的运行。构建状态完全以工作流结束时的
|
注意:本平台不主动取消 Gitea 上的运行,停止/删除只作用于本平台本地记录。
|
||||||
回调(update_github_run)为准,停止/删除只作用于本平台本地记录。
|
构建状态以工作流结束时的回调(update_github_run)为第一来源;get_run 仅做
|
||||||
|
只读终态核对,兜底回调丢失(脚本未跑到收尾或 GENURL 不可达)的情况。
|
||||||
"""
|
"""
|
||||||
from .. import app_settings, ghnet
|
from .. import app_settings, ghnet
|
||||||
from .base import BackendError, DispatchResult, WORKFLOW_FILES, BuildBackend
|
from .base import BackendError, DispatchResult, RunInfo, WORKFLOW_FILES, BuildBackend
|
||||||
|
|
||||||
|
# 部分 Gitea 版本 run.status 直接返回终态(无 completed+conclusion 包裹)
|
||||||
|
_GITEA_TERMINAL = {'success', 'failure', 'cancelled', 'skipped', 'timed_out', 'stopped', 'dead'}
|
||||||
|
# Gitea 结论 -> rdgen 内部终态
|
||||||
|
_GITEA_CONCLUSION_MAP = {
|
||||||
|
'success': 'success',
|
||||||
|
'failure': 'failure',
|
||||||
|
'cancelled': 'cancelled',
|
||||||
|
'skipped': 'skipped',
|
||||||
|
'timed_out': 'timed_out',
|
||||||
|
'stopped': 'cancelled',
|
||||||
|
'dead': 'failure',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class GiteaBackend(BuildBackend):
|
class GiteaBackend(BuildBackend):
|
||||||
@@ -130,6 +144,42 @@ class GiteaBackend(BuildBackend):
|
|||||||
print(f'查询 Gitea 最新运行失败:{e}')
|
print(f'查询 Gitea 最新运行失败:{e}')
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
# ---- 只读状态查询(回调丢失时的终态兜底) ----
|
||||||
|
def get_run(self, run_id, *, timeout=8):
|
||||||
|
url = self._api(f'/repos/{self._owner()}/{self._repo()}/actions/runs/{run_id}')
|
||||||
|
try:
|
||||||
|
resp = ghnet.get(
|
||||||
|
url, token=app_settings.get_value('GITEA_TOKEN'),
|
||||||
|
timeout=timeout, flavor='gitea', proxy=self._proxy(),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
print(f'查询 Gitea 状态出错:{e}')
|
||||||
|
return None
|
||||||
|
if resp.status_code != 200:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
data = resp.json()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
html_url = data.get('html_url') or self.log_url(run_id)
|
||||||
|
status = (data.get('status') or '').lower()
|
||||||
|
conclusion = (data.get('conclusion') or '').lower()
|
||||||
|
if status == 'completed':
|
||||||
|
return RunInfo(
|
||||||
|
finished=True,
|
||||||
|
status=_GITEA_CONCLUSION_MAP.get(conclusion, 'failure'),
|
||||||
|
html_url=html_url,
|
||||||
|
)
|
||||||
|
# 部分 Gitea 版本直接在 status 返回终态(无 completed 包裹)
|
||||||
|
if status in _GITEA_TERMINAL:
|
||||||
|
return RunInfo(
|
||||||
|
finished=True,
|
||||||
|
status=_GITEA_CONCLUSION_MAP.get(status, 'failure'),
|
||||||
|
html_url=html_url,
|
||||||
|
)
|
||||||
|
# queued/in_progress/waiting/blocked 等在途状态返回 None,交给回调与下次核对
|
||||||
|
return None
|
||||||
|
|
||||||
# ---- 日志链接 ----
|
# ---- 日志链接 ----
|
||||||
def log_url(self, run_id):
|
def log_url(self, run_id):
|
||||||
if not run_id:
|
if not run_id:
|
||||||
|
|||||||
@@ -1,13 +1,24 @@
|
|||||||
"""GitHub Actions 构建后端(迁自 views.py / web_views.py 的原有逻辑,行为保持一致)。
|
"""GitHub Actions 构建后端(迁自 views.py / web_views.py 的原有逻辑,行为保持一致)。
|
||||||
|
|
||||||
注意:本平台不主动查询/取消 GitHub 上的运行。构建状态完全以工作流结束时的
|
注意:本平台不主动取消 GitHub 上的运行,停止/删除只作用于本平台本地记录。
|
||||||
回调(update_github_run)为准,停止/删除只作用于本平台本地记录。
|
构建状态以工作流结束时的回调(update_github_run)为第一来源;get_run 仅做
|
||||||
|
只读终态核对,兜底回调丢失(脚本未跑到收尾或 GENURL 不可达)的情况。
|
||||||
"""
|
"""
|
||||||
from .. import app_settings, ghnet
|
from .. import app_settings, ghnet
|
||||||
from .base import BackendError, DispatchResult, WORKFLOW_FILES, BuildBackend
|
from .base import BackendError, DispatchResult, RunInfo, WORKFLOW_FILES, BuildBackend
|
||||||
|
|
||||||
API = 'https://api.github.com'
|
API = 'https://api.github.com'
|
||||||
|
|
||||||
|
# GitHub conclusion -> rdgen 内部终态;无人值守构建不会有人处理审批,
|
||||||
|
# action_required/neutral/stale/startup_failure 等异常结论一律按失败呈现
|
||||||
|
_GH_CONCLUSION_MAP = {
|
||||||
|
'success': 'success',
|
||||||
|
'failure': 'failure',
|
||||||
|
'cancelled': 'cancelled',
|
||||||
|
'skipped': 'skipped',
|
||||||
|
'timed_out': 'timed_out',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class GitHubBackend(BuildBackend):
|
class GitHubBackend(BuildBackend):
|
||||||
name = 'github'
|
name = 'github'
|
||||||
@@ -57,6 +68,32 @@ class GitHubBackend(BuildBackend):
|
|||||||
pass
|
pass
|
||||||
return DispatchResult(run_id=data.get('workflow_run_id'), html_url=data.get('html_url') or '')
|
return DispatchResult(run_id=data.get('workflow_run_id'), html_url=data.get('html_url') or '')
|
||||||
|
|
||||||
|
# ---- 只读状态查询(回调丢失时的终态兜底) ----
|
||||||
|
def get_run(self, run_id, *, timeout=8):
|
||||||
|
user = app_settings.get_value('GHUSER')
|
||||||
|
repo = app_settings.get_value('REPONAME')
|
||||||
|
url = f'{API}/repos/{user}/{repo}/actions/runs/{run_id}'
|
||||||
|
try:
|
||||||
|
resp = ghnet.get(url, token=app_settings.get_value('GHBEARER'), timeout=timeout)
|
||||||
|
except Exception as e:
|
||||||
|
print(f'查询 GitHub 状态出错:{e}')
|
||||||
|
return None
|
||||||
|
if resp.status_code != 200:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
data = resp.json()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
# 只关心终态:在途(queued/in_progress/waiting)返回 None,交给回调与下次核对
|
||||||
|
if data.get('status') != 'completed':
|
||||||
|
return None
|
||||||
|
conclusion = (data.get('conclusion') or 'failure').lower()
|
||||||
|
return RunInfo(
|
||||||
|
finished=True,
|
||||||
|
status=_GH_CONCLUSION_MAP.get(conclusion, 'failure'),
|
||||||
|
html_url=data.get('html_url') or self.log_url(run_id),
|
||||||
|
)
|
||||||
|
|
||||||
# ---- 日志链接 ----
|
# ---- 日志链接 ----
|
||||||
def log_url(self, run_id):
|
def log_url(self, run_id):
|
||||||
if not run_id:
|
if not run_id:
|
||||||
|
|||||||
+91
-2
@@ -2,6 +2,8 @@ import io
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -33,8 +35,93 @@ from .models import GithubRun, STATUS_LABELS, STATUS_BADGE
|
|||||||
# 终态状态集合
|
# 终态状态集合
|
||||||
TERMINAL_STATUSES = ('success', 'failure', 'cancelled', 'timed_out', 'skipped')
|
TERMINAL_STATUSES = ('success', 'failure', 'cancelled', 'timed_out', 'skipped')
|
||||||
# 正常构建约 30~45 分钟;超过该时长仍在进行中的,视为 runner 离线/作业僵死。
|
# 正常构建约 30~45 分钟;超过该时长仍在进行中的,视为 runner 离线/作业僵死。
|
||||||
# 状态以构建脚本结束回调为准,页面不再向构建平台主动查询;超时兜底只做本地判断。
|
# 状态以构建脚本结束回调为第一来源;回调丢失时由 get_run 只读核对平台终态兜底,
|
||||||
|
# 超时判断只做本地兜底。
|
||||||
STALE_BUILD_HOURS = 6
|
STALE_BUILD_HOURS = 6
|
||||||
|
# 主动核对节流:同一在途任务两次平台查询的最小间隔,避免轮询/刷页面打满 API
|
||||||
|
REFRESH_THROTTLE_SECONDS = 20
|
||||||
|
# 平台查询短超时,平台抖动时等待页轮询最多多等这几秒,不拖垮页面
|
||||||
|
REFRESH_TIMEOUT = 6
|
||||||
|
_refresh_stamps = {}
|
||||||
|
_refresh_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _claim_refresh_slot(run_pk):
|
||||||
|
"""节流:到间隔才占位成功;先占位再请求,避免并发页面/线程重复打平台。"""
|
||||||
|
now = time.monotonic()
|
||||||
|
with _refresh_lock:
|
||||||
|
if now - _refresh_stamps.get(run_pk, 0) >= REFRESH_THROTTLE_SECONDS:
|
||||||
|
_refresh_stamps[run_pk] = now
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_remote_terminal(gh_run, info):
|
||||||
|
"""平台明确终态且本地仍在途时落库(条件更新,避免覆盖刚到达的回调)。返回是否变化。"""
|
||||||
|
if info is None or not info.finished:
|
||||||
|
return False
|
||||||
|
new_status = info.status
|
||||||
|
if new_status not in TERMINAL_STATUSES or new_status == gh_run.status:
|
||||||
|
return False
|
||||||
|
updated = GithubRun.objects.filter(pk=gh_run.pk) \
|
||||||
|
.exclude(status__in=TERMINAL_STATUSES) \
|
||||||
|
.update(status=new_status, updated_at=timezone.now())
|
||||||
|
if updated:
|
||||||
|
gh_run.status = new_status
|
||||||
|
return bool(updated)
|
||||||
|
|
||||||
|
|
||||||
|
def refresh_active_run(gh_run):
|
||||||
|
"""同步向构建平台核对单个在途任务(等待页 30 秒轮询 / 状态接口用)。
|
||||||
|
|
||||||
|
20 秒节流、6 秒超时;任何网络异常静默。仅当平台报告终态时落库,
|
||||||
|
工作流结束回调仍是第一状态来源,本核对只兜底回调丢失。
|
||||||
|
"""
|
||||||
|
if gh_run.is_finished() or not gh_run.github_run_id:
|
||||||
|
return False
|
||||||
|
if not _claim_refresh_slot(gh_run.pk):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
backend = build_backends.get_backend(gh_run.backend or 'github')
|
||||||
|
info = backend.get_run(gh_run.github_run_id, timeout=REFRESH_TIMEOUT)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"核对构建状态出错:{e}")
|
||||||
|
return False
|
||||||
|
return _apply_remote_terminal(gh_run, info)
|
||||||
|
|
||||||
|
|
||||||
|
def _refresh_runs_in_background(run_pks):
|
||||||
|
"""后台线程体:逐个核对终态;与请求/渲染完全解耦,异常不影响任何页面。"""
|
||||||
|
try:
|
||||||
|
for pk in run_pks:
|
||||||
|
gh_run = GithubRun.objects.filter(pk=pk).first()
|
||||||
|
if gh_run is None or gh_run.is_finished() or not gh_run.github_run_id:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
backend = build_backends.get_backend(gh_run.backend or 'github')
|
||||||
|
info = backend.get_run(gh_run.github_run_id, timeout=REFRESH_TIMEOUT)
|
||||||
|
_apply_remote_terminal(gh_run, info)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"后台核对构建状态出错:{e}")
|
||||||
|
finally:
|
||||||
|
# 子线程使用独立数据库连接,结束时必须关闭
|
||||||
|
from django.db import connections
|
||||||
|
connections.close_all()
|
||||||
|
|
||||||
|
|
||||||
|
def schedule_refresh_for_runs(gh_runs):
|
||||||
|
"""列表/仪表盘用:为在途任务安排后台核对,页面渲染零等待。
|
||||||
|
|
||||||
|
与等待页共用 20 秒节流;无在途任务或刚核对过时不创建线程。
|
||||||
|
"""
|
||||||
|
due_pks = [
|
||||||
|
r.pk for r in gh_runs
|
||||||
|
if not r.is_finished() and r.github_run_id and _claim_refresh_slot(r.pk)
|
||||||
|
]
|
||||||
|
if not due_pks:
|
||||||
|
return
|
||||||
|
threading.Thread(
|
||||||
|
target=_refresh_runs_in_background, args=(due_pks,), daemon=True).start()
|
||||||
|
|
||||||
|
|
||||||
def mark_stale_builds(gh_runs=None):
|
def mark_stale_builds(gh_runs=None):
|
||||||
@@ -422,7 +509,9 @@ def _get_run_status(uuid_val):
|
|||||||
backend = build_backends.get_backend(gh_run.backend or 'github')
|
backend = build_backends.get_backend(gh_run.backend or 'github')
|
||||||
github_log_url = backend.log_url(gh_run.github_run_id) if gh_run.github_run_id else ''
|
github_log_url = backend.log_url(gh_run.github_run_id) if gh_run.github_run_id else ''
|
||||||
|
|
||||||
# 状态以构建脚本结束回调为准;这里只做纯本地的 6 小时超时兜底,不请求构建平台
|
# 状态以构建脚本结束回调为准;此处先做平台终态只读核对(20 秒节流、6 秒超时,
|
||||||
|
# 回调丢失时能及时发现平台侧已失败/取消),再做纯本地的 6 小时超时兜底
|
||||||
|
refresh_active_run(gh_run)
|
||||||
mark_run_stale_if_needed(gh_run)
|
mark_run_stale_if_needed(gh_run)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -32,9 +32,10 @@ def staff_required(view_func):
|
|||||||
@login_required
|
@login_required
|
||||||
def my_builds(request):
|
def my_builds(request):
|
||||||
builds = list(GithubRun.objects.filter(created_by=request.user))
|
builds = list(GithubRun.objects.filter(created_by=request.user))
|
||||||
# 仅做纯本地的 6 小时超时兜底,不向构建平台发起任何请求,页面秒开
|
# 回调为第一状态来源;本地 6 小时超时兜底 + 后台线程只读核对平台终态(不阻塞页面)
|
||||||
from .views import mark_stale_builds
|
from .views import mark_stale_builds, schedule_refresh_for_runs
|
||||||
mark_stale_builds(builds)
|
mark_stale_builds(builds)
|
||||||
|
schedule_refresh_for_runs(builds)
|
||||||
return render(request, 'my_builds.html', {'builds': builds})
|
return render(request, 'my_builds.html', {'builds': builds})
|
||||||
|
|
||||||
|
|
||||||
@@ -149,9 +150,10 @@ def delete_token(request, key):
|
|||||||
@staff_required
|
@staff_required
|
||||||
def dashboard(request):
|
def dashboard(request):
|
||||||
builds = list(GithubRun.objects.select_related('created_by').all()[:100])
|
builds = list(GithubRun.objects.select_related('created_by').all()[:100])
|
||||||
# 仅做纯本地的 6 小时超时兜底,不向构建平台发起任何请求,页面秒开
|
# 回调为第一状态来源;本地 6 小时超时兜底 + 后台线程只读核对平台终态(不阻塞页面)
|
||||||
from .views import mark_stale_builds
|
from .views import mark_stale_builds, schedule_refresh_for_runs
|
||||||
mark_stale_builds(builds)
|
mark_stale_builds(builds)
|
||||||
|
schedule_refresh_for_runs(builds)
|
||||||
stats = {
|
stats = {
|
||||||
'total': GithubRun.objects.count(),
|
'total': GithubRun.objects.count(),
|
||||||
'in_progress': GithubRun.objects.exclude(
|
'in_progress': GithubRun.objects.exclude(
|
||||||
|
|||||||
Reference in New Issue
Block a user