"""上传解析任务的进度:排队、解析比例、预计剩余时间 任务只存在进程内存里:单实例部署,重启后进行中的任务随进程丢失,原文件目录由过期清理删除 所有方法都在事件循环里调用;解析子进程的进度由上传流程用 call_soon_threadsafe 转进来 """ import time from collections.abc import Callable from dataclasses import dataclass from itertools import count from app.schemas.data_file import UploadProgress, UploadStage, UploadStatus # 失败的任务保留这么久,刷新页面后还能看到失败原因;成功的任务不保留,文件已经有记录 FAILED_RETENTION = 600 # 解析中的进度不到 1,1 只留给已完成 MAX_PARSING_PROGRESS = 0.99 # 进度太小或解析刚开始时估不准,不给预计时间 ETA_MIN_PROGRESS = 0.05 ETA_MIN_ELAPSED = 1.0 # 预计时间的平滑系数:新的估算占多少,其余沿用上一次,免得数字来回跳 ETA_SMOOTHING = 0.3 @dataclass class UploadJob: file_id: str owner_id: int conversation_id: str name: str seq: int status: UploadStatus = "queued" stage: UploadStage | None = None progress: float = 0.0 error: tuple[int, str] | None = None # 以下时间都来自 UploadBoard 的时钟 started_at: float | None = None failed_at: float | None = None eta: float | None = None eta_at: float = 0.0 @property def active(self) -> bool: return self.status in ("queued", "parsing") class UploadBoard: def __init__(self, clock: Callable[[], float] = time.monotonic): self._clock = clock self._jobs: dict[str, UploadJob] = {} self._numbers = count() def create( self, file_id: str, owner_id: int, conversation_id: str, name: str ) -> UploadJob: job = UploadJob(file_id, owner_id, conversation_id, name, next(self._numbers)) self._jobs[file_id] = job return job def active_count(self, conversation_id: str) -> int: """会话里排队和解析中的任务数,它们都占着会话内的文件数名额""" return sum( job.active and job.conversation_id == conversation_id for job in self._jobs.values() ) def start(self, job: UploadJob) -> None: job.status = "parsing" job.started_at = self._clock() def report(self, job: UploadJob, stage: UploadStage, value: float) -> None: """解析子进程报告进度:只增不减,预计时间按已用时间和进度外推后平滑""" if job.status != "parsing" or job.started_at is None: return now = self._clock() job.stage = stage job.progress = min(max(value, job.progress), MAX_PARSING_PROGRESS) elapsed = now - job.started_at if job.progress < ETA_MIN_PROGRESS or elapsed < ETA_MIN_ELAPSED: return raw = elapsed * (1 - job.progress) / job.progress previous = self._eta(job, now) job.eta = ( raw if previous is None else ETA_SMOOTHING * raw + (1 - ETA_SMOOTHING) * previous ) job.eta_at = now def fail(self, job: UploadJob, code: int, message: str) -> None: job.status = "failed" job.stage = None job.error = (code, message) job.failed_at = self._clock() def remove(self, job: UploadJob) -> None: self._jobs.pop(job.file_id, None) def get(self, file_id: str) -> UploadJob | None: self._purge() return self._jobs.get(file_id) def for_conversation(self, owner_id: int, conversation_id: str) -> list[UploadJob]: self._purge() return [ job for job in self._jobs.values() if job.owner_id == owner_id and job.conversation_id == conversation_id ] def view(self, job: UploadJob) -> UploadProgress: return UploadProgress( file_id=job.file_id, name=job.name, status=job.status, progress=round(job.progress, 3), stage=job.stage, eta_seconds=self._eta_seconds(job), queued_ahead=self._queued_ahead(job), error=( None if job.error is None else {"code": job.error[0], "msg": job.error[1]} ), ) def _eta(self, job: UploadJob, now: float) -> float | None: """上次估算之后又过了一段时间,剩余时间相应减少""" if job.eta is None: return None return max(job.eta - (now - job.eta_at), 0.0) def _eta_seconds(self, job: UploadJob) -> float | None: if job.status != "parsing": return None eta = self._eta(job, self._clock()) return None if eta is None else round(eta, 1) def _queued_ahead(self, job: UploadJob) -> int: if job.status != "queued": return 0 return sum( other.status == "queued" and other.seq < job.seq for other in self._jobs.values() ) def _purge(self) -> None: now = self._clock() for job in [ job for job in self._jobs.values() if job.failed_at is not None and now - job.failed_at > FAILED_RETENTION ]: self.remove(job)