feat:新增大屏存取发布接口
This commit is contained in:
@@ -21,7 +21,7 @@ def make_title(prompt: str) -> str:
|
||||
class ConversationStore(Protocol):
|
||||
"""内存和 MongoDB 实现同一组方法,业务代码不需要关心具体存储"""
|
||||
|
||||
async def create(self, user_id: int) -> str: ...
|
||||
async def create(self, user_id: int, screen_id: str | None = None) -> str: ...
|
||||
|
||||
async def exists(self, conversation_id: str, user_id: int) -> bool:
|
||||
"""会话已签发、未过期且属于该用户;不属于时与不存在同样返回 False"""
|
||||
@@ -34,9 +34,21 @@ class ConversationStore(Protocol):
|
||||
async def save_turn(self, turn: Turn) -> None: ...
|
||||
|
||||
async def list_by_user(
|
||||
self, user_id: int, limit: int, offset: int
|
||||
self, user_id: int, limit: int, offset: int, screen_id: str | None = None
|
||||
) -> list[ConversationSummary]:
|
||||
"""该用户已产生对话的会话,按最近更新时间倒序;没有对话的空会话不返回"""
|
||||
"""该用户已产生对话的会话,按最近更新时间倒序;没有对话的空会话不返回;给了 screen_id 只返回该大屏的"""
|
||||
...
|
||||
|
||||
async def count_by_user(self, user_id: int, screen_id: str | None = None) -> int:
|
||||
"""与 list_by_user 同一口径的总数"""
|
||||
...
|
||||
|
||||
async def rename(self, conversation_id: str, user_id: int, title: str) -> bool:
|
||||
"""会话不存在或不属于该用户返回 False"""
|
||||
...
|
||||
|
||||
async def delete(self, conversation_id: str, user_id: int) -> bool:
|
||||
"""连同对话轮次一起删除;会话不存在或不属于该用户返回 False"""
|
||||
...
|
||||
|
||||
|
||||
@@ -51,6 +63,8 @@ class InMemoryConversationStore:
|
||||
# 会话 ID -> 创建时间;轮次按 thread_id 去重并保持首次写入顺序
|
||||
self._created: dict[str, datetime] = {}
|
||||
self._owner: dict[str, int] = {}
|
||||
self._screen: dict[str, str | None] = {}
|
||||
self._titles: dict[str, str] = {}
|
||||
self._turns: dict[str, dict[str, Turn]] = {}
|
||||
|
||||
def _purge_expired(self) -> None:
|
||||
@@ -61,15 +75,18 @@ class InMemoryConversationStore:
|
||||
for key, created in self._created.items()
|
||||
if created < deadline and not self._turns.get(key)
|
||||
]:
|
||||
del self._created[conversation_id]
|
||||
self._owner.pop(conversation_id, None)
|
||||
self._turns.pop(conversation_id, None)
|
||||
self._forget(conversation_id)
|
||||
|
||||
async def create(self, user_id: int) -> str:
|
||||
def _forget(self, conversation_id: str) -> None:
|
||||
for table in (self._created, self._owner, self._screen, self._titles, self._turns):
|
||||
table.pop(conversation_id, None)
|
||||
|
||||
async def create(self, user_id: int, screen_id: str | None = None) -> str:
|
||||
self._purge_expired()
|
||||
conversation_id = str(uuid4())
|
||||
self._created[conversation_id] = _now()
|
||||
self._owner[conversation_id] = user_id
|
||||
self._screen[conversation_id] = screen_id
|
||||
return conversation_id
|
||||
|
||||
async def exists(self, conversation_id: str, user_id: int) -> bool:
|
||||
@@ -94,8 +111,8 @@ class InMemoryConversationStore:
|
||||
turn = turn.model_copy(update={"created_at": existing.created_at})
|
||||
self._turns[turn.conversation_id][turn.thread_id] = turn
|
||||
|
||||
async def list_by_user(
|
||||
self, user_id: int, limit: int, offset: int
|
||||
def _summaries(
|
||||
self, user_id: int, screen_id: str | None
|
||||
) -> list[ConversationSummary]:
|
||||
summaries = []
|
||||
for conversation_id, owner in self._owner.items():
|
||||
@@ -105,14 +122,37 @@ class InMemoryConversationStore:
|
||||
)
|
||||
if owner != user_id or not turns:
|
||||
continue
|
||||
if screen_id is not None and self._screen.get(conversation_id) != screen_id:
|
||||
continue
|
||||
summaries.append(
|
||||
ConversationSummary(
|
||||
conversation_id=conversation_id,
|
||||
title=make_title(turns[0].prompt),
|
||||
title=self._titles.get(conversation_id) or make_title(turns[0].prompt),
|
||||
screen_id=self._screen.get(conversation_id),
|
||||
updated_at=turns[-1].created_at,
|
||||
)
|
||||
)
|
||||
summaries.sort(
|
||||
key=lambda item: (item.updated_at, item.conversation_id), reverse=True
|
||||
)
|
||||
return summaries[offset : offset + limit]
|
||||
return summaries
|
||||
|
||||
async def list_by_user(
|
||||
self, user_id: int, limit: int, offset: int, screen_id: str | None = None
|
||||
) -> list[ConversationSummary]:
|
||||
return self._summaries(user_id, screen_id)[offset : offset + limit]
|
||||
|
||||
async def count_by_user(self, user_id: int, screen_id: str | None = None) -> int:
|
||||
return len(self._summaries(user_id, screen_id))
|
||||
|
||||
async def rename(self, conversation_id: str, user_id: int, title: str) -> bool:
|
||||
if not await self.exists(conversation_id, user_id):
|
||||
return False
|
||||
self._titles[conversation_id] = title
|
||||
return True
|
||||
|
||||
async def delete(self, conversation_id: str, user_id: int) -> bool:
|
||||
if not await self.exists(conversation_id, user_id):
|
||||
return False
|
||||
self._forget(conversation_id)
|
||||
return True
|
||||
|
||||
@@ -45,6 +45,8 @@ class MongoConversationStore:
|
||||
await self._conversations.create_index("created_at")
|
||||
# 按用户查询会话
|
||||
await self._conversations.create_index("user_id")
|
||||
# 按大屏过滤会话
|
||||
await self._conversations.create_index([("user_id", ASCENDING), ("screen_id", ASCENDING)])
|
||||
|
||||
async def _purge_expired(self) -> None:
|
||||
# 只清理没有任何轮次且超过保留时长的空会话,有对话的会话不受影响
|
||||
@@ -66,12 +68,17 @@ class MongoConversationStore:
|
||||
if empty:
|
||||
await self._conversations.delete_many({"_id": {"$in": empty}})
|
||||
|
||||
async def create(self, user_id: int) -> str:
|
||||
async def create(self, user_id: int, screen_id: str | None = None) -> str:
|
||||
await self._purge_expired()
|
||||
conversation_id = str(uuid4())
|
||||
# user_id 来自用户中心,会话只对创建者可见
|
||||
await self._conversations.insert_one(
|
||||
{"_id": conversation_id, "created_at": _now(), "user_id": user_id}
|
||||
{
|
||||
"_id": conversation_id,
|
||||
"created_at": _now(),
|
||||
"user_id": user_id,
|
||||
"screen_id": screen_id,
|
||||
}
|
||||
)
|
||||
return conversation_id
|
||||
|
||||
@@ -121,12 +128,13 @@ class MongoConversationStore:
|
||||
upsert=True,
|
||||
)
|
||||
|
||||
async def list_by_user(
|
||||
self, user_id: int, limit: int, offset: int
|
||||
) -> list[ConversationSummary]:
|
||||
def _with_stats(self, user_id: int, screen_id: str | None) -> list[dict]:
|
||||
match: dict = {"user_id": user_id}
|
||||
if screen_id is not None:
|
||||
match["screen_id"] = screen_id
|
||||
# $unwind 会丢弃没有任何轮次的空会话
|
||||
pipeline = [
|
||||
{"$match": {"user_id": user_id}},
|
||||
return [
|
||||
{"$match": match},
|
||||
{
|
||||
"$lookup": {
|
||||
"from": self._turns.name,
|
||||
@@ -146,6 +154,13 @@ class MongoConversationStore:
|
||||
}
|
||||
},
|
||||
{"$unwind": "$stats"},
|
||||
]
|
||||
|
||||
async def list_by_user(
|
||||
self, user_id: int, limit: int, offset: int, screen_id: str | None = None
|
||||
) -> list[ConversationSummary]:
|
||||
pipeline = [
|
||||
*self._with_stats(user_id, screen_id),
|
||||
{"$sort": {"stats.updated_at": -1, "_id": -1}},
|
||||
{"$skip": offset},
|
||||
{"$limit": limit},
|
||||
@@ -154,8 +169,30 @@ class MongoConversationStore:
|
||||
return [
|
||||
ConversationSummary(
|
||||
conversation_id=item["_id"],
|
||||
title=make_title(item["stats"]["first_prompt"]),
|
||||
title=item.get("title") or make_title(item["stats"]["first_prompt"]),
|
||||
screen_id=item.get("screen_id"),
|
||||
updated_at=item["stats"]["updated_at"],
|
||||
)
|
||||
for item in await cursor.to_list()
|
||||
]
|
||||
|
||||
async def count_by_user(self, user_id: int, screen_id: str | None = None) -> int:
|
||||
pipeline = [*self._with_stats(user_id, screen_id), {"$count": "total"}]
|
||||
cursor = await self._conversations.aggregate(pipeline)
|
||||
counted = await cursor.to_list()
|
||||
return counted[0]["total"] if counted else 0
|
||||
|
||||
async def rename(self, conversation_id: str, user_id: int, title: str) -> bool:
|
||||
if not await self.exists(conversation_id, user_id):
|
||||
return False
|
||||
await self._conversations.update_one(
|
||||
{"_id": conversation_id, "user_id": user_id}, {"$set": {"title": title}}
|
||||
)
|
||||
return True
|
||||
|
||||
async def delete(self, conversation_id: str, user_id: int) -> bool:
|
||||
if not await self.exists(conversation_id, user_id):
|
||||
return False
|
||||
await self._turns.delete_many({"conversation_id": conversation_id})
|
||||
await self._conversations.delete_one({"_id": conversation_id, "user_id": user_id})
|
||||
return True
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
"""数据源代理:服务端替浏览器请求第三方接口
|
||||
|
||||
两个入口共用 fetch_json:编辑态由登录用户传 URL;线上态只传数据源 ID,配置取自已发布快照
|
||||
防护:只允许 http/https;DNS 解析后每个地址都必须是公网地址,并把连接钉在已检查过的地址上
|
||||
(不再让 httpx 二次解析,防 DNS 重绑定);不读环境代理;重定向逐跳重新检查,跨域时丢掉调用方的密钥头;
|
||||
总超时 5 秒,响应体上限 2 MB,按解压后的字节计,解压时带输出上限(防压缩炸弹)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
import zlib
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.exceptions import BizException
|
||||
|
||||
TIMEOUT = 5.0
|
||||
MAX_BYTES = 2 * 1024 * 1024
|
||||
MAX_REDIRECTS = 3
|
||||
REDIRECT_STATUS = {301, 302, 303, 307, 308}
|
||||
# 这些头由 httpx 或连接本身决定,不让调用方覆盖
|
||||
FORBIDDEN_HEADERS = {
|
||||
"host",
|
||||
"content-length",
|
||||
"connection",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
"te",
|
||||
"keep-alive",
|
||||
"proxy-authorization",
|
||||
"proxy-connection",
|
||||
"accept-encoding",
|
||||
}
|
||||
# 只接受自己能带上限解压的编码;httpx 默认还会协商 br / zstd,那些解码器没有输出上限
|
||||
ACCEPT_ENCODING = "gzip, deflate"
|
||||
# is_global 会放过的几段:NAT64(可以嵌入任意 IPv4)、IPv4 兼容地址、已废弃的站点本地地址
|
||||
EXTRA_BLOCKED = [
|
||||
ipaddress.ip_network(net)
|
||||
for net in ("64:ff9b::/96", "64:ff9b:1::/48", "::/96", "fec0::/10")
|
||||
]
|
||||
# 十进制、十六进制、省略写法的 IPv4(2130706433、0x7f.1、127.1):不是标准写法,
|
||||
# 各平台的解析器解释不一,一律当作内网地址拒绝,不交给 DNS
|
||||
NUMERIC_LABEL = re.compile(r"0x[0-9a-f]+|[0-9]+", re.IGNORECASE)
|
||||
|
||||
Method = Literal["GET", "POST"]
|
||||
|
||||
|
||||
def _blocked() -> BizException:
|
||||
return BizException(422, "不允许访问内网或本机地址")
|
||||
|
||||
|
||||
def _is_public(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
||||
# IPv4 映射的 IPv6 地址按内嵌的 IPv4 判断
|
||||
if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None:
|
||||
address = address.ipv4_mapped
|
||||
return (
|
||||
address.is_global
|
||||
and not address.is_multicast
|
||||
and not any(address in net for net in EXTRA_BLOCKED)
|
||||
)
|
||||
|
||||
|
||||
def _checked(
|
||||
address: ipaddress.IPv4Address | ipaddress.IPv6Address,
|
||||
) -> ipaddress.IPv4Address | ipaddress.IPv6Address:
|
||||
if not _is_public(address):
|
||||
raise _blocked()
|
||||
return address
|
||||
|
||||
|
||||
async def _resolve(host: str, port: int) -> str:
|
||||
"""解析主机名,所有地址都是公网才放行,返回第一个地址"""
|
||||
host = host.rstrip(".")
|
||||
try:
|
||||
literal = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
return str(_checked(literal))
|
||||
if host.lower() == "localhost" or host.lower().endswith(".localhost"):
|
||||
raise _blocked()
|
||||
if all(NUMERIC_LABEL.fullmatch(label) for label in host.split(".")):
|
||||
raise _blocked()
|
||||
try:
|
||||
infos = await asyncio.get_running_loop().getaddrinfo(
|
||||
host, port, type=socket.SOCK_STREAM
|
||||
)
|
||||
except socket.gaierror:
|
||||
raise BizException(502, "数据源域名无法解析") from None
|
||||
addresses = [ipaddress.ip_address(info[4][0].split("%")[0]) for info in infos]
|
||||
if not addresses:
|
||||
raise BizException(502, "数据源域名无法解析")
|
||||
for address in addresses:
|
||||
_checked(address)
|
||||
return str(addresses[0])
|
||||
|
||||
|
||||
def _clean_headers(headers: dict[str, Any] | None) -> dict[str, str]:
|
||||
cleaned = {"Accept": "application/json", "Accept-Encoding": ACCEPT_ENCODING}
|
||||
for name, value in (headers or {}).items():
|
||||
if name.lower() in FORBIDDEN_HEADERS or not isinstance(
|
||||
value, (str, int, float)
|
||||
):
|
||||
continue
|
||||
cleaned[name] = str(value)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _query_value(value: Any) -> Any:
|
||||
return value if isinstance(value, (str, int, float, bool)) else json.dumps(value)
|
||||
|
||||
|
||||
def _parse_url(raw: str) -> httpx.URL:
|
||||
try:
|
||||
url = httpx.URL(raw)
|
||||
except httpx.InvalidURL:
|
||||
raise BizException(422, "数据源地址不合法") from None
|
||||
if url.scheme not in ("http", "https") or not url.host or url.userinfo:
|
||||
raise BizException(422, "数据源地址只支持 http / https,且不能带账号密码")
|
||||
return url
|
||||
|
||||
|
||||
def _too_large() -> BizException:
|
||||
return BizException(502, "数据源响应超过 2 MB")
|
||||
|
||||
|
||||
async def _read_body(response: httpx.Response) -> bytes:
|
||||
"""读响应体:原始字节和解压后的字节都不超过 MAX_BYTES
|
||||
|
||||
不用 aiter_bytes:它对每个 64 KB 的块整块解压,没有输出上限,一个块就能膨胀到上 GB
|
||||
"""
|
||||
encoding = response.headers.get("content-encoding", "identity").strip().lower()
|
||||
if encoding in ("", "identity"):
|
||||
decoder = None
|
||||
elif encoding in ("gzip", "x-gzip"):
|
||||
decoder = zlib.decompressobj(16 + zlib.MAX_WBITS)
|
||||
elif encoding == "deflate":
|
||||
decoder = zlib.decompressobj()
|
||||
else:
|
||||
raise BizException(502, "数据源响应的压缩格式不支持")
|
||||
declared = response.headers.get("content-length", "")
|
||||
if declared.isdigit() and int(declared) > MAX_BYTES:
|
||||
raise _too_large()
|
||||
chunks: list[bytes] = []
|
||||
raw_size = size = 0
|
||||
try:
|
||||
async for chunk in response.aiter_raw():
|
||||
raw_size += len(chunk)
|
||||
if raw_size > MAX_BYTES:
|
||||
raise _too_large()
|
||||
if decoder is None:
|
||||
chunks.append(chunk)
|
||||
size += len(chunk)
|
||||
continue
|
||||
pending = chunk
|
||||
while pending:
|
||||
# max_length 为 0 表示不限,这里恒大于 0
|
||||
part = decoder.decompress(pending, MAX_BYTES - size + 1)
|
||||
size += len(part)
|
||||
if size > MAX_BYTES:
|
||||
raise _too_large()
|
||||
chunks.append(part)
|
||||
pending = decoder.unconsumed_tail
|
||||
if not part:
|
||||
break
|
||||
# 压缩流没有正常结束(被截断)也算失败,别把半截数据当成功
|
||||
if decoder is not None and not decoder.eof:
|
||||
raise BizException(502, "数据源响应解压失败")
|
||||
except zlib.error:
|
||||
raise BizException(502, "数据源响应解压失败") from None
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
async def _request(
|
||||
client: httpx.AsyncClient,
|
||||
method: str,
|
||||
url: httpx.URL,
|
||||
headers: dict[str, str],
|
||||
body: Any,
|
||||
) -> tuple[int, httpx.Headers, bytes]:
|
||||
port = url.port or (443 if url.scheme == "https" else 80)
|
||||
pinned = url.copy_with(host=await _resolve(url.host, port))
|
||||
request = client.build_request(
|
||||
method,
|
||||
pinned,
|
||||
headers={**headers, "Host": url.netloc.decode()},
|
||||
json=body if method == "POST" else None,
|
||||
# 连接用 IP,TLS 的 SNI 和证书校验仍然用原主机名
|
||||
extensions={"sni_hostname": url.host},
|
||||
)
|
||||
response = await client.send(request, stream=True)
|
||||
try:
|
||||
return response.status_code, response.headers, await _read_body(response)
|
||||
finally:
|
||||
await response.aclose()
|
||||
|
||||
|
||||
def _origin(url: httpx.URL) -> tuple[str, str, int | None]:
|
||||
return url.scheme, url.host, url.port or (443 if url.scheme == "https" else 80)
|
||||
|
||||
|
||||
async def fetch_json(
|
||||
url: str,
|
||||
method: Method = "GET",
|
||||
params: dict[str, Any] | None = None,
|
||||
headers: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | list[Any]:
|
||||
"""GET 的 params 作为查询参数,POST 的 params 作为 JSON 请求体,和前端 fetchData 一致"""
|
||||
target = _parse_url(url)
|
||||
request_headers = _clean_headers(headers)
|
||||
body = None
|
||||
if method == "GET":
|
||||
if params:
|
||||
target = target.copy_merge_params(
|
||||
{key: _query_value(value) for key, value in params.items()}
|
||||
)
|
||||
else:
|
||||
body = params or {}
|
||||
try:
|
||||
async with asyncio.timeout(TIMEOUT):
|
||||
# trust_env=False:环境里的代理会绕过上面的地址检查
|
||||
# max_keepalive_connections=0:连接按 IP 复用,不关的话同一 IP 上的下一跳会沿用上一跳的 TLS 校验结果
|
||||
async with httpx.AsyncClient(
|
||||
trust_env=False,
|
||||
timeout=TIMEOUT,
|
||||
limits=httpx.Limits(max_keepalive_connections=0),
|
||||
) as client:
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
status, response_headers, content = await _request(
|
||||
client, method, target, request_headers, body
|
||||
)
|
||||
location = response_headers.get("location")
|
||||
if status not in REDIRECT_STATUS or not location:
|
||||
break
|
||||
previous = target
|
||||
target = _parse_url(urljoin(str(target), location))
|
||||
if _origin(target) != _origin(previous):
|
||||
# 密钥头只发给调用方指定的那个源
|
||||
request_headers = _clean_headers(None)
|
||||
if status not in (307, 308):
|
||||
method, body = "GET", None
|
||||
else:
|
||||
raise BizException(502, "数据源重定向次数过多")
|
||||
except (TimeoutError, httpx.TimeoutException):
|
||||
raise BizException(504, "数据源请求超时") from None
|
||||
except httpx.HTTPError:
|
||||
raise BizException(502, "数据源请求失败") from None
|
||||
if not 200 <= status < 300:
|
||||
raise BizException(502, f"数据源返回 HTTP {status}")
|
||||
try:
|
||||
data = json.loads(content)
|
||||
except (ValueError, RecursionError):
|
||||
# 2 MB 以内的深层嵌套会打满递归,同样当作非法 JSON
|
||||
raise BizException(502, "数据源返回的不是 JSON") from None
|
||||
if not isinstance(data, (dict, list)):
|
||||
raise BizException(502, "数据源返回的不是 JSON 对象或数组")
|
||||
return data
|
||||
|
||||
|
||||
class FetchCache:
|
||||
"""线上大屏多人同时开、按 interval 轮询:同一个 key 几秒内只真正请求一次,并发的请求共用同一次
|
||||
|
||||
失败不缓存;单实例部署,缓存放进程内存。单条最大 2 MB,条数上限乘出来就是内存上限,所以条数要小
|
||||
"""
|
||||
|
||||
def __init__(self, ttl: float = 3.0, max_entries: int = 64):
|
||||
self._ttl = ttl
|
||||
self._max_entries = max_entries
|
||||
self._entries: dict[Any, tuple[float, Any]] = {}
|
||||
self._inflight: dict[Any, asyncio.Future] = {}
|
||||
|
||||
async def get(self, key: Any, fetch: Callable[[], Awaitable[Any]]) -> Any:
|
||||
now = time.monotonic()
|
||||
hit = self._entries.get(key)
|
||||
if hit is not None and hit[0] > now:
|
||||
return hit[1]
|
||||
future = self._inflight.get(key)
|
||||
if future is None:
|
||||
future = asyncio.ensure_future(fetch())
|
||||
self._inflight[key] = future
|
||||
future.add_done_callback(lambda done: self._finish(key, done))
|
||||
# shield:某个请求者断开不能取消共用的那次请求
|
||||
return await asyncio.shield(future)
|
||||
|
||||
def _finish(self, key: Any, future: asyncio.Future) -> None:
|
||||
self._inflight.pop(key, None)
|
||||
if future.cancelled() or future.exception() is not None:
|
||||
return
|
||||
now = time.monotonic()
|
||||
# 每次写入先丢掉过期项;清完还满就不缓存这一条
|
||||
self._entries = {k: v for k, v in self._entries.items() if v[0] > now}
|
||||
if len(self._entries) < self._max_entries:
|
||||
self._entries[key] = (now + self._ttl, future.result())
|
||||
@@ -0,0 +1,238 @@
|
||||
"""大屏存储的 MongoDB 实现,方法语义与内存实现一致
|
||||
|
||||
集合:screens(草稿,一屏一条)、screen_publishes(发布快照,只增不改)
|
||||
"""
|
||||
|
||||
import re
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC
|
||||
from typing import Any
|
||||
|
||||
from bson.codec_options import CodecOptions
|
||||
from bson.errors import InvalidDocument
|
||||
from pydantic import BaseModel
|
||||
from pymongo import ASCENDING, DESCENDING, ReturnDocument
|
||||
from pymongo.errors import DocumentTooLarge, DuplicateKeyError
|
||||
|
||||
from app.core.exceptions import BizException
|
||||
from app.schemas.saved_screen import (
|
||||
PublishEntry,
|
||||
PublishSnapshot,
|
||||
ScreenRecord,
|
||||
ScreenSummary,
|
||||
)
|
||||
from app.services.screen_store import canvas_size
|
||||
|
||||
# 列表和元数据更新不需要页面,只取画布;Mongo 的投影不能同时"排除 page"和"包含 page.canvas",所以列字段
|
||||
SUMMARY_PROJECTION = {
|
||||
"owner_id": 1,
|
||||
"name": 1,
|
||||
"thumbnail": 1,
|
||||
"page.canvas": 1,
|
||||
"revision": 1,
|
||||
"created_at": 1,
|
||||
"updated_at": 1,
|
||||
"deleted_at": 1,
|
||||
"publish": 1,
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _storable():
|
||||
"""页面里的超大整数、含 NUL 的字段名、编码后超过 16 MB 的文档,驱动会抛各种非业务异常
|
||||
|
||||
10 MB 的 JSON 编码成 BSON 后会涨(数组的下标要当字符串键存),所以体积上限靠这里兜底
|
||||
"""
|
||||
try:
|
||||
yield
|
||||
except (OverflowError, InvalidDocument, DocumentTooLarge):
|
||||
raise BizException(
|
||||
422, "大屏数据无法保存:数字超出范围、字段名不合法或体积过大"
|
||||
) from None
|
||||
|
||||
|
||||
# update 自己拼的条件,调用方的 where 不许覆盖
|
||||
RESERVED_FILTER_KEYS = {"_id", "owner_id", "deleted_at", "revision"}
|
||||
|
||||
|
||||
def _owned(screen_id: str, owner_id: int, deleted: bool) -> dict[str, Any]:
|
||||
return {
|
||||
"_id": screen_id,
|
||||
"owner_id": owner_id,
|
||||
"deleted_at": {"$ne": None} if deleted else None,
|
||||
}
|
||||
|
||||
|
||||
def _encode(value: Any) -> Any:
|
||||
return value.model_dump() if isinstance(value, BaseModel) else value
|
||||
|
||||
|
||||
def _summary(document: dict[str, Any]) -> ScreenSummary:
|
||||
canvas = (document.get("page") or {}).get("canvas")
|
||||
data = {k: v for k, v in document.items() if k not in ("_id", "page")}
|
||||
return ScreenSummary.model_validate(
|
||||
{**data, "screen_id": document["_id"], "canvas": canvas_size(canvas)}
|
||||
)
|
||||
|
||||
|
||||
class MongoScreenStore:
|
||||
def __init__(self, db: Any, prefix: str = ""):
|
||||
# 读回的时间统一带 UTC 时区;集合名前缀用于测试隔离
|
||||
options = CodecOptions(tz_aware=True, tzinfo=UTC)
|
||||
self._screens = db.get_collection(f"{prefix}screens", codec_options=options)
|
||||
self._publishes = db.get_collection(
|
||||
f"{prefix}screen_publishes", codec_options=options
|
||||
)
|
||||
|
||||
async def ensure_indexes(self) -> None:
|
||||
await self._screens.create_index(
|
||||
[
|
||||
("owner_id", ASCENDING),
|
||||
("deleted_at", ASCENDING),
|
||||
("updated_at", DESCENDING),
|
||||
]
|
||||
)
|
||||
# 分享链接唯一;没发布过的大屏没有 publish.share_id,不进索引
|
||||
await self._screens.create_index(
|
||||
"publish.share_id",
|
||||
unique=True,
|
||||
partialFilterExpression={"publish.share_id": {"$type": "string"}},
|
||||
)
|
||||
await self._publishes.create_index(
|
||||
[("screen_id", ASCENDING), ("version", DESCENDING)], unique=True
|
||||
)
|
||||
# 回收站过期清理按 deleted_at 跨用户扫描;没删除的不进索引
|
||||
await self._screens.create_index(
|
||||
"deleted_at", partialFilterExpression={"deleted_at": {"$type": "date"}}
|
||||
)
|
||||
|
||||
async def insert(self, record: ScreenRecord) -> None:
|
||||
document = record.model_dump(exclude={"screen_id", "canvas"})
|
||||
with _storable():
|
||||
await self._screens.insert_one({"_id": record.screen_id, **document})
|
||||
|
||||
async def get(self, screen_id, owner_id, *, deleted=False):
|
||||
document = await self._screens.find_one(_owned(screen_id, owner_id, deleted))
|
||||
if document is None:
|
||||
return None
|
||||
return ScreenRecord.model_validate(
|
||||
{**_summary(document).model_dump(), "page": document["page"]}
|
||||
)
|
||||
|
||||
async def get_summary(self, screen_id, owner_id, *, deleted=False):
|
||||
document = await self._screens.find_one(
|
||||
_owned(screen_id, owner_id, deleted), SUMMARY_PROJECTION
|
||||
)
|
||||
return None if document is None else _summary(document)
|
||||
|
||||
async def update(
|
||||
self, screen_id, owner_id, changes, *, revision=None, where=None, deleted=False
|
||||
):
|
||||
query = _owned(screen_id, owner_id, deleted)
|
||||
if revision is not None:
|
||||
query["revision"] = revision
|
||||
if clash := RESERVED_FILTER_KEYS & (where or {}).keys():
|
||||
raise ValueError(f"where 不能覆盖所有权和版本条件:{sorted(clash)}")
|
||||
query.update(where or {})
|
||||
with _storable():
|
||||
document = await self._screens.find_one_and_update(
|
||||
query,
|
||||
{"$set": {key: _encode(value) for key, value in changes.items()}},
|
||||
projection=SUMMARY_PROJECTION,
|
||||
return_document=ReturnDocument.AFTER,
|
||||
)
|
||||
return None if document is None else _summary(document)
|
||||
|
||||
async def search(
|
||||
self, owner_id, *, keyword, status, sort, descending, limit, offset, deleted
|
||||
):
|
||||
query: dict[str, Any] = {
|
||||
"owner_id": owner_id,
|
||||
"deleted_at": {"$ne": None} if deleted else None,
|
||||
}
|
||||
if keyword:
|
||||
query["name"] = {"$regex": re.escape(keyword), "$options": "i"}
|
||||
if status == "draft":
|
||||
query["publish"] = None
|
||||
elif status is not None:
|
||||
query["publish.status"] = status
|
||||
direction = DESCENDING if descending else ASCENDING
|
||||
documents = await (
|
||||
self._screens.find(query, SUMMARY_PROJECTION)
|
||||
.sort([(sort, direction), ("_id", direction)])
|
||||
.skip(offset)
|
||||
.limit(limit)
|
||||
.to_list()
|
||||
)
|
||||
return [_summary(d) for d in documents], await self._screens.count_documents(
|
||||
query
|
||||
)
|
||||
|
||||
async def get_by_share_id(self, share_id):
|
||||
document = await self._screens.find_one(
|
||||
{"publish.share_id": share_id, "deleted_at": None}, SUMMARY_PROJECTION
|
||||
)
|
||||
return None if document is None else _summary(document)
|
||||
|
||||
async def list_expired_trash(self, deleted_before, limit):
|
||||
documents = await (
|
||||
self._screens.find(
|
||||
# 带 $type 才蕴含部分索引的过滤条件,查询才用得上索引
|
||||
{"deleted_at": {"$type": "date", "$lt": deleted_before}},
|
||||
SUMMARY_PROJECTION,
|
||||
)
|
||||
.sort("deleted_at", ASCENDING)
|
||||
.limit(limit)
|
||||
.to_list()
|
||||
)
|
||||
return [_summary(d) for d in documents]
|
||||
|
||||
async def purge(self, screen_id, owner_id):
|
||||
# 带"仍在回收站"的条件:清理期间被还原的大屏不能删
|
||||
result = await self._screens.delete_one(_owned(screen_id, owner_id, True))
|
||||
return result.deleted_count == 1
|
||||
|
||||
async def delete_publishes(self, screen_id):
|
||||
await self._publishes.delete_many({"screen_id": screen_id})
|
||||
|
||||
async def last_publish_version(self, screen_id):
|
||||
document = await self._publishes.find_one(
|
||||
{"screen_id": screen_id}, {"version": 1}, sort=[("version", DESCENDING)]
|
||||
)
|
||||
return 0 if document is None else document["version"]
|
||||
|
||||
async def insert_publish(self, snapshot):
|
||||
try:
|
||||
with _storable():
|
||||
await self._publishes.insert_one(snapshot.model_dump())
|
||||
except DuplicateKeyError:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def get_publish(self, screen_id, version):
|
||||
document = await self._publishes.find_one(
|
||||
{"screen_id": screen_id, "version": version}
|
||||
)
|
||||
if document is None:
|
||||
return None
|
||||
return PublishSnapshot.model_validate(
|
||||
{k: v for k, v in document.items() if k != "_id"}
|
||||
)
|
||||
|
||||
async def list_publishes(self, screen_id, limit, offset):
|
||||
query = {"screen_id": screen_id}
|
||||
documents = await (
|
||||
self._publishes.find(query, {"page": 0})
|
||||
.sort("version", DESCENDING)
|
||||
.skip(offset)
|
||||
.limit(limit)
|
||||
.to_list()
|
||||
)
|
||||
entries = [
|
||||
PublishEntry.model_validate({k: v for k, v in d.items() if k != "_id"})
|
||||
for d in documents
|
||||
]
|
||||
return entries, await self._publishes.count_documents(query)
|
||||
|
||||
async def delete_publish(self, screen_id, version):
|
||||
await self._publishes.delete_one({"screen_id": screen_id, "version": version})
|
||||
@@ -0,0 +1,241 @@
|
||||
"""大屏存储:协议和进程内存实现;MongoDB 实现见 screen_mongo
|
||||
|
||||
内存实现只给测试用——大屏数据不能丢,没配 MongoDB 时 screens 接口直接返回 503
|
||||
所有查询都带 owner_id:别人的大屏与不存在同样返回 None
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Protocol
|
||||
|
||||
from app.schemas.saved_screen import (
|
||||
ListStatus,
|
||||
PublishEntry,
|
||||
PublishSnapshot,
|
||||
ScreenRecord,
|
||||
ScreenSummary,
|
||||
SortField,
|
||||
)
|
||||
|
||||
|
||||
def canvas_size(canvas: Any) -> dict[str, Any] | None:
|
||||
"""列表卡片只需要画布宽高"""
|
||||
if not isinstance(canvas, dict):
|
||||
return None
|
||||
return {key: canvas[key] for key in ("width", "height") if key in canvas}
|
||||
|
||||
|
||||
def summarize(record: ScreenRecord) -> ScreenSummary:
|
||||
"""去掉页面,只保留画布宽高"""
|
||||
data = record.model_dump(exclude={"page"})
|
||||
data["canvas"] = canvas_size(record.page.get("canvas"))
|
||||
return ScreenSummary.model_validate(data)
|
||||
|
||||
|
||||
class ScreenStore(Protocol):
|
||||
async def insert(self, record: ScreenRecord) -> None: ...
|
||||
|
||||
async def get(
|
||||
self, screen_id: str, owner_id: int, *, deleted: bool = False
|
||||
) -> ScreenRecord | None:
|
||||
"""带页面的记录;deleted 为 True 时只找回收站里的"""
|
||||
...
|
||||
|
||||
async def get_summary(
|
||||
self, screen_id: str, owner_id: int, *, deleted: bool = False
|
||||
) -> ScreenSummary | None: ...
|
||||
|
||||
async def update(
|
||||
self,
|
||||
screen_id: str,
|
||||
owner_id: int,
|
||||
changes: dict[str, Any],
|
||||
*,
|
||||
revision: int | None = None,
|
||||
where: dict[str, Any] | None = None,
|
||||
deleted: bool = False,
|
||||
) -> ScreenSummary | None:
|
||||
"""一步完成"匹配并更新";不匹配返回 None
|
||||
|
||||
给了 revision 就要求库里的 revision 与之相等;where 是额外的等值条件,键可以带点,
|
||||
如 {"publish.version": 3}、{"publish": None}——更新是基于先前读到的值做的,值变了就不能盖上去
|
||||
"""
|
||||
...
|
||||
|
||||
async def search(
|
||||
self,
|
||||
owner_id: int,
|
||||
*,
|
||||
keyword: str | None,
|
||||
status: ListStatus | None,
|
||||
sort: SortField,
|
||||
descending: bool,
|
||||
limit: int,
|
||||
offset: int,
|
||||
deleted: bool,
|
||||
) -> tuple[list[ScreenSummary], int]: ...
|
||||
|
||||
async def get_by_share_id(self, share_id: str) -> ScreenSummary | None:
|
||||
"""未删除的大屏;是否已下线由调用方看 publish.status"""
|
||||
...
|
||||
|
||||
async def list_expired_trash(
|
||||
self, deleted_before: datetime, limit: int
|
||||
) -> list[ScreenSummary]:
|
||||
"""所有人的回收站里 deleted_at 早于 deleted_before 的大屏"""
|
||||
...
|
||||
|
||||
async def purge(self, screen_id: str, owner_id: int) -> bool:
|
||||
"""彻底删除回收站里的大屏;已经被还原(不在回收站)或不存在时返回 False"""
|
||||
...
|
||||
|
||||
async def delete_publishes(self, screen_id: str) -> None: ...
|
||||
|
||||
async def last_publish_version(self, screen_id: str) -> int: ...
|
||||
|
||||
async def insert_publish(self, snapshot: PublishSnapshot) -> bool:
|
||||
"""同一大屏同一版本号已存在时返回 False"""
|
||||
...
|
||||
|
||||
async def get_publish(
|
||||
self, screen_id: str, version: int
|
||||
) -> PublishSnapshot | None: ...
|
||||
|
||||
async def list_publishes(
|
||||
self, screen_id: str, limit: int, offset: int
|
||||
) -> tuple[list[PublishEntry], int]:
|
||||
"""版本号倒序"""
|
||||
...
|
||||
|
||||
async def delete_publish(self, screen_id: str, version: int) -> None: ...
|
||||
|
||||
|
||||
class InMemoryScreenStore:
|
||||
def __init__(self) -> None:
|
||||
self._screens: dict[str, ScreenRecord] = {}
|
||||
self._publishes: dict[tuple[str, int], PublishSnapshot] = {}
|
||||
|
||||
def _find(
|
||||
self, screen_id: str, owner_id: int, deleted: bool
|
||||
) -> ScreenRecord | None:
|
||||
record = self._screens.get(screen_id)
|
||||
if record is None or record.owner_id != owner_id:
|
||||
return None
|
||||
return record if (record.deleted_at is not None) == deleted else None
|
||||
|
||||
async def insert(self, record: ScreenRecord) -> None:
|
||||
self._screens[record.screen_id] = record.model_copy(deep=True)
|
||||
|
||||
async def get(self, screen_id, owner_id, *, deleted=False):
|
||||
record = self._find(screen_id, owner_id, deleted)
|
||||
return None if record is None else record.model_copy(deep=True)
|
||||
|
||||
async def get_summary(self, screen_id, owner_id, *, deleted=False):
|
||||
record = self._find(screen_id, owner_id, deleted)
|
||||
return None if record is None else summarize(record)
|
||||
|
||||
async def update(
|
||||
self, screen_id, owner_id, changes, *, revision=None, where=None, deleted=False
|
||||
):
|
||||
record = self._find(screen_id, owner_id, deleted)
|
||||
if record is None or (revision is not None and record.revision != revision):
|
||||
return None
|
||||
if where is not None and not _matches(record, where):
|
||||
return None
|
||||
updated = record.model_copy(update=changes, deep=True)
|
||||
self._screens[screen_id] = updated
|
||||
return summarize(updated)
|
||||
|
||||
async def search(
|
||||
self, owner_id, *, keyword, status, sort, descending, limit, offset, deleted
|
||||
):
|
||||
records = [
|
||||
record
|
||||
for record in self._screens.values()
|
||||
if record.owner_id == owner_id
|
||||
and (record.deleted_at is not None) == deleted
|
||||
and (keyword is None or keyword.lower() in record.name.lower())
|
||||
and _status_matches(record, status)
|
||||
]
|
||||
# 排序字段相同时按 ID 兜底,翻页才稳定
|
||||
records.sort(key=lambda r: (getattr(r, sort), r.screen_id), reverse=descending)
|
||||
return [summarize(r) for r in records[offset : offset + limit]], len(records)
|
||||
|
||||
async def get_by_share_id(self, share_id):
|
||||
for record in self._screens.values():
|
||||
if (
|
||||
record.deleted_at is None
|
||||
and record.publish is not None
|
||||
and record.publish.share_id == share_id
|
||||
):
|
||||
return summarize(record)
|
||||
return None
|
||||
|
||||
async def list_expired_trash(self, deleted_before, limit):
|
||||
expired = sorted(
|
||||
(
|
||||
r
|
||||
for r in self._screens.values()
|
||||
if r.deleted_at is not None and r.deleted_at < deleted_before
|
||||
),
|
||||
key=lambda r: r.deleted_at,
|
||||
)
|
||||
return [summarize(r) for r in expired[:limit]]
|
||||
|
||||
async def purge(self, screen_id, owner_id):
|
||||
if self._find(screen_id, owner_id, True) is None:
|
||||
return False
|
||||
del self._screens[screen_id]
|
||||
return True
|
||||
|
||||
async def delete_publishes(self, screen_id):
|
||||
for key in [key for key in self._publishes if key[0] == screen_id]:
|
||||
del self._publishes[key]
|
||||
|
||||
async def last_publish_version(self, screen_id):
|
||||
return max(
|
||||
(version for sid, version in self._publishes if sid == screen_id), default=0
|
||||
)
|
||||
|
||||
async def insert_publish(self, snapshot):
|
||||
key = (snapshot.screen_id, snapshot.version)
|
||||
if key in self._publishes:
|
||||
return False
|
||||
self._publishes[key] = snapshot.model_copy(deep=True)
|
||||
return True
|
||||
|
||||
async def get_publish(self, screen_id, version):
|
||||
snapshot = self._publishes.get((screen_id, version))
|
||||
return None if snapshot is None else snapshot.model_copy(deep=True)
|
||||
|
||||
async def list_publishes(self, screen_id, limit, offset):
|
||||
snapshots = sorted(
|
||||
(s for (sid, _), s in self._publishes.items() if sid == screen_id),
|
||||
key=lambda s: s.version,
|
||||
reverse=True,
|
||||
)
|
||||
entries = [
|
||||
PublishEntry.model_validate(s.model_dump(exclude={"page"}))
|
||||
for s in snapshots[offset : offset + limit]
|
||||
]
|
||||
return entries, len(snapshots)
|
||||
|
||||
async def delete_publish(self, screen_id, version):
|
||||
self._publishes.pop((screen_id, version), None)
|
||||
|
||||
|
||||
def _matches(record: ScreenRecord, where: dict[str, Any]) -> bool:
|
||||
for path, expected in where.items():
|
||||
value: Any = record
|
||||
for part in path.split("."):
|
||||
value = None if value is None else getattr(value, part, None)
|
||||
if value != expected:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _status_matches(record: ScreenSummary, status: ListStatus | None) -> bool:
|
||||
if status is None:
|
||||
return True
|
||||
if status == "draft":
|
||||
return record.publish is None
|
||||
return record.publish is not None and record.publish.status == status
|
||||
@@ -0,0 +1,562 @@
|
||||
"""大屏的业务逻辑:存取、乐观锁、发布、历史、复制、封面、线上读取和数据源取数
|
||||
|
||||
存储层只提供"匹配并更新"这类原语,404 / 409 的判断都在这里
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import secrets
|
||||
import weakref
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.core.exceptions import BizException
|
||||
from app.schemas.saved_screen import (
|
||||
NAME_MAX_LENGTH,
|
||||
ListStatus,
|
||||
PublishEntry,
|
||||
PublishInfo,
|
||||
PublishSnapshot,
|
||||
ScreenRecord,
|
||||
ScreenSummary,
|
||||
SortField,
|
||||
)
|
||||
from app.schemas.screen import Screen
|
||||
from app.services.datasource_proxy import FetchCache, fetch_json
|
||||
from app.services.screen_store import ScreenStore
|
||||
from app.services.thumbnails import ThumbnailStorage
|
||||
|
||||
# 事件代码和数据源转换函数是字符串形式的 JS,线上会在浏览器里执行;写入权限只给 owner,再限制长度
|
||||
SCRIPT_MAX_CHARS = 64 * 1024
|
||||
COPY_SUFFIX = " 副本"
|
||||
EMPTY_PAGE = {
|
||||
"canvas": {"width": 1920, "height": 1080, "backgroundColor": "#ffffff10"},
|
||||
"nodes": [],
|
||||
"dataSource": [],
|
||||
}
|
||||
|
||||
# 回收站过期清理:一批最多清多少个,两次扫描的间隔(秒)
|
||||
PURGE_BATCH = 100
|
||||
TRASH_SWEEP_INTERVAL = 3600
|
||||
|
||||
STALE = "大屏已在别处修改"
|
||||
# 下线 / 删除 / 换封面遇到并发修改时的重读次数
|
||||
UPDATE_ATTEMPTS = 3
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class FilePinner(Protocol):
|
||||
"""数据文件服务里和大屏有关的两个方法"""
|
||||
|
||||
async def sync_screen_files(
|
||||
self, user_id: int, screen_id: str, file_ids: set[str]
|
||||
) -> None: ...
|
||||
|
||||
async def release_screen_files(self, screen_id: str) -> None: ...
|
||||
|
||||
|
||||
def referenced_file_ids(page: dict[str, Any]) -> set[str]:
|
||||
"""页面里图表绑定的数据文件:nodes[].fileQuery.file_id"""
|
||||
ids = set()
|
||||
nodes = page.get("nodes")
|
||||
for node in nodes if isinstance(nodes, list) else []:
|
||||
query = node.get("fileQuery") if isinstance(node, dict) else None
|
||||
file_id = query.get("file_id") if isinstance(query, dict) else None
|
||||
if isinstance(file_id, str) and file_id:
|
||||
ids.add(file_id)
|
||||
return ids
|
||||
|
||||
|
||||
def _publish_is(publish: PublishInfo | None) -> dict[str, Any]:
|
||||
""" "线上指针还是我读到的那个"的更新条件"""
|
||||
if publish is None:
|
||||
return {"publish": None}
|
||||
return {"publish.version": publish.version, "publish.status": publish.status}
|
||||
|
||||
|
||||
def _not_found() -> BizException:
|
||||
# 别人的大屏与不存在同样返回 404
|
||||
return BizException(404, "大屏不存在或已删除")
|
||||
|
||||
|
||||
def validate_page(page: dict[str, Any]) -> dict[str, Any]:
|
||||
"""按 Screen 校验结构,但返回的仍是原始 JSON:model_dump 会给没传的字段补出 null,前端读回来就变了"""
|
||||
try:
|
||||
Screen.model_validate(page)
|
||||
except ValidationError as error:
|
||||
first = error.errors()[0]
|
||||
where = ".".join(str(part) for part in first["loc"])
|
||||
raise BizException(422, f"大屏数据不合法:{where}:{first['msg']}") from None
|
||||
_check_script_length(page)
|
||||
return page
|
||||
|
||||
|
||||
def _check_script_length(page: dict[str, Any]) -> None:
|
||||
scripts = [
|
||||
event.get("code")
|
||||
for node in page["nodes"]
|
||||
for event in (node.get("events") or [])
|
||||
if isinstance(event, dict)
|
||||
] + [
|
||||
source.get("transform")
|
||||
for source in page.get("dataSource", [])
|
||||
if isinstance(source, dict)
|
||||
]
|
||||
if any(isinstance(text, str) and len(text) > SCRIPT_MAX_CHARS for text in scripts):
|
||||
raise BizException(
|
||||
422, f"事件代码和数据转换函数不能超过 {SCRIPT_MAX_CHARS // 1024} KB"
|
||||
)
|
||||
|
||||
|
||||
def public_page(page: dict[str, Any]) -> dict[str, Any]:
|
||||
"""线上页面里去掉数据源的 headers:它们常放 API key,线上取数由服务端按快照里的完整配置发起,浏览器用不着"""
|
||||
sources = page.get("dataSource")
|
||||
if not isinstance(sources, list):
|
||||
return {**page}
|
||||
return {
|
||||
**page,
|
||||
"dataSource": [
|
||||
{key: value for key, value in item.items() if key != "headers"}
|
||||
if isinstance(item, dict)
|
||||
else item
|
||||
for item in sources
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class ScreenService:
|
||||
def __init__(
|
||||
self,
|
||||
store: ScreenStore,
|
||||
thumbnails: ThumbnailStorage,
|
||||
files: FilePinner | None = None,
|
||||
trash_days: int = 30,
|
||||
):
|
||||
self._store = store
|
||||
self._thumbnails = thumbnails
|
||||
# 已保存的大屏引用的数据文件不能过期,否则"刷新数据"会 404;没有文件服务时不管
|
||||
self._files = files
|
||||
if trash_days < 1:
|
||||
raise ValueError("trash_days 必须是正整数")
|
||||
self.trash_days = trash_days
|
||||
# 按大屏的锁:保存 / 回滚 / 删除 / 还原都是"改库再同步固定的文件"两步,
|
||||
# 两个请求交错会让固定的文件跟着先开始、后结束的那个请求的旧页面走。单实例部署,进程内锁就够
|
||||
self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = (
|
||||
weakref.WeakValueDictionary()
|
||||
)
|
||||
self._datasource_cache = FetchCache()
|
||||
|
||||
def _lock(self, screen_id: str) -> asyncio.Lock:
|
||||
lock = self._locks.get(screen_id)
|
||||
if lock is None:
|
||||
lock = self._locks[screen_id] = asyncio.Lock()
|
||||
return lock
|
||||
|
||||
async def _pin_files(self, owner_id: int, screen_id: str, page: dict[str, Any]):
|
||||
"""让数据文件跟着大屏的引用走。主操作已经成功,这一步失败只记日志,不让保存报错"""
|
||||
if self._files is None:
|
||||
return
|
||||
try:
|
||||
await self._files.sync_screen_files(
|
||||
owner_id, screen_id, referenced_file_ids(page)
|
||||
)
|
||||
except Exception as error:
|
||||
logger.opt(exception=error).error(
|
||||
"同步大屏 {} 引用的数据文件失败", screen_id
|
||||
)
|
||||
|
||||
async def _release_files(self, screen_id: str) -> None:
|
||||
if self._files is None:
|
||||
return
|
||||
try:
|
||||
await self._files.release_screen_files(screen_id)
|
||||
except Exception as error:
|
||||
logger.opt(exception=error).error(
|
||||
"放开大屏 {} 固定的数据文件失败", screen_id
|
||||
)
|
||||
|
||||
# ---- 草稿 ----
|
||||
|
||||
async def create(
|
||||
self, owner_id: int, name: str, page: dict[str, Any] | None
|
||||
) -> ScreenRecord:
|
||||
now = _now()
|
||||
record = ScreenRecord(
|
||||
screen_id=f"s_{uuid4().hex}",
|
||||
owner_id=owner_id,
|
||||
name=name,
|
||||
page=validate_page(copy.deepcopy(EMPTY_PAGE) if page is None else page),
|
||||
revision=1,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
await self._store.insert(record)
|
||||
await self._pin_files(owner_id, record.screen_id, record.page)
|
||||
return record
|
||||
|
||||
async def get(self, screen_id: str, owner_id: int) -> ScreenRecord:
|
||||
record = await self._store.get(screen_id, owner_id)
|
||||
if record is None:
|
||||
raise _not_found()
|
||||
return record
|
||||
|
||||
async def save(
|
||||
self, screen_id: str, owner_id: int, page: dict[str, Any], revision: int
|
||||
) -> ScreenSummary:
|
||||
validate_page(page)
|
||||
async with self._lock(screen_id):
|
||||
saved = await self._update_with_revision(
|
||||
screen_id,
|
||||
owner_id,
|
||||
{"page": page, "revision": revision + 1, "updated_at": _now()},
|
||||
revision,
|
||||
)
|
||||
await self._pin_files(owner_id, screen_id, page)
|
||||
return saved
|
||||
|
||||
async def _update_with_revision(
|
||||
self, screen_id: str, owner_id: int, changes: dict[str, Any], revision: int
|
||||
) -> ScreenSummary:
|
||||
updated = await self._store.update(
|
||||
screen_id, owner_id, changes, revision=revision
|
||||
)
|
||||
if updated is not None:
|
||||
return updated
|
||||
# 没更新成功:大屏不存在是 404,存在但 revision 不一致是 409
|
||||
if await self._store.get_summary(screen_id, owner_id) is None:
|
||||
raise _not_found()
|
||||
raise BizException(409, STALE)
|
||||
|
||||
async def rename(self, screen_id: str, owner_id: int, name: str) -> None:
|
||||
# 改名不算改内容,revision 不动
|
||||
changes = {"name": name, "updated_at": _now()}
|
||||
if await self._store.update(screen_id, owner_id, changes) is None:
|
||||
raise _not_found()
|
||||
|
||||
async def _update_from_summary(
|
||||
self,
|
||||
screen_id: str,
|
||||
owner_id: int,
|
||||
build: Callable[[ScreenSummary], tuple[dict[str, Any], dict[str, Any]] | None],
|
||||
) -> None:
|
||||
"""读出当前状态,据此算出修改,再带着"状态没变"的条件写回;并发改了就重读再来
|
||||
|
||||
下线、删除要整段改写 publish,不带条件就会把并发发布写进去的新指针盖回旧值
|
||||
build 返回 (changes, where);返回 None 表示已经是目标状态,什么都不用做
|
||||
"""
|
||||
for _ in range(UPDATE_ATTEMPTS):
|
||||
summary = await self._store.get_summary(screen_id, owner_id)
|
||||
if summary is None:
|
||||
raise _not_found()
|
||||
built = build(summary)
|
||||
if built is None:
|
||||
return
|
||||
changes, where = built
|
||||
if await self._store.update(screen_id, owner_id, changes, where=where):
|
||||
return
|
||||
raise BizException(409, "操作冲突,请重试")
|
||||
|
||||
async def delete(self, screen_id: str, owner_id: int) -> None:
|
||||
def build(summary: ScreenSummary):
|
||||
changes: dict[str, Any] = {"deleted_at": _now()}
|
||||
if summary.publish is None:
|
||||
return changes, {"publish": None}
|
||||
# 线上版本同时下线,分享链接立即 404
|
||||
changes["publish"] = summary.publish.model_copy(
|
||||
update={"status": "offline"}
|
||||
)
|
||||
return changes, _publish_is(summary.publish)
|
||||
|
||||
async with self._lock(screen_id):
|
||||
await self._update_from_summary(screen_id, owner_id, build)
|
||||
# 进了回收站的大屏不再固定文件,还原时再固定回来
|
||||
await self._release_files(screen_id)
|
||||
|
||||
async def restore_deleted(self, screen_id: str, owner_id: int) -> None:
|
||||
async with self._lock(screen_id):
|
||||
record = await self._store.get(screen_id, owner_id, deleted=True)
|
||||
if record is None:
|
||||
raise _not_found()
|
||||
changes = {"deleted_at": None}
|
||||
if (
|
||||
await self._store.update(screen_id, owner_id, changes, deleted=True)
|
||||
is None
|
||||
):
|
||||
raise _not_found()
|
||||
await self._pin_files(owner_id, screen_id, record.page)
|
||||
|
||||
# ---- 回收站 ----
|
||||
|
||||
async def _purge(self, summary: ScreenSummary) -> bool:
|
||||
"""彻底删除回收站里的一个大屏
|
||||
|
||||
先删大屏本身(带"仍在回收站"的条件,清理期间被还原的不会误删),再清理附属数据
|
||||
"""
|
||||
if not await self._store.purge(summary.screen_id, summary.owner_id):
|
||||
return False
|
||||
await self._store.delete_publishes(summary.screen_id)
|
||||
self._thumbnails.delete(summary.thumbnail)
|
||||
await self._release_files(summary.screen_id)
|
||||
return True
|
||||
|
||||
async def empty_trash(self, owner_id: int) -> int:
|
||||
"""清空自己的回收站,返回彻底删除的个数"""
|
||||
count = 0
|
||||
previous: set[str] = set()
|
||||
while True:
|
||||
batch, _ = await self._store.search(
|
||||
owner_id,
|
||||
keyword=None,
|
||||
status=None,
|
||||
sort="updated_at",
|
||||
descending=True,
|
||||
limit=PURGE_BATCH,
|
||||
offset=0,
|
||||
deleted=True,
|
||||
)
|
||||
ids = {summary.screen_id for summary in batch}
|
||||
purged = [await self._purge(summary) for summary in batch]
|
||||
count += sum(purged)
|
||||
# 空批才算清完;整批都没删成(被并发还原了)就再查一次,
|
||||
# 但同一批连续两次都删不掉说明有东西卡住,别空转
|
||||
if not batch or (not any(purged) and ids == previous):
|
||||
return count
|
||||
previous = ids
|
||||
|
||||
async def purge_expired_trash(self) -> int:
|
||||
"""彻底删除在回收站里超过保留天数的大屏(所有人的)"""
|
||||
deadline = _now() - timedelta(days=self.trash_days)
|
||||
count = 0
|
||||
previous: set[str] = set()
|
||||
while True:
|
||||
batch = await self._store.list_expired_trash(deadline, PURGE_BATCH)
|
||||
ids = {summary.screen_id for summary in batch}
|
||||
purged = [await self._purge(summary) for summary in batch]
|
||||
count += sum(purged)
|
||||
if not batch or (not any(purged) and ids == previous):
|
||||
return count
|
||||
previous = ids
|
||||
|
||||
async def run_trash_sweeper(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
if removed := await self.purge_expired_trash():
|
||||
logger.info(
|
||||
"已彻底删除 {} 个超过 {} 天的回收站大屏",
|
||||
removed,
|
||||
self.trash_days,
|
||||
)
|
||||
except Exception as error:
|
||||
logger.opt(exception=error).error("清理回收站失败")
|
||||
await asyncio.sleep(TRASH_SWEEP_INTERVAL)
|
||||
|
||||
# ---- 管理页 ----
|
||||
|
||||
async def list_screens(
|
||||
self,
|
||||
owner_id: int,
|
||||
*,
|
||||
keyword: str | None,
|
||||
status: ListStatus | None,
|
||||
sort: SortField,
|
||||
descending: bool,
|
||||
limit: int,
|
||||
offset: int,
|
||||
deleted: bool,
|
||||
) -> tuple[list[ScreenSummary], int]:
|
||||
return await self._store.search(
|
||||
owner_id,
|
||||
keyword=keyword,
|
||||
status=status,
|
||||
sort=sort,
|
||||
descending=descending,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
deleted=deleted,
|
||||
)
|
||||
|
||||
async def copy_screen(self, screen_id: str, owner_id: int, name: str | None) -> str:
|
||||
"""只复制草稿和封面,不复制发布状态和历史"""
|
||||
source = await self.get(screen_id, owner_id)
|
||||
base = source.name[: NAME_MAX_LENGTH - len(COPY_SUFFIX)]
|
||||
copied = await self.create(
|
||||
owner_id,
|
||||
name or f"{base}{COPY_SUFFIX}",
|
||||
copy.deepcopy(source.page),
|
||||
)
|
||||
if source.thumbnail is not None:
|
||||
thumbnail = self._thumbnails.copy(source.thumbnail)
|
||||
if thumbnail is not None:
|
||||
try:
|
||||
attached = await self._store.update(
|
||||
copied.screen_id, owner_id, {"thumbnail": thumbnail}
|
||||
)
|
||||
except BaseException:
|
||||
self._thumbnails.delete(thumbnail)
|
||||
raise
|
||||
if not attached:
|
||||
self._thumbnails.delete(thumbnail)
|
||||
return copied.screen_id
|
||||
|
||||
async def set_thumbnail(self, screen_id: str, owner_id: int, data: bytes) -> str:
|
||||
thumbnail = self._thumbnails.save(data)
|
||||
replaced: list[str | None] = []
|
||||
|
||||
def build(summary: ScreenSummary):
|
||||
# 封面只是展示用的,不算改内容:不动 revision 和 updated_at
|
||||
replaced[:] = [summary.thumbnail]
|
||||
return {"thumbnail": thumbnail}, {"thumbnail": summary.thumbnail}
|
||||
|
||||
try:
|
||||
await self._update_from_summary(screen_id, owner_id, build)
|
||||
except BaseException:
|
||||
self._thumbnails.delete(thumbnail)
|
||||
raise
|
||||
# 删的是这次真正换掉的那张;并发上传时各自换掉各自读到的,不会留下孤儿文件
|
||||
self._thumbnails.delete(replaced[0])
|
||||
return thumbnail
|
||||
|
||||
def open_thumbnail(self, thumbnail_id: str):
|
||||
return self._thumbnails.open(thumbnail_id)
|
||||
|
||||
# ---- 发布 ----
|
||||
|
||||
async def publish(
|
||||
self, screen_id: str, owner_id: int, revision: int
|
||||
) -> PublishInfo:
|
||||
record = await self.get(screen_id, owner_id)
|
||||
if record.revision != revision:
|
||||
raise BizException(409, "大屏有未保存或已在别处修改的内容,请先保存")
|
||||
previous = record.publish
|
||||
snapshot = PublishSnapshot(
|
||||
screen_id=screen_id,
|
||||
# 版本号取已有快照的最大值加一:写快照后没来得及更新指针留下的孤儿快照也不会撞号
|
||||
version=await self._store.last_publish_version(screen_id) + 1,
|
||||
revision=record.revision,
|
||||
page=record.page,
|
||||
published_at=_now(),
|
||||
published_by=owner_id,
|
||||
)
|
||||
if not await self._store.insert_publish(snapshot):
|
||||
raise BizException(409, "发布冲突,请重试")
|
||||
info = PublishInfo(
|
||||
status="published",
|
||||
version=snapshot.version,
|
||||
revision=snapshot.revision,
|
||||
# 第一次发布生成 share_id,之后沿用;下线再发布也沿用
|
||||
share_id=previous.share_id
|
||||
if previous
|
||||
else f"p_{secrets.token_urlsafe(16)}",
|
||||
published_at=snapshot.published_at,
|
||||
)
|
||||
# 条件:草稿没被再保存过(revision),线上指针还是读到的那个——并发发布只有一个能成,
|
||||
# 否则第一次发布时两边各生成一个 share_id,先返回给前端的那个会失效
|
||||
try:
|
||||
updated = await self._store.update(
|
||||
screen_id,
|
||||
owner_id,
|
||||
{"publish": info},
|
||||
revision=revision,
|
||||
where=_publish_is(previous),
|
||||
)
|
||||
except BaseException:
|
||||
await self._store.delete_publish(screen_id, snapshot.version)
|
||||
raise
|
||||
if updated is None:
|
||||
await self._store.delete_publish(screen_id, snapshot.version)
|
||||
raise BizException(409, STALE)
|
||||
return info
|
||||
|
||||
async def unpublish(self, screen_id: str, owner_id: int) -> None:
|
||||
def build(summary: ScreenSummary):
|
||||
if summary.publish is None or summary.publish.status == "offline":
|
||||
return None
|
||||
offline = summary.publish.model_copy(update={"status": "offline"})
|
||||
return {"publish": offline}, _publish_is(summary.publish)
|
||||
|
||||
await self._update_from_summary(screen_id, owner_id, build)
|
||||
|
||||
async def exists(self, screen_id: str, owner_id: int) -> bool:
|
||||
return await self._store.get_summary(screen_id, owner_id) is not None
|
||||
|
||||
async def list_publishes(
|
||||
self, screen_id: str, owner_id: int, limit: int, offset: int
|
||||
) -> tuple[list[PublishEntry], int]:
|
||||
if await self._store.get_summary(screen_id, owner_id) is None:
|
||||
raise _not_found()
|
||||
return await self._store.list_publishes(screen_id, limit, offset)
|
||||
|
||||
async def restore_publish(self, screen_id: str, owner_id: int, version: int) -> int:
|
||||
"""把历史版本的页面覆盖回草稿,不自动发布"""
|
||||
async with self._lock(screen_id):
|
||||
record = await self.get(screen_id, owner_id)
|
||||
snapshot = await self._store.get_publish(screen_id, version)
|
||||
if snapshot is None:
|
||||
raise BizException(404, "发布版本不存在")
|
||||
updated = await self._update_with_revision(
|
||||
screen_id,
|
||||
owner_id,
|
||||
{
|
||||
"page": snapshot.page,
|
||||
"revision": record.revision + 1,
|
||||
"updated_at": _now(),
|
||||
},
|
||||
record.revision,
|
||||
)
|
||||
await self._pin_files(owner_id, screen_id, snapshot.page)
|
||||
return updated.revision
|
||||
|
||||
# ---- 线上(匿名) ----
|
||||
|
||||
async def _live(self, share_id: str) -> tuple[ScreenSummary, PublishSnapshot]:
|
||||
"""线上版本:已下线 / 已删除 / 不存在都是 404;返回的是发布快照,不是草稿"""
|
||||
summary = await self._store.get_by_share_id(share_id)
|
||||
if (
|
||||
summary is None
|
||||
or summary.publish is None
|
||||
or summary.publish.status != "published"
|
||||
):
|
||||
raise _not_found()
|
||||
snapshot = await self._store.get_publish(
|
||||
summary.screen_id, summary.publish.version
|
||||
)
|
||||
if snapshot is None:
|
||||
raise _not_found()
|
||||
return summary, snapshot
|
||||
|
||||
async def get_live(self, share_id: str) -> tuple[ScreenSummary, PublishSnapshot]:
|
||||
return await self._live(share_id)
|
||||
|
||||
async def fetch_live_datasource(self, share_id: str, datasource_id: str):
|
||||
"""前端只传数据源 ID,请求地址取自已发布快照;不能做成"前端传 URL 后端转发",否则是匿名开放代理"""
|
||||
_, snapshot = await self._live(share_id)
|
||||
source = next(
|
||||
(
|
||||
item
|
||||
for item in snapshot.page.get("dataSource", [])
|
||||
if isinstance(item, dict) and item.get("id") == datasource_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if source is None or source.get("type") != "api" or not source.get("url"):
|
||||
raise BizException(404, "数据源不存在")
|
||||
method = "POST" if source.get("method") == "POST" else "GET"
|
||||
params = source.get("params")
|
||||
headers = source.get("headers")
|
||||
return await self._datasource_cache.get(
|
||||
(share_id, snapshot.version, datasource_id),
|
||||
lambda: fetch_json(
|
||||
str(source["url"]),
|
||||
method,
|
||||
params if isinstance(params, dict) else None,
|
||||
headers if isinstance(headers, dict) else None,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,63 @@
|
||||
"""大屏封面图:放本地磁盘,文件名就是封面 ID;每次上传换新 ID,旧图删掉
|
||||
|
||||
图片类 URL 要能被 <img src> 匿名访问,靠不可猜的 ID 兜底
|
||||
"""
|
||||
|
||||
import re
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
from app.core.exceptions import BizException
|
||||
|
||||
MAX_BYTES = 1024 * 1024
|
||||
# 按文件头判断类型,不信任客户端声明的 Content-Type
|
||||
MAGIC = {b"\x89PNG\r\n\x1a\n": "image/png", b"\xff\xd8\xff": "image/jpeg"}
|
||||
# ID 进了文件路径,读写前必须过这个格式检查,防路径穿越
|
||||
ID_PATTERN = re.compile(r"th_[A-Za-z0-9_-]{16,64}")
|
||||
|
||||
|
||||
def sniff(head: bytes) -> str | None:
|
||||
for magic, media_type in MAGIC.items():
|
||||
if head.startswith(magic):
|
||||
return media_type
|
||||
return None
|
||||
|
||||
|
||||
class ThumbnailStorage:
|
||||
def __init__(self, root: Path):
|
||||
self._root = root
|
||||
|
||||
def _path(self, thumbnail_id: str) -> Path | None:
|
||||
if not ID_PATTERN.fullmatch(thumbnail_id):
|
||||
return None
|
||||
return self._root / thumbnail_id
|
||||
|
||||
def save(self, data: bytes) -> str:
|
||||
if len(data) > MAX_BYTES:
|
||||
raise BizException(413, "封面图不能超过 1 MB")
|
||||
if sniff(data[:8]) is None:
|
||||
raise BizException(422, "封面图只支持 PNG 或 JPEG")
|
||||
thumbnail_id = f"th_{secrets.token_urlsafe(16)}"
|
||||
self._root.mkdir(parents=True, exist_ok=True)
|
||||
self._path(thumbnail_id).write_bytes(data)
|
||||
return thumbnail_id
|
||||
|
||||
def open(self, thumbnail_id: str) -> tuple[Path, str] | None:
|
||||
"""文件路径和图片类型;ID 不合法、文件不存在或内容不是图片时返回 None"""
|
||||
path = self._path(thumbnail_id)
|
||||
if path is None or not path.is_file():
|
||||
return None
|
||||
with path.open("rb") as file:
|
||||
media_type = sniff(file.read(8))
|
||||
return None if media_type is None else (path, media_type)
|
||||
|
||||
def copy(self, thumbnail_id: str) -> str | None:
|
||||
path = self._path(thumbnail_id)
|
||||
if path is None or not path.is_file():
|
||||
return None
|
||||
return self.save(path.read_bytes())
|
||||
|
||||
def delete(self, thumbnail_id: str | None) -> None:
|
||||
path = None if thumbnail_id is None else self._path(thumbnail_id)
|
||||
if path is not None:
|
||||
path.unlink(missing_ok=True)
|
||||
Reference in New Issue
Block a user