feat:新增大屏存取发布接口

This commit is contained in:
2026-10-11 08:27:30 -07:00
parent dadd797391
commit afc07ba695
23 changed files with 2672 additions and 36 deletions
+51 -11
View File
@@ -21,7 +21,7 @@ def make_title(prompt: str) -> str:
class ConversationStore(Protocol):
"""内存和 MongoDB 实现同一组方法,业务代码不需要关心具体存储"""
async def create(self, user_id: int) -> str: ...
async def create(self, user_id: int, screen_id: str | None = None) -> str: ...
async def exists(self, conversation_id: str, user_id: int) -> bool:
"""会话已签发、未过期且属于该用户;不属于时与不存在同样返回 False"""
@@ -34,9 +34,21 @@ class ConversationStore(Protocol):
async def save_turn(self, turn: Turn) -> None: ...
async def list_by_user(
self, user_id: int, limit: int, offset: int
self, user_id: int, limit: int, offset: int, screen_id: str | None = None
) -> list[ConversationSummary]:
"""该用户已产生对话的会话,按最近更新时间倒序;没有对话的空会话不返回"""
"""该用户已产生对话的会话,按最近更新时间倒序;没有对话的空会话不返回;给了 screen_id 只返回该大屏的"""
...
async def count_by_user(self, user_id: int, screen_id: str | None = None) -> int:
"""与 list_by_user 同一口径的总数"""
...
async def rename(self, conversation_id: str, user_id: int, title: str) -> bool:
"""会话不存在或不属于该用户返回 False"""
...
async def delete(self, conversation_id: str, user_id: int) -> bool:
"""连同对话轮次一起删除;会话不存在或不属于该用户返回 False"""
...
@@ -51,6 +63,8 @@ class InMemoryConversationStore:
# 会话 ID -> 创建时间;轮次按 thread_id 去重并保持首次写入顺序
self._created: dict[str, datetime] = {}
self._owner: dict[str, int] = {}
self._screen: dict[str, str | None] = {}
self._titles: dict[str, str] = {}
self._turns: dict[str, dict[str, Turn]] = {}
def _purge_expired(self) -> None:
@@ -61,15 +75,18 @@ class InMemoryConversationStore:
for key, created in self._created.items()
if created < deadline and not self._turns.get(key)
]:
del self._created[conversation_id]
self._owner.pop(conversation_id, None)
self._turns.pop(conversation_id, None)
self._forget(conversation_id)
async def create(self, user_id: int) -> str:
def _forget(self, conversation_id: str) -> None:
for table in (self._created, self._owner, self._screen, self._titles, self._turns):
table.pop(conversation_id, None)
async def create(self, user_id: int, screen_id: str | None = None) -> str:
self._purge_expired()
conversation_id = str(uuid4())
self._created[conversation_id] = _now()
self._owner[conversation_id] = user_id
self._screen[conversation_id] = screen_id
return conversation_id
async def exists(self, conversation_id: str, user_id: int) -> bool:
@@ -94,8 +111,8 @@ class InMemoryConversationStore:
turn = turn.model_copy(update={"created_at": existing.created_at})
self._turns[turn.conversation_id][turn.thread_id] = turn
async def list_by_user(
self, user_id: int, limit: int, offset: int
def _summaries(
self, user_id: int, screen_id: str | None
) -> list[ConversationSummary]:
summaries = []
for conversation_id, owner in self._owner.items():
@@ -105,14 +122,37 @@ class InMemoryConversationStore:
)
if owner != user_id or not turns:
continue
if screen_id is not None and self._screen.get(conversation_id) != screen_id:
continue
summaries.append(
ConversationSummary(
conversation_id=conversation_id,
title=make_title(turns[0].prompt),
title=self._titles.get(conversation_id) or make_title(turns[0].prompt),
screen_id=self._screen.get(conversation_id),
updated_at=turns[-1].created_at,
)
)
summaries.sort(
key=lambda item: (item.updated_at, item.conversation_id), reverse=True
)
return summaries[offset : offset + limit]
return summaries
async def list_by_user(
self, user_id: int, limit: int, offset: int, screen_id: str | None = None
) -> list[ConversationSummary]:
return self._summaries(user_id, screen_id)[offset : offset + limit]
async def count_by_user(self, user_id: int, screen_id: str | None = None) -> int:
return len(self._summaries(user_id, screen_id))
async def rename(self, conversation_id: str, user_id: int, title: str) -> bool:
if not await self.exists(conversation_id, user_id):
return False
self._titles[conversation_id] = title
return True
async def delete(self, conversation_id: str, user_id: int) -> bool:
if not await self.exists(conversation_id, user_id):
return False
self._forget(conversation_id)
return True
+45 -8
View File
@@ -45,6 +45,8 @@ class MongoConversationStore:
await self._conversations.create_index("created_at")
# 按用户查询会话
await self._conversations.create_index("user_id")
# 按大屏过滤会话
await self._conversations.create_index([("user_id", ASCENDING), ("screen_id", ASCENDING)])
async def _purge_expired(self) -> None:
# 只清理没有任何轮次且超过保留时长的空会话,有对话的会话不受影响
@@ -66,12 +68,17 @@ class MongoConversationStore:
if empty:
await self._conversations.delete_many({"_id": {"$in": empty}})
async def create(self, user_id: int) -> str:
async def create(self, user_id: int, screen_id: str | None = None) -> str:
await self._purge_expired()
conversation_id = str(uuid4())
# user_id 来自用户中心,会话只对创建者可见
await self._conversations.insert_one(
{"_id": conversation_id, "created_at": _now(), "user_id": user_id}
{
"_id": conversation_id,
"created_at": _now(),
"user_id": user_id,
"screen_id": screen_id,
}
)
return conversation_id
@@ -121,12 +128,13 @@ class MongoConversationStore:
upsert=True,
)
async def list_by_user(
self, user_id: int, limit: int, offset: int
) -> list[ConversationSummary]:
def _with_stats(self, user_id: int, screen_id: str | None) -> list[dict]:
match: dict = {"user_id": user_id}
if screen_id is not None:
match["screen_id"] = screen_id
# $unwind 会丢弃没有任何轮次的空会话
pipeline = [
{"$match": {"user_id": user_id}},
return [
{"$match": match},
{
"$lookup": {
"from": self._turns.name,
@@ -146,6 +154,13 @@ class MongoConversationStore:
}
},
{"$unwind": "$stats"},
]
async def list_by_user(
self, user_id: int, limit: int, offset: int, screen_id: str | None = None
) -> list[ConversationSummary]:
pipeline = [
*self._with_stats(user_id, screen_id),
{"$sort": {"stats.updated_at": -1, "_id": -1}},
{"$skip": offset},
{"$limit": limit},
@@ -154,8 +169,30 @@ class MongoConversationStore:
return [
ConversationSummary(
conversation_id=item["_id"],
title=make_title(item["stats"]["first_prompt"]),
title=item.get("title") or make_title(item["stats"]["first_prompt"]),
screen_id=item.get("screen_id"),
updated_at=item["stats"]["updated_at"],
)
for item in await cursor.to_list()
]
async def count_by_user(self, user_id: int, screen_id: str | None = None) -> int:
pipeline = [*self._with_stats(user_id, screen_id), {"$count": "total"}]
cursor = await self._conversations.aggregate(pipeline)
counted = await cursor.to_list()
return counted[0]["total"] if counted else 0
async def rename(self, conversation_id: str, user_id: int, title: str) -> bool:
if not await self.exists(conversation_id, user_id):
return False
await self._conversations.update_one(
{"_id": conversation_id, "user_id": user_id}, {"$set": {"title": title}}
)
return True
async def delete(self, conversation_id: str, user_id: int) -> bool:
if not await self.exists(conversation_id, user_id):
return False
await self._turns.delete_many({"conversation_id": conversation_id})
await self._conversations.delete_one({"_id": conversation_id, "user_id": user_id})
return True
+300
View File
@@ -0,0 +1,300 @@
"""数据源代理:服务端替浏览器请求第三方接口
两个入口共用 fetch_json:编辑态由登录用户传 URL;线上态只传数据源 ID,配置取自已发布快照
防护:只允许 http/https;DNS 解析后每个地址都必须是公网地址,并把连接钉在已检查过的地址上
(不再让 httpx 二次解析,防 DNS 重绑定);不读环境代理;重定向逐跳重新检查,跨域时丢掉调用方的密钥头;
总超时 5 秒,响应体上限 2 MB,按解压后的字节计,解压时带输出上限(防压缩炸弹)
"""
import asyncio
import ipaddress
import json
import re
import socket
import time
import zlib
from collections.abc import Awaitable, Callable
from typing import Any, Literal
from urllib.parse import urljoin
import httpx
from app.core.exceptions import BizException
TIMEOUT = 5.0
MAX_BYTES = 2 * 1024 * 1024
MAX_REDIRECTS = 3
REDIRECT_STATUS = {301, 302, 303, 307, 308}
# 这些头由 httpx 或连接本身决定,不让调用方覆盖
FORBIDDEN_HEADERS = {
"host",
"content-length",
"connection",
"transfer-encoding",
"upgrade",
"te",
"keep-alive",
"proxy-authorization",
"proxy-connection",
"accept-encoding",
}
# 只接受自己能带上限解压的编码;httpx 默认还会协商 br / zstd,那些解码器没有输出上限
ACCEPT_ENCODING = "gzip, deflate"
# is_global 会放过的几段:NAT64(可以嵌入任意 IPv4)、IPv4 兼容地址、已废弃的站点本地地址
EXTRA_BLOCKED = [
ipaddress.ip_network(net)
for net in ("64:ff9b::/96", "64:ff9b:1::/48", "::/96", "fec0::/10")
]
# 十进制、十六进制、省略写法的 IPv4(2130706433、0x7f.1、127.1):不是标准写法,
# 各平台的解析器解释不一,一律当作内网地址拒绝,不交给 DNS
NUMERIC_LABEL = re.compile(r"0x[0-9a-f]+|[0-9]+", re.IGNORECASE)
Method = Literal["GET", "POST"]
def _blocked() -> BizException:
return BizException(422, "不允许访问内网或本机地址")
def _is_public(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
# IPv4 映射的 IPv6 地址按内嵌的 IPv4 判断
if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None:
address = address.ipv4_mapped
return (
address.is_global
and not address.is_multicast
and not any(address in net for net in EXTRA_BLOCKED)
)
def _checked(
address: ipaddress.IPv4Address | ipaddress.IPv6Address,
) -> ipaddress.IPv4Address | ipaddress.IPv6Address:
if not _is_public(address):
raise _blocked()
return address
async def _resolve(host: str, port: int) -> str:
"""解析主机名,所有地址都是公网才放行,返回第一个地址"""
host = host.rstrip(".")
try:
literal = ipaddress.ip_address(host)
except ValueError:
pass
else:
return str(_checked(literal))
if host.lower() == "localhost" or host.lower().endswith(".localhost"):
raise _blocked()
if all(NUMERIC_LABEL.fullmatch(label) for label in host.split(".")):
raise _blocked()
try:
infos = await asyncio.get_running_loop().getaddrinfo(
host, port, type=socket.SOCK_STREAM
)
except socket.gaierror:
raise BizException(502, "数据源域名无法解析") from None
addresses = [ipaddress.ip_address(info[4][0].split("%")[0]) for info in infos]
if not addresses:
raise BizException(502, "数据源域名无法解析")
for address in addresses:
_checked(address)
return str(addresses[0])
def _clean_headers(headers: dict[str, Any] | None) -> dict[str, str]:
cleaned = {"Accept": "application/json", "Accept-Encoding": ACCEPT_ENCODING}
for name, value in (headers or {}).items():
if name.lower() in FORBIDDEN_HEADERS or not isinstance(
value, (str, int, float)
):
continue
cleaned[name] = str(value)
return cleaned
def _query_value(value: Any) -> Any:
return value if isinstance(value, (str, int, float, bool)) else json.dumps(value)
def _parse_url(raw: str) -> httpx.URL:
try:
url = httpx.URL(raw)
except httpx.InvalidURL:
raise BizException(422, "数据源地址不合法") from None
if url.scheme not in ("http", "https") or not url.host or url.userinfo:
raise BizException(422, "数据源地址只支持 http / https,且不能带账号密码")
return url
def _too_large() -> BizException:
return BizException(502, "数据源响应超过 2 MB")
async def _read_body(response: httpx.Response) -> bytes:
"""读响应体:原始字节和解压后的字节都不超过 MAX_BYTES
不用 aiter_bytes:它对每个 64 KB 的块整块解压,没有输出上限,一个块就能膨胀到上 GB
"""
encoding = response.headers.get("content-encoding", "identity").strip().lower()
if encoding in ("", "identity"):
decoder = None
elif encoding in ("gzip", "x-gzip"):
decoder = zlib.decompressobj(16 + zlib.MAX_WBITS)
elif encoding == "deflate":
decoder = zlib.decompressobj()
else:
raise BizException(502, "数据源响应的压缩格式不支持")
declared = response.headers.get("content-length", "")
if declared.isdigit() and int(declared) > MAX_BYTES:
raise _too_large()
chunks: list[bytes] = []
raw_size = size = 0
try:
async for chunk in response.aiter_raw():
raw_size += len(chunk)
if raw_size > MAX_BYTES:
raise _too_large()
if decoder is None:
chunks.append(chunk)
size += len(chunk)
continue
pending = chunk
while pending:
# max_length 为 0 表示不限,这里恒大于 0
part = decoder.decompress(pending, MAX_BYTES - size + 1)
size += len(part)
if size > MAX_BYTES:
raise _too_large()
chunks.append(part)
pending = decoder.unconsumed_tail
if not part:
break
# 压缩流没有正常结束(被截断)也算失败,别把半截数据当成功
if decoder is not None and not decoder.eof:
raise BizException(502, "数据源响应解压失败")
except zlib.error:
raise BizException(502, "数据源响应解压失败") from None
return b"".join(chunks)
async def _request(
client: httpx.AsyncClient,
method: str,
url: httpx.URL,
headers: dict[str, str],
body: Any,
) -> tuple[int, httpx.Headers, bytes]:
port = url.port or (443 if url.scheme == "https" else 80)
pinned = url.copy_with(host=await _resolve(url.host, port))
request = client.build_request(
method,
pinned,
headers={**headers, "Host": url.netloc.decode()},
json=body if method == "POST" else None,
# 连接用 IP,TLS 的 SNI 和证书校验仍然用原主机名
extensions={"sni_hostname": url.host},
)
response = await client.send(request, stream=True)
try:
return response.status_code, response.headers, await _read_body(response)
finally:
await response.aclose()
def _origin(url: httpx.URL) -> tuple[str, str, int | None]:
return url.scheme, url.host, url.port or (443 if url.scheme == "https" else 80)
async def fetch_json(
url: str,
method: Method = "GET",
params: dict[str, Any] | None = None,
headers: dict[str, Any] | None = None,
) -> dict[str, Any] | list[Any]:
"""GET 的 params 作为查询参数,POST 的 params 作为 JSON 请求体,和前端 fetchData 一致"""
target = _parse_url(url)
request_headers = _clean_headers(headers)
body = None
if method == "GET":
if params:
target = target.copy_merge_params(
{key: _query_value(value) for key, value in params.items()}
)
else:
body = params or {}
try:
async with asyncio.timeout(TIMEOUT):
# trust_env=False:环境里的代理会绕过上面的地址检查
# max_keepalive_connections=0:连接按 IP 复用,不关的话同一 IP 上的下一跳会沿用上一跳的 TLS 校验结果
async with httpx.AsyncClient(
trust_env=False,
timeout=TIMEOUT,
limits=httpx.Limits(max_keepalive_connections=0),
) as client:
for _ in range(MAX_REDIRECTS + 1):
status, response_headers, content = await _request(
client, method, target, request_headers, body
)
location = response_headers.get("location")
if status not in REDIRECT_STATUS or not location:
break
previous = target
target = _parse_url(urljoin(str(target), location))
if _origin(target) != _origin(previous):
# 密钥头只发给调用方指定的那个源
request_headers = _clean_headers(None)
if status not in (307, 308):
method, body = "GET", None
else:
raise BizException(502, "数据源重定向次数过多")
except (TimeoutError, httpx.TimeoutException):
raise BizException(504, "数据源请求超时") from None
except httpx.HTTPError:
raise BizException(502, "数据源请求失败") from None
if not 200 <= status < 300:
raise BizException(502, f"数据源返回 HTTP {status}")
try:
data = json.loads(content)
except (ValueError, RecursionError):
# 2 MB 以内的深层嵌套会打满递归,同样当作非法 JSON
raise BizException(502, "数据源返回的不是 JSON") from None
if not isinstance(data, (dict, list)):
raise BizException(502, "数据源返回的不是 JSON 对象或数组")
return data
class FetchCache:
"""线上大屏多人同时开、按 interval 轮询:同一个 key 几秒内只真正请求一次,并发的请求共用同一次
失败不缓存;单实例部署,缓存放进程内存。单条最大 2 MB,条数上限乘出来就是内存上限,所以条数要小
"""
def __init__(self, ttl: float = 3.0, max_entries: int = 64):
self._ttl = ttl
self._max_entries = max_entries
self._entries: dict[Any, tuple[float, Any]] = {}
self._inflight: dict[Any, asyncio.Future] = {}
async def get(self, key: Any, fetch: Callable[[], Awaitable[Any]]) -> Any:
now = time.monotonic()
hit = self._entries.get(key)
if hit is not None and hit[0] > now:
return hit[1]
future = self._inflight.get(key)
if future is None:
future = asyncio.ensure_future(fetch())
self._inflight[key] = future
future.add_done_callback(lambda done: self._finish(key, done))
# shield:某个请求者断开不能取消共用的那次请求
return await asyncio.shield(future)
def _finish(self, key: Any, future: asyncio.Future) -> None:
self._inflight.pop(key, None)
if future.cancelled() or future.exception() is not None:
return
now = time.monotonic()
# 每次写入先丢掉过期项;清完还满就不缓存这一条
self._entries = {k: v for k, v in self._entries.items() if v[0] > now}
if len(self._entries) < self._max_entries:
self._entries[key] = (now + self._ttl, future.result())
+238
View File
@@ -0,0 +1,238 @@
"""大屏存储的 MongoDB 实现,方法语义与内存实现一致
集合:screens(草稿,一屏一条)、screen_publishes(发布快照,只增不改)
"""
import re
from contextlib import contextmanager
from datetime import UTC
from typing import Any
from bson.codec_options import CodecOptions
from bson.errors import InvalidDocument
from pydantic import BaseModel
from pymongo import ASCENDING, DESCENDING, ReturnDocument
from pymongo.errors import DocumentTooLarge, DuplicateKeyError
from app.core.exceptions import BizException
from app.schemas.saved_screen import (
PublishEntry,
PublishSnapshot,
ScreenRecord,
ScreenSummary,
)
from app.services.screen_store import canvas_size
# 列表和元数据更新不需要页面,只取画布;Mongo 的投影不能同时"排除 page"和"包含 page.canvas",所以列字段
SUMMARY_PROJECTION = {
"owner_id": 1,
"name": 1,
"thumbnail": 1,
"page.canvas": 1,
"revision": 1,
"created_at": 1,
"updated_at": 1,
"deleted_at": 1,
"publish": 1,
}
@contextmanager
def _storable():
"""页面里的超大整数、含 NUL 的字段名、编码后超过 16 MB 的文档,驱动会抛各种非业务异常
10 MB 的 JSON 编码成 BSON 后会涨(数组的下标要当字符串键存),所以体积上限靠这里兜底
"""
try:
yield
except (OverflowError, InvalidDocument, DocumentTooLarge):
raise BizException(
422, "大屏数据无法保存:数字超出范围、字段名不合法或体积过大"
) from None
# update 自己拼的条件,调用方的 where 不许覆盖
RESERVED_FILTER_KEYS = {"_id", "owner_id", "deleted_at", "revision"}
def _owned(screen_id: str, owner_id: int, deleted: bool) -> dict[str, Any]:
return {
"_id": screen_id,
"owner_id": owner_id,
"deleted_at": {"$ne": None} if deleted else None,
}
def _encode(value: Any) -> Any:
return value.model_dump() if isinstance(value, BaseModel) else value
def _summary(document: dict[str, Any]) -> ScreenSummary:
canvas = (document.get("page") or {}).get("canvas")
data = {k: v for k, v in document.items() if k not in ("_id", "page")}
return ScreenSummary.model_validate(
{**data, "screen_id": document["_id"], "canvas": canvas_size(canvas)}
)
class MongoScreenStore:
def __init__(self, db: Any, prefix: str = ""):
# 读回的时间统一带 UTC 时区;集合名前缀用于测试隔离
options = CodecOptions(tz_aware=True, tzinfo=UTC)
self._screens = db.get_collection(f"{prefix}screens", codec_options=options)
self._publishes = db.get_collection(
f"{prefix}screen_publishes", codec_options=options
)
async def ensure_indexes(self) -> None:
await self._screens.create_index(
[
("owner_id", ASCENDING),
("deleted_at", ASCENDING),
("updated_at", DESCENDING),
]
)
# 分享链接唯一;没发布过的大屏没有 publish.share_id,不进索引
await self._screens.create_index(
"publish.share_id",
unique=True,
partialFilterExpression={"publish.share_id": {"$type": "string"}},
)
await self._publishes.create_index(
[("screen_id", ASCENDING), ("version", DESCENDING)], unique=True
)
# 回收站过期清理按 deleted_at 跨用户扫描;没删除的不进索引
await self._screens.create_index(
"deleted_at", partialFilterExpression={"deleted_at": {"$type": "date"}}
)
async def insert(self, record: ScreenRecord) -> None:
document = record.model_dump(exclude={"screen_id", "canvas"})
with _storable():
await self._screens.insert_one({"_id": record.screen_id, **document})
async def get(self, screen_id, owner_id, *, deleted=False):
document = await self._screens.find_one(_owned(screen_id, owner_id, deleted))
if document is None:
return None
return ScreenRecord.model_validate(
{**_summary(document).model_dump(), "page": document["page"]}
)
async def get_summary(self, screen_id, owner_id, *, deleted=False):
document = await self._screens.find_one(
_owned(screen_id, owner_id, deleted), SUMMARY_PROJECTION
)
return None if document is None else _summary(document)
async def update(
self, screen_id, owner_id, changes, *, revision=None, where=None, deleted=False
):
query = _owned(screen_id, owner_id, deleted)
if revision is not None:
query["revision"] = revision
if clash := RESERVED_FILTER_KEYS & (where or {}).keys():
raise ValueError(f"where 不能覆盖所有权和版本条件:{sorted(clash)}")
query.update(where or {})
with _storable():
document = await self._screens.find_one_and_update(
query,
{"$set": {key: _encode(value) for key, value in changes.items()}},
projection=SUMMARY_PROJECTION,
return_document=ReturnDocument.AFTER,
)
return None if document is None else _summary(document)
async def search(
self, owner_id, *, keyword, status, sort, descending, limit, offset, deleted
):
query: dict[str, Any] = {
"owner_id": owner_id,
"deleted_at": {"$ne": None} if deleted else None,
}
if keyword:
query["name"] = {"$regex": re.escape(keyword), "$options": "i"}
if status == "draft":
query["publish"] = None
elif status is not None:
query["publish.status"] = status
direction = DESCENDING if descending else ASCENDING
documents = await (
self._screens.find(query, SUMMARY_PROJECTION)
.sort([(sort, direction), ("_id", direction)])
.skip(offset)
.limit(limit)
.to_list()
)
return [_summary(d) for d in documents], await self._screens.count_documents(
query
)
async def get_by_share_id(self, share_id):
document = await self._screens.find_one(
{"publish.share_id": share_id, "deleted_at": None}, SUMMARY_PROJECTION
)
return None if document is None else _summary(document)
async def list_expired_trash(self, deleted_before, limit):
documents = await (
self._screens.find(
# 带 $type 才蕴含部分索引的过滤条件,查询才用得上索引
{"deleted_at": {"$type": "date", "$lt": deleted_before}},
SUMMARY_PROJECTION,
)
.sort("deleted_at", ASCENDING)
.limit(limit)
.to_list()
)
return [_summary(d) for d in documents]
async def purge(self, screen_id, owner_id):
# 带"仍在回收站"的条件:清理期间被还原的大屏不能删
result = await self._screens.delete_one(_owned(screen_id, owner_id, True))
return result.deleted_count == 1
async def delete_publishes(self, screen_id):
await self._publishes.delete_many({"screen_id": screen_id})
async def last_publish_version(self, screen_id):
document = await self._publishes.find_one(
{"screen_id": screen_id}, {"version": 1}, sort=[("version", DESCENDING)]
)
return 0 if document is None else document["version"]
async def insert_publish(self, snapshot):
try:
with _storable():
await self._publishes.insert_one(snapshot.model_dump())
except DuplicateKeyError:
return False
return True
async def get_publish(self, screen_id, version):
document = await self._publishes.find_one(
{"screen_id": screen_id, "version": version}
)
if document is None:
return None
return PublishSnapshot.model_validate(
{k: v for k, v in document.items() if k != "_id"}
)
async def list_publishes(self, screen_id, limit, offset):
query = {"screen_id": screen_id}
documents = await (
self._publishes.find(query, {"page": 0})
.sort("version", DESCENDING)
.skip(offset)
.limit(limit)
.to_list()
)
entries = [
PublishEntry.model_validate({k: v for k, v in d.items() if k != "_id"})
for d in documents
]
return entries, await self._publishes.count_documents(query)
async def delete_publish(self, screen_id, version):
await self._publishes.delete_one({"screen_id": screen_id, "version": version})
+241
View File
@@ -0,0 +1,241 @@
"""大屏存储:协议和进程内存实现;MongoDB 实现见 screen_mongo
内存实现只给测试用——大屏数据不能丢,没配 MongoDB 时 screens 接口直接返回 503
所有查询都带 owner_id:别人的大屏与不存在同样返回 None
"""
from datetime import datetime
from typing import Any, Protocol
from app.schemas.saved_screen import (
ListStatus,
PublishEntry,
PublishSnapshot,
ScreenRecord,
ScreenSummary,
SortField,
)
def canvas_size(canvas: Any) -> dict[str, Any] | None:
"""列表卡片只需要画布宽高"""
if not isinstance(canvas, dict):
return None
return {key: canvas[key] for key in ("width", "height") if key in canvas}
def summarize(record: ScreenRecord) -> ScreenSummary:
"""去掉页面,只保留画布宽高"""
data = record.model_dump(exclude={"page"})
data["canvas"] = canvas_size(record.page.get("canvas"))
return ScreenSummary.model_validate(data)
class ScreenStore(Protocol):
async def insert(self, record: ScreenRecord) -> None: ...
async def get(
self, screen_id: str, owner_id: int, *, deleted: bool = False
) -> ScreenRecord | None:
"""带页面的记录;deleted 为 True 时只找回收站里的"""
...
async def get_summary(
self, screen_id: str, owner_id: int, *, deleted: bool = False
) -> ScreenSummary | None: ...
async def update(
self,
screen_id: str,
owner_id: int,
changes: dict[str, Any],
*,
revision: int | None = None,
where: dict[str, Any] | None = None,
deleted: bool = False,
) -> ScreenSummary | None:
"""一步完成"匹配并更新";不匹配返回 None
给了 revision 就要求库里的 revision 与之相等;where 是额外的等值条件,键可以带点,
如 {"publish.version": 3}、{"publish": None}——更新是基于先前读到的值做的,值变了就不能盖上去
"""
...
async def search(
self,
owner_id: int,
*,
keyword: str | None,
status: ListStatus | None,
sort: SortField,
descending: bool,
limit: int,
offset: int,
deleted: bool,
) -> tuple[list[ScreenSummary], int]: ...
async def get_by_share_id(self, share_id: str) -> ScreenSummary | None:
"""未删除的大屏;是否已下线由调用方看 publish.status"""
...
async def list_expired_trash(
self, deleted_before: datetime, limit: int
) -> list[ScreenSummary]:
"""所有人的回收站里 deleted_at 早于 deleted_before 的大屏"""
...
async def purge(self, screen_id: str, owner_id: int) -> bool:
"""彻底删除回收站里的大屏;已经被还原(不在回收站)或不存在时返回 False"""
...
async def delete_publishes(self, screen_id: str) -> None: ...
async def last_publish_version(self, screen_id: str) -> int: ...
async def insert_publish(self, snapshot: PublishSnapshot) -> bool:
"""同一大屏同一版本号已存在时返回 False"""
...
async def get_publish(
self, screen_id: str, version: int
) -> PublishSnapshot | None: ...
async def list_publishes(
self, screen_id: str, limit: int, offset: int
) -> tuple[list[PublishEntry], int]:
"""版本号倒序"""
...
async def delete_publish(self, screen_id: str, version: int) -> None: ...
class InMemoryScreenStore:
def __init__(self) -> None:
self._screens: dict[str, ScreenRecord] = {}
self._publishes: dict[tuple[str, int], PublishSnapshot] = {}
def _find(
self, screen_id: str, owner_id: int, deleted: bool
) -> ScreenRecord | None:
record = self._screens.get(screen_id)
if record is None or record.owner_id != owner_id:
return None
return record if (record.deleted_at is not None) == deleted else None
async def insert(self, record: ScreenRecord) -> None:
self._screens[record.screen_id] = record.model_copy(deep=True)
async def get(self, screen_id, owner_id, *, deleted=False):
record = self._find(screen_id, owner_id, deleted)
return None if record is None else record.model_copy(deep=True)
async def get_summary(self, screen_id, owner_id, *, deleted=False):
record = self._find(screen_id, owner_id, deleted)
return None if record is None else summarize(record)
async def update(
self, screen_id, owner_id, changes, *, revision=None, where=None, deleted=False
):
record = self._find(screen_id, owner_id, deleted)
if record is None or (revision is not None and record.revision != revision):
return None
if where is not None and not _matches(record, where):
return None
updated = record.model_copy(update=changes, deep=True)
self._screens[screen_id] = updated
return summarize(updated)
async def search(
self, owner_id, *, keyword, status, sort, descending, limit, offset, deleted
):
records = [
record
for record in self._screens.values()
if record.owner_id == owner_id
and (record.deleted_at is not None) == deleted
and (keyword is None or keyword.lower() in record.name.lower())
and _status_matches(record, status)
]
# 排序字段相同时按 ID 兜底,翻页才稳定
records.sort(key=lambda r: (getattr(r, sort), r.screen_id), reverse=descending)
return [summarize(r) for r in records[offset : offset + limit]], len(records)
async def get_by_share_id(self, share_id):
for record in self._screens.values():
if (
record.deleted_at is None
and record.publish is not None
and record.publish.share_id == share_id
):
return summarize(record)
return None
async def list_expired_trash(self, deleted_before, limit):
expired = sorted(
(
r
for r in self._screens.values()
if r.deleted_at is not None and r.deleted_at < deleted_before
),
key=lambda r: r.deleted_at,
)
return [summarize(r) for r in expired[:limit]]
async def purge(self, screen_id, owner_id):
if self._find(screen_id, owner_id, True) is None:
return False
del self._screens[screen_id]
return True
async def delete_publishes(self, screen_id):
for key in [key for key in self._publishes if key[0] == screen_id]:
del self._publishes[key]
async def last_publish_version(self, screen_id):
return max(
(version for sid, version in self._publishes if sid == screen_id), default=0
)
async def insert_publish(self, snapshot):
key = (snapshot.screen_id, snapshot.version)
if key in self._publishes:
return False
self._publishes[key] = snapshot.model_copy(deep=True)
return True
async def get_publish(self, screen_id, version):
snapshot = self._publishes.get((screen_id, version))
return None if snapshot is None else snapshot.model_copy(deep=True)
async def list_publishes(self, screen_id, limit, offset):
snapshots = sorted(
(s for (sid, _), s in self._publishes.items() if sid == screen_id),
key=lambda s: s.version,
reverse=True,
)
entries = [
PublishEntry.model_validate(s.model_dump(exclude={"page"}))
for s in snapshots[offset : offset + limit]
]
return entries, len(snapshots)
async def delete_publish(self, screen_id, version):
self._publishes.pop((screen_id, version), None)
def _matches(record: ScreenRecord, where: dict[str, Any]) -> bool:
for path, expected in where.items():
value: Any = record
for part in path.split("."):
value = None if value is None else getattr(value, part, None)
if value != expected:
return False
return True
def _status_matches(record: ScreenSummary, status: ListStatus | None) -> bool:
if status is None:
return True
if status == "draft":
return record.publish is None
return record.publish is not None and record.publish.status == status
+562
View File
@@ -0,0 +1,562 @@
"""大屏的业务逻辑:存取、乐观锁、发布、历史、复制、封面、线上读取和数据源取数
存储层只提供"匹配并更新"这类原语,404 / 409 的判断都在这里
"""
import asyncio
import copy
import secrets
import weakref
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from typing import Any, Protocol
from uuid import uuid4
from loguru import logger
from pydantic import ValidationError
from app.core.exceptions import BizException
from app.schemas.saved_screen import (
NAME_MAX_LENGTH,
ListStatus,
PublishEntry,
PublishInfo,
PublishSnapshot,
ScreenRecord,
ScreenSummary,
SortField,
)
from app.schemas.screen import Screen
from app.services.datasource_proxy import FetchCache, fetch_json
from app.services.screen_store import ScreenStore
from app.services.thumbnails import ThumbnailStorage
# 事件代码和数据源转换函数是字符串形式的 JS,线上会在浏览器里执行;写入权限只给 owner,再限制长度
SCRIPT_MAX_CHARS = 64 * 1024
COPY_SUFFIX = " 副本"
EMPTY_PAGE = {
"canvas": {"width": 1920, "height": 1080, "backgroundColor": "#ffffff10"},
"nodes": [],
"dataSource": [],
}
# 回收站过期清理:一批最多清多少个,两次扫描的间隔(秒)
PURGE_BATCH = 100
TRASH_SWEEP_INTERVAL = 3600
STALE = "大屏已在别处修改"
# 下线 / 删除 / 换封面遇到并发修改时的重读次数
UPDATE_ATTEMPTS = 3
def _now() -> datetime:
return datetime.now(UTC)
class FilePinner(Protocol):
"""数据文件服务里和大屏有关的两个方法"""
async def sync_screen_files(
self, user_id: int, screen_id: str, file_ids: set[str]
) -> None: ...
async def release_screen_files(self, screen_id: str) -> None: ...
def referenced_file_ids(page: dict[str, Any]) -> set[str]:
"""页面里图表绑定的数据文件:nodes[].fileQuery.file_id"""
ids = set()
nodes = page.get("nodes")
for node in nodes if isinstance(nodes, list) else []:
query = node.get("fileQuery") if isinstance(node, dict) else None
file_id = query.get("file_id") if isinstance(query, dict) else None
if isinstance(file_id, str) and file_id:
ids.add(file_id)
return ids
def _publish_is(publish: PublishInfo | None) -> dict[str, Any]:
""" "线上指针还是我读到的那个"的更新条件"""
if publish is None:
return {"publish": None}
return {"publish.version": publish.version, "publish.status": publish.status}
def _not_found() -> BizException:
# 别人的大屏与不存在同样返回 404
return BizException(404, "大屏不存在或已删除")
def validate_page(page: dict[str, Any]) -> dict[str, Any]:
"""按 Screen 校验结构,但返回的仍是原始 JSON:model_dump 会给没传的字段补出 null,前端读回来就变了"""
try:
Screen.model_validate(page)
except ValidationError as error:
first = error.errors()[0]
where = ".".join(str(part) for part in first["loc"])
raise BizException(422, f"大屏数据不合法:{where}:{first['msg']}") from None
_check_script_length(page)
return page
def _check_script_length(page: dict[str, Any]) -> None:
scripts = [
event.get("code")
for node in page["nodes"]
for event in (node.get("events") or [])
if isinstance(event, dict)
] + [
source.get("transform")
for source in page.get("dataSource", [])
if isinstance(source, dict)
]
if any(isinstance(text, str) and len(text) > SCRIPT_MAX_CHARS for text in scripts):
raise BizException(
422, f"事件代码和数据转换函数不能超过 {SCRIPT_MAX_CHARS // 1024} KB"
)
def public_page(page: dict[str, Any]) -> dict[str, Any]:
"""线上页面里去掉数据源的 headers:它们常放 API key,线上取数由服务端按快照里的完整配置发起,浏览器用不着"""
sources = page.get("dataSource")
if not isinstance(sources, list):
return {**page}
return {
**page,
"dataSource": [
{key: value for key, value in item.items() if key != "headers"}
if isinstance(item, dict)
else item
for item in sources
],
}
class ScreenService:
def __init__(
self,
store: ScreenStore,
thumbnails: ThumbnailStorage,
files: FilePinner | None = None,
trash_days: int = 30,
):
self._store = store
self._thumbnails = thumbnails
# 已保存的大屏引用的数据文件不能过期,否则"刷新数据"会 404;没有文件服务时不管
self._files = files
if trash_days < 1:
raise ValueError("trash_days 必须是正整数")
self.trash_days = trash_days
# 按大屏的锁:保存 / 回滚 / 删除 / 还原都是"改库再同步固定的文件"两步,
# 两个请求交错会让固定的文件跟着先开始、后结束的那个请求的旧页面走。单实例部署,进程内锁就够
self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = (
weakref.WeakValueDictionary()
)
self._datasource_cache = FetchCache()
def _lock(self, screen_id: str) -> asyncio.Lock:
lock = self._locks.get(screen_id)
if lock is None:
lock = self._locks[screen_id] = asyncio.Lock()
return lock
async def _pin_files(self, owner_id: int, screen_id: str, page: dict[str, Any]):
"""让数据文件跟着大屏的引用走。主操作已经成功,这一步失败只记日志,不让保存报错"""
if self._files is None:
return
try:
await self._files.sync_screen_files(
owner_id, screen_id, referenced_file_ids(page)
)
except Exception as error:
logger.opt(exception=error).error(
"同步大屏 {} 引用的数据文件失败", screen_id
)
async def _release_files(self, screen_id: str) -> None:
if self._files is None:
return
try:
await self._files.release_screen_files(screen_id)
except Exception as error:
logger.opt(exception=error).error(
"放开大屏 {} 固定的数据文件失败", screen_id
)
# ---- 草稿 ----
async def create(
self, owner_id: int, name: str, page: dict[str, Any] | None
) -> ScreenRecord:
now = _now()
record = ScreenRecord(
screen_id=f"s_{uuid4().hex}",
owner_id=owner_id,
name=name,
page=validate_page(copy.deepcopy(EMPTY_PAGE) if page is None else page),
revision=1,
created_at=now,
updated_at=now,
)
await self._store.insert(record)
await self._pin_files(owner_id, record.screen_id, record.page)
return record
async def get(self, screen_id: str, owner_id: int) -> ScreenRecord:
record = await self._store.get(screen_id, owner_id)
if record is None:
raise _not_found()
return record
async def save(
self, screen_id: str, owner_id: int, page: dict[str, Any], revision: int
) -> ScreenSummary:
validate_page(page)
async with self._lock(screen_id):
saved = await self._update_with_revision(
screen_id,
owner_id,
{"page": page, "revision": revision + 1, "updated_at": _now()},
revision,
)
await self._pin_files(owner_id, screen_id, page)
return saved
async def _update_with_revision(
self, screen_id: str, owner_id: int, changes: dict[str, Any], revision: int
) -> ScreenSummary:
updated = await self._store.update(
screen_id, owner_id, changes, revision=revision
)
if updated is not None:
return updated
# 没更新成功:大屏不存在是 404,存在但 revision 不一致是 409
if await self._store.get_summary(screen_id, owner_id) is None:
raise _not_found()
raise BizException(409, STALE)
async def rename(self, screen_id: str, owner_id: int, name: str) -> None:
# 改名不算改内容,revision 不动
changes = {"name": name, "updated_at": _now()}
if await self._store.update(screen_id, owner_id, changes) is None:
raise _not_found()
async def _update_from_summary(
self,
screen_id: str,
owner_id: int,
build: Callable[[ScreenSummary], tuple[dict[str, Any], dict[str, Any]] | None],
) -> None:
"""读出当前状态,据此算出修改,再带着"状态没变"的条件写回;并发改了就重读再来
下线、删除要整段改写 publish,不带条件就会把并发发布写进去的新指针盖回旧值
build 返回 (changes, where);返回 None 表示已经是目标状态,什么都不用做
"""
for _ in range(UPDATE_ATTEMPTS):
summary = await self._store.get_summary(screen_id, owner_id)
if summary is None:
raise _not_found()
built = build(summary)
if built is None:
return
changes, where = built
if await self._store.update(screen_id, owner_id, changes, where=where):
return
raise BizException(409, "操作冲突,请重试")
async def delete(self, screen_id: str, owner_id: int) -> None:
def build(summary: ScreenSummary):
changes: dict[str, Any] = {"deleted_at": _now()}
if summary.publish is None:
return changes, {"publish": None}
# 线上版本同时下线,分享链接立即 404
changes["publish"] = summary.publish.model_copy(
update={"status": "offline"}
)
return changes, _publish_is(summary.publish)
async with self._lock(screen_id):
await self._update_from_summary(screen_id, owner_id, build)
# 进了回收站的大屏不再固定文件,还原时再固定回来
await self._release_files(screen_id)
async def restore_deleted(self, screen_id: str, owner_id: int) -> None:
async with self._lock(screen_id):
record = await self._store.get(screen_id, owner_id, deleted=True)
if record is None:
raise _not_found()
changes = {"deleted_at": None}
if (
await self._store.update(screen_id, owner_id, changes, deleted=True)
is None
):
raise _not_found()
await self._pin_files(owner_id, screen_id, record.page)
# ---- 回收站 ----
async def _purge(self, summary: ScreenSummary) -> bool:
"""彻底删除回收站里的一个大屏
先删大屏本身(带"仍在回收站"的条件,清理期间被还原的不会误删),再清理附属数据
"""
if not await self._store.purge(summary.screen_id, summary.owner_id):
return False
await self._store.delete_publishes(summary.screen_id)
self._thumbnails.delete(summary.thumbnail)
await self._release_files(summary.screen_id)
return True
async def empty_trash(self, owner_id: int) -> int:
"""清空自己的回收站,返回彻底删除的个数"""
count = 0
previous: set[str] = set()
while True:
batch, _ = await self._store.search(
owner_id,
keyword=None,
status=None,
sort="updated_at",
descending=True,
limit=PURGE_BATCH,
offset=0,
deleted=True,
)
ids = {summary.screen_id for summary in batch}
purged = [await self._purge(summary) for summary in batch]
count += sum(purged)
# 空批才算清完;整批都没删成(被并发还原了)就再查一次,
# 但同一批连续两次都删不掉说明有东西卡住,别空转
if not batch or (not any(purged) and ids == previous):
return count
previous = ids
async def purge_expired_trash(self) -> int:
"""彻底删除在回收站里超过保留天数的大屏(所有人的)"""
deadline = _now() - timedelta(days=self.trash_days)
count = 0
previous: set[str] = set()
while True:
batch = await self._store.list_expired_trash(deadline, PURGE_BATCH)
ids = {summary.screen_id for summary in batch}
purged = [await self._purge(summary) for summary in batch]
count += sum(purged)
if not batch or (not any(purged) and ids == previous):
return count
previous = ids
async def run_trash_sweeper(self) -> None:
while True:
try:
if removed := await self.purge_expired_trash():
logger.info(
"已彻底删除 {} 个超过 {} 天的回收站大屏",
removed,
self.trash_days,
)
except Exception as error:
logger.opt(exception=error).error("清理回收站失败")
await asyncio.sleep(TRASH_SWEEP_INTERVAL)
# ---- 管理页 ----
async def list_screens(
self,
owner_id: int,
*,
keyword: str | None,
status: ListStatus | None,
sort: SortField,
descending: bool,
limit: int,
offset: int,
deleted: bool,
) -> tuple[list[ScreenSummary], int]:
return await self._store.search(
owner_id,
keyword=keyword,
status=status,
sort=sort,
descending=descending,
limit=limit,
offset=offset,
deleted=deleted,
)
async def copy_screen(self, screen_id: str, owner_id: int, name: str | None) -> str:
"""只复制草稿和封面,不复制发布状态和历史"""
source = await self.get(screen_id, owner_id)
base = source.name[: NAME_MAX_LENGTH - len(COPY_SUFFIX)]
copied = await self.create(
owner_id,
name or f"{base}{COPY_SUFFIX}",
copy.deepcopy(source.page),
)
if source.thumbnail is not None:
thumbnail = self._thumbnails.copy(source.thumbnail)
if thumbnail is not None:
try:
attached = await self._store.update(
copied.screen_id, owner_id, {"thumbnail": thumbnail}
)
except BaseException:
self._thumbnails.delete(thumbnail)
raise
if not attached:
self._thumbnails.delete(thumbnail)
return copied.screen_id
async def set_thumbnail(self, screen_id: str, owner_id: int, data: bytes) -> str:
thumbnail = self._thumbnails.save(data)
replaced: list[str | None] = []
def build(summary: ScreenSummary):
# 封面只是展示用的,不算改内容:不动 revision 和 updated_at
replaced[:] = [summary.thumbnail]
return {"thumbnail": thumbnail}, {"thumbnail": summary.thumbnail}
try:
await self._update_from_summary(screen_id, owner_id, build)
except BaseException:
self._thumbnails.delete(thumbnail)
raise
# 删的是这次真正换掉的那张;并发上传时各自换掉各自读到的,不会留下孤儿文件
self._thumbnails.delete(replaced[0])
return thumbnail
def open_thumbnail(self, thumbnail_id: str):
return self._thumbnails.open(thumbnail_id)
# ---- 发布 ----
async def publish(
self, screen_id: str, owner_id: int, revision: int
) -> PublishInfo:
record = await self.get(screen_id, owner_id)
if record.revision != revision:
raise BizException(409, "大屏有未保存或已在别处修改的内容,请先保存")
previous = record.publish
snapshot = PublishSnapshot(
screen_id=screen_id,
# 版本号取已有快照的最大值加一:写快照后没来得及更新指针留下的孤儿快照也不会撞号
version=await self._store.last_publish_version(screen_id) + 1,
revision=record.revision,
page=record.page,
published_at=_now(),
published_by=owner_id,
)
if not await self._store.insert_publish(snapshot):
raise BizException(409, "发布冲突,请重试")
info = PublishInfo(
status="published",
version=snapshot.version,
revision=snapshot.revision,
# 第一次发布生成 share_id,之后沿用;下线再发布也沿用
share_id=previous.share_id
if previous
else f"p_{secrets.token_urlsafe(16)}",
published_at=snapshot.published_at,
)
# 条件:草稿没被再保存过(revision),线上指针还是读到的那个——并发发布只有一个能成,
# 否则第一次发布时两边各生成一个 share_id,先返回给前端的那个会失效
try:
updated = await self._store.update(
screen_id,
owner_id,
{"publish": info},
revision=revision,
where=_publish_is(previous),
)
except BaseException:
await self._store.delete_publish(screen_id, snapshot.version)
raise
if updated is None:
await self._store.delete_publish(screen_id, snapshot.version)
raise BizException(409, STALE)
return info
async def unpublish(self, screen_id: str, owner_id: int) -> None:
def build(summary: ScreenSummary):
if summary.publish is None or summary.publish.status == "offline":
return None
offline = summary.publish.model_copy(update={"status": "offline"})
return {"publish": offline}, _publish_is(summary.publish)
await self._update_from_summary(screen_id, owner_id, build)
async def exists(self, screen_id: str, owner_id: int) -> bool:
return await self._store.get_summary(screen_id, owner_id) is not None
async def list_publishes(
self, screen_id: str, owner_id: int, limit: int, offset: int
) -> tuple[list[PublishEntry], int]:
if await self._store.get_summary(screen_id, owner_id) is None:
raise _not_found()
return await self._store.list_publishes(screen_id, limit, offset)
async def restore_publish(self, screen_id: str, owner_id: int, version: int) -> int:
"""把历史版本的页面覆盖回草稿,不自动发布"""
async with self._lock(screen_id):
record = await self.get(screen_id, owner_id)
snapshot = await self._store.get_publish(screen_id, version)
if snapshot is None:
raise BizException(404, "发布版本不存在")
updated = await self._update_with_revision(
screen_id,
owner_id,
{
"page": snapshot.page,
"revision": record.revision + 1,
"updated_at": _now(),
},
record.revision,
)
await self._pin_files(owner_id, screen_id, snapshot.page)
return updated.revision
# ---- 线上(匿名) ----
async def _live(self, share_id: str) -> tuple[ScreenSummary, PublishSnapshot]:
"""线上版本:已下线 / 已删除 / 不存在都是 404;返回的是发布快照,不是草稿"""
summary = await self._store.get_by_share_id(share_id)
if (
summary is None
or summary.publish is None
or summary.publish.status != "published"
):
raise _not_found()
snapshot = await self._store.get_publish(
summary.screen_id, summary.publish.version
)
if snapshot is None:
raise _not_found()
return summary, snapshot
async def get_live(self, share_id: str) -> tuple[ScreenSummary, PublishSnapshot]:
return await self._live(share_id)
async def fetch_live_datasource(self, share_id: str, datasource_id: str):
"""前端只传数据源 ID,请求地址取自已发布快照;不能做成"前端传 URL 后端转发",否则是匿名开放代理"""
_, snapshot = await self._live(share_id)
source = next(
(
item
for item in snapshot.page.get("dataSource", [])
if isinstance(item, dict) and item.get("id") == datasource_id
),
None,
)
if source is None or source.get("type") != "api" or not source.get("url"):
raise BizException(404, "数据源不存在")
method = "POST" if source.get("method") == "POST" else "GET"
params = source.get("params")
headers = source.get("headers")
return await self._datasource_cache.get(
(share_id, snapshot.version, datasource_id),
lambda: fetch_json(
str(source["url"]),
method,
params if isinstance(params, dict) else None,
headers if isinstance(headers, dict) else None,
),
)
+63
View File
@@ -0,0 +1,63 @@
"""大屏封面图:放本地磁盘,文件名就是封面 ID;每次上传换新 ID,旧图删掉
图片类 URL 要能被 <img src> 匿名访问,靠不可猜的 ID 兜底
"""
import re
import secrets
from pathlib import Path
from app.core.exceptions import BizException
MAX_BYTES = 1024 * 1024
# 按文件头判断类型,不信任客户端声明的 Content-Type
MAGIC = {b"\x89PNG\r\n\x1a\n": "image/png", b"\xff\xd8\xff": "image/jpeg"}
# ID 进了文件路径,读写前必须过这个格式检查,防路径穿越
ID_PATTERN = re.compile(r"th_[A-Za-z0-9_-]{16,64}")
def sniff(head: bytes) -> str | None:
for magic, media_type in MAGIC.items():
if head.startswith(magic):
return media_type
return None
class ThumbnailStorage:
def __init__(self, root: Path):
self._root = root
def _path(self, thumbnail_id: str) -> Path | None:
if not ID_PATTERN.fullmatch(thumbnail_id):
return None
return self._root / thumbnail_id
def save(self, data: bytes) -> str:
if len(data) > MAX_BYTES:
raise BizException(413, "封面图不能超过 1 MB")
if sniff(data[:8]) is None:
raise BizException(422, "封面图只支持 PNG 或 JPEG")
thumbnail_id = f"th_{secrets.token_urlsafe(16)}"
self._root.mkdir(parents=True, exist_ok=True)
self._path(thumbnail_id).write_bytes(data)
return thumbnail_id
def open(self, thumbnail_id: str) -> tuple[Path, str] | None:
"""文件路径和图片类型;ID 不合法、文件不存在或内容不是图片时返回 None"""
path = self._path(thumbnail_id)
if path is None or not path.is_file():
return None
with path.open("rb") as file:
media_type = sniff(file.read(8))
return None if media_type is None else (path, media_type)
def copy(self, thumbnail_id: str) -> str | None:
path = self._path(thumbnail_id)
if path is None or not path.is_file():
return None
return self.save(path.read_bytes())
def delete(self, thumbnail_id: str | None) -> None:
path = None if thumbnail_id is None else self._path(thumbnail_id)
if path is not None:
path.unlink(missing_ok=True)