backend: keep a figure in the object store, and serve it back by key
A paper needs its experiment figures, and a figure is not text: the bytes go to
MinIO and the sentence keeps a Markdown address. That is what keeps a sentence
one editable string — a picture moves between paragraphs by being cut and
pasted, and survives a template switch with the sentence it belongs to.
Three decisions carry the safety, and each is a way to serve something other
than what was uploaded:
* the key is the digest of the bytes, so uploading the same screenshot twice is
one object and the address can never come to mean a different picture — which
is what makes the year-long `immutable` cache header safe;
* the format is sniffed from the magic bytes, not read from the client's
`Content-Type` or filename, and SVG is refused because an uploaded SVG served
from this origin is stored XSS;
* serving is restricted to the key space this application writes, so
`GET /api/images/{key}` cannot become a browser for the rest of the bucket.
Images are served through the API rather than from a public bucket URL so a
sentence never embeds a host name — the LAN in development and the public
address later would otherwise disagree — and the bucket can stay private. The
object is streamed rather than read into memory, and the connection is released
when the reader stops.
Storage unconfigured is a 503 that names the missing variables, not a crash:
the writing path does not depend on the bucket.
`scripts/smoke_images.py` walks the whole thing: each accepted format, the
refusals (text, SVG, a mislabelled file, an empty file, a key we did not
write), and a sentence that carries a figure through the paper endpoints.
This commit is contained in:
@@ -17,3 +17,18 @@ DB_NAME=paper_doc
|
||||
# The password must be percent-encoded (a raw '@' would be parsed as the
|
||||
# user/host separator). Prefer the DB_* parts above and let the app encode it.
|
||||
# DATABASE_URL=mysql+pymysql://Govin:change-me%40123@192.168.1.88:32738/paper_doc
|
||||
|
||||
# --- Object storage (MinIO) — where the images a paper embeds are kept ---
|
||||
# Leave these empty and the paper still writes and reads; only the image
|
||||
# endpoints answer 503, saying exactly which variables are missing.
|
||||
# The bucket is created on the first upload if it does not exist yet, so a
|
||||
# dedicated access key with read/write on that one bucket is enough.
|
||||
# Inside the cluster, use the Service: http://minio.minio.svc.cluster.local:9000
|
||||
MINIO_ENDPOINT=http://192.168.1.88:30090
|
||||
MINIO_ACCESS_KEY=change-me
|
||||
MINIO_SECRET_KEY=change-me
|
||||
MINIO_BUCKET=paper-images
|
||||
MINIO_REGION=us-east-1
|
||||
|
||||
# Largest image accepted, in bytes (default 20 MiB).
|
||||
# MAX_IMAGE_BYTES=20971520
|
||||
|
||||
@@ -4,6 +4,7 @@ from fastapi import APIRouter
|
||||
|
||||
from app.api.routes import (
|
||||
health,
|
||||
images,
|
||||
papers,
|
||||
reference_tags,
|
||||
references,
|
||||
@@ -13,6 +14,7 @@ from app.api.routes import (
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(health.router)
|
||||
api_router.include_router(images.router)
|
||||
api_router.include_router(papers.router)
|
||||
api_router.include_router(reference_tags.router)
|
||||
api_router.include_router(references.router)
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Image endpoints: put a figure in the bucket, and hand it back to the page.
|
||||
|
||||
The paper's sentences store pictures as Markdown — ```` —
|
||||
which is what keeps a sentence one editable string and what lets a model insert
|
||||
a figure through the same API it already writes text with. This module is the
|
||||
other half of that: the place the bytes go, and the place they come back from.
|
||||
|
||||
Uploading is one request and answers with the Markdown as well as the URL, so a
|
||||
client never has to assemble the syntax itself. Serving is by *key*, restricted
|
||||
to the keys this application writes (see :mod:`app.core.storage`), and streams
|
||||
from the bucket rather than loading the object into memory — an experiment
|
||||
figure is a megabyte or two and several of them appear on one screen.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, UploadFile, status
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.storage import (
|
||||
ImageNotFound,
|
||||
UnsupportedImage,
|
||||
content_type_for,
|
||||
get_store,
|
||||
image_url,
|
||||
markdown_for,
|
||||
)
|
||||
from app.schemas.image import ImageRead
|
||||
|
||||
router = APIRouter(prefix="/images", tags=["images"])
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
_READ_CHUNK = 1024 * 1024
|
||||
|
||||
|
||||
def _read_capped(handle, limit: int) -> bytes:
|
||||
"""Read one upload whole, refusing it the moment it passes ``limit``.
|
||||
|
||||
Read in pieces rather than with a single ``read()`` so an oversized file is
|
||||
rejected after the first megabyte instead of after all of it.
|
||||
"""
|
||||
pieces: list[bytes] = []
|
||||
total = 0
|
||||
while True:
|
||||
piece = handle.read(_READ_CHUNK)
|
||||
if not piece:
|
||||
break
|
||||
total += len(piece)
|
||||
if total > limit:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
detail=(
|
||||
f"图片太大了,超过 {limit // (1024 * 1024)} MB 上限。"
|
||||
"先压缩或裁剪一下再上传。"
|
||||
),
|
||||
)
|
||||
pieces.append(piece)
|
||||
return b"".join(pieces)
|
||||
|
||||
|
||||
@router.post(
|
||||
"",
|
||||
response_model=ImageRead,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
summary="Upload an image",
|
||||
)
|
||||
def upload_image(
|
||||
file: UploadFile = File(..., description="图片文件:PNG / JPEG / GIF / WebP"),
|
||||
) -> ImageRead:
|
||||
"""Store one image and return the Markdown that embeds it.
|
||||
|
||||
The declared content type is not trusted and neither is the filename: the
|
||||
format is decided from the file's own magic bytes, and the stored key comes
|
||||
from the content's digest. Uploading the same picture again is therefore not
|
||||
a duplicate — it is the same object, and a sentence that already points at
|
||||
it keeps working.
|
||||
"""
|
||||
data = _read_capped(file.file, settings.max_image_bytes)
|
||||
if not data:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="上传的文件是空的。",
|
||||
)
|
||||
|
||||
try:
|
||||
key, content_type, digest = get_store().save(data)
|
||||
except UnsupportedImage as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
detail=str(error),
|
||||
) from error
|
||||
|
||||
# The filename's stem becomes the caption's starting point: a screenshot is
|
||||
# usually already named after what it shows, and the writer edits the
|
||||
# brackets afterwards.
|
||||
caption = Path(file.filename or "").stem
|
||||
|
||||
return ImageRead(
|
||||
key=key,
|
||||
url=image_url(key),
|
||||
markdown=markdown_for(key, caption),
|
||||
content_type=content_type,
|
||||
size=len(data),
|
||||
sha256=digest,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{key:path}", summary="Serve an image")
|
||||
def read_image(key: str) -> StreamingResponse:
|
||||
"""Stream one image out of the bucket.
|
||||
|
||||
Served from this origin, so the URL in a sentence does not name a host that
|
||||
differs between the LAN and the public address, and the bucket never has to
|
||||
be made world-readable for the paper to show its figures.
|
||||
"""
|
||||
try:
|
||||
chunks, content_type, length = get_store().stream(key)
|
||||
except ImageNotFound as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="没有这张图片:地址不对,或者它已经不在存储里了。",
|
||||
) from error
|
||||
|
||||
return StreamingResponse(
|
||||
chunks,
|
||||
media_type=content_type or content_type_for(key) or "application/octet-stream",
|
||||
headers={
|
||||
# The key is the digest of the bytes, so this URL can never come to
|
||||
# mean a different picture — a year is safe, and it saves the paper
|
||||
# from re-fetching every figure on every scroll.
|
||||
"Cache-Control": "public, max-age=31536000, immutable",
|
||||
"Content-Length": str(length),
|
||||
# The type is from our own whitelist, never from the upload; this
|
||||
# stops a browser from second-guessing it.
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
@@ -53,6 +53,24 @@ class Settings(BaseSettings):
|
||||
db_pool_pre_ping: bool = True
|
||||
db_pool_recycle: int = 3600
|
||||
|
||||
# --- Object storage (MinIO) ------------------------------------------
|
||||
#
|
||||
# Where the images a paper embeds are kept. Empty by default rather than
|
||||
# pointed at a plausible host: a wrong-but-set address fails as a timeout
|
||||
# deep inside boto3, while an empty one fails immediately and says which
|
||||
# variables are missing. The bucket is created on first upload if it is not
|
||||
# there, so a fresh deployment needs the credentials and nothing else.
|
||||
minio_endpoint: str = ""
|
||||
minio_access_key: str = ""
|
||||
minio_secret_key: str = ""
|
||||
minio_bucket: str = "paper-images"
|
||||
minio_region: str = "us-east-1"
|
||||
|
||||
#: Largest image accepted, in bytes. An experiment figure is a few hundred
|
||||
#: kilobytes; the ceiling is here to stop a mistake (a raw TIFF, a video)
|
||||
#: from filling the bucket, not to be hit in normal use.
|
||||
max_image_bytes: int = 20 * 1024 * 1024
|
||||
|
||||
@property
|
||||
def sqlalchemy_url(self) -> Union[URL, str]:
|
||||
"""The SQLAlchemy connection URL.
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
"""Object storage for the images a paper embeds (MinIO, spoken over S3).
|
||||
|
||||
An image is not text, so it does not live in the database: the sentence keeps a
|
||||
short URL and the bytes live in the bucket. That split is what keeps a sentence
|
||||
editable as one string — the writer can move a figure between paragraphs by
|
||||
cutting and pasting text, and the object itself never has to move.
|
||||
|
||||
Three decisions are worth stating, because each of them is a way to serve
|
||||
something other than what was uploaded:
|
||||
|
||||
* **The key is content-addressed.** ``2026/09/9f8c1a4b7e2d0c33.png`` is derived
|
||||
from the bytes, so uploading the same screenshot twice stores one object, and
|
||||
a URL cannot quietly come to mean a *different* picture than the one it was
|
||||
written for. That is also what makes the year-long ``immutable`` cache header
|
||||
safe to hand out.
|
||||
* **The format is sniffed, not trusted.** The upload's ``Content-Type`` and its
|
||||
filename are the client's claims; the magic bytes are the fact. Only formats a
|
||||
browser draws in an ``<img>`` are accepted, and SVG is deliberately not among
|
||||
them — an uploaded SVG served from this origin is stored XSS.
|
||||
* **Images are served from our own path.** ``/api/images/{key}`` streams the
|
||||
object, so a paper never embeds a host name that differs between the LAN and
|
||||
the public address, and the bucket can stay private.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from functools import lru_cache
|
||||
from typing import Iterator
|
||||
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
|
||||
|
||||
class StorageUnavailable(RuntimeError):
|
||||
"""The object store is not configured, or could not be reached."""
|
||||
|
||||
|
||||
class UnsupportedImage(ValueError):
|
||||
"""The uploaded bytes are not a picture we are willing to store."""
|
||||
|
||||
|
||||
class ImageNotFound(KeyError):
|
||||
"""No image is stored under that key."""
|
||||
|
||||
|
||||
#: Sniffed format -> how the object is stored. Every format listed is one a
|
||||
#: browser renders; the extension is what the key ends in, and it is also how
|
||||
#: the content type is decided again when the object is served.
|
||||
IMAGE_FORMATS: dict[str, tuple[str, str]] = {
|
||||
"png": ("image/png", ".png"),
|
||||
"jpeg": ("image/jpeg", ".jpg"),
|
||||
"gif": ("image/gif", ".gif"),
|
||||
"webp": ("image/webp", ".webp"),
|
||||
}
|
||||
|
||||
#: The shape of a key this application writes: ``YYYY/MM/<16 hex><ext>``.
|
||||
#:
|
||||
#: Serving is restricted to it, so the read endpoint cannot be turned into a
|
||||
#: browser for the rest of the bucket — a request for anything this module did
|
||||
#: not store is a 404, whatever else happens to be in there.
|
||||
KEY_PATTERN = re.compile(r"^\d{4}/\d{2}/[0-9a-f]{16}\.(?:png|jpg|gif|webp)$")
|
||||
|
||||
#: Content type by extension, for serving. A whitelist rather than a parse of
|
||||
#: whatever the object's metadata says.
|
||||
CONTENT_TYPE_BY_EXTENSION: dict[str, str] = {
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
}
|
||||
|
||||
#: How much of a key's digest goes into it. 64 bits of SHA-256: a collision
|
||||
#: would need more distinct images than a paper will ever hold, and 16 hex
|
||||
#: characters keep the URL short enough to read in a sentence.
|
||||
DIGEST_CHARS = 16
|
||||
|
||||
_CHUNK = 64 * 1024
|
||||
|
||||
|
||||
def sniff(data: bytes) -> str:
|
||||
"""Name the format of ``data``, or refuse it.
|
||||
|
||||
Returns a key of :data:`IMAGE_FORMATS`. The check is on the leading bytes
|
||||
because that is the only part of the claim the uploader cannot choose.
|
||||
"""
|
||||
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return "png"
|
||||
if data.startswith(b"\xff\xd8\xff"):
|
||||
return "jpeg"
|
||||
if data.startswith((b"GIF87a", b"GIF89a")):
|
||||
return "gif"
|
||||
if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||||
return "webp"
|
||||
|
||||
raise UnsupportedImage(
|
||||
"只接受 PNG / JPEG / GIF / WebP 图片,这个文件的格式无法识别。"
|
||||
"(SVG 不支持:它会在页面上执行脚本。)"
|
||||
)
|
||||
|
||||
|
||||
def is_key(key: str) -> bool:
|
||||
"""Whether ``key`` has the shape this application stores."""
|
||||
return bool(KEY_PATTERN.match(key))
|
||||
|
||||
|
||||
def content_type_for(key: str) -> str | None:
|
||||
"""The content type to serve ``key`` as, or ``None`` if we would not serve it."""
|
||||
if not is_key(key):
|
||||
return None
|
||||
return CONTENT_TYPE_BY_EXTENSION.get(f".{key.rsplit('.', 1)[-1].lower()}")
|
||||
|
||||
|
||||
def image_url(key: str, settings: Settings | None = None) -> str:
|
||||
"""The URL a sentence stores for ``key``.
|
||||
|
||||
Relative on purpose: the paper is read from the LAN in development and from
|
||||
the public name later, and a sentence that hard-coded either would break on
|
||||
the other. The prefix comes from the API settings, so it follows the mount
|
||||
point instead of being spelled out twice.
|
||||
"""
|
||||
settings = settings or get_settings()
|
||||
return f"{settings.api_prefix}/images/{key}"
|
||||
|
||||
|
||||
def markdown_for(key: str, caption: str = "", settings: Settings | None = None) -> str:
|
||||
"""The Markdown a writer (or a model) pastes into a sentence.
|
||||
|
||||
Generating it here rather than in each client is the same argument as
|
||||
numbering citations on the server: the syntax is one rule, and two
|
||||
implementations of a rule eventually disagree.
|
||||
"""
|
||||
safe = caption.replace("[", "(").replace("]", ")").replace("(", "(").replace(")", ")")
|
||||
return f"})"
|
||||
|
||||
|
||||
class ImageStore:
|
||||
"""The bucket, seen through the four operations the API needs."""
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self._settings = settings
|
||||
self._client = None
|
||||
|
||||
@property
|
||||
def bucket(self) -> str:
|
||||
return self._settings.minio_bucket
|
||||
|
||||
@property
|
||||
def endpoint(self) -> str:
|
||||
return self._settings.minio_endpoint or "(未配置)"
|
||||
|
||||
# --- connection ------------------------------------------------------
|
||||
def _require(self) -> None:
|
||||
"""Refuse early, and in the API's words, when storage is not set up.
|
||||
|
||||
A missing configuration is an operator's mistake, not the caller's, so
|
||||
it must not surface as a stack trace from deep inside boto3.
|
||||
"""
|
||||
settings = self._settings
|
||||
if not (
|
||||
settings.minio_endpoint
|
||||
and settings.minio_access_key
|
||||
and settings.minio_secret_key
|
||||
):
|
||||
raise StorageUnavailable(
|
||||
"图片存储还没有配置好(缺少 MINIO_ENDPOINT / MINIO_ACCESS_KEY / "
|
||||
"MINIO_SECRET_KEY),暂时不能上传图片。"
|
||||
)
|
||||
|
||||
def client(self):
|
||||
"""The boto3 client, built on first use so importing costs nothing."""
|
||||
self._require()
|
||||
if self._client is None:
|
||||
self._client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=self._settings.minio_endpoint,
|
||||
aws_access_key_id=self._settings.minio_access_key,
|
||||
aws_secret_access_key=self._settings.minio_secret_key,
|
||||
region_name=self._settings.minio_region,
|
||||
config=Config(signature_version="s3v4", retries={"max_attempts": 3}),
|
||||
)
|
||||
return self._client
|
||||
|
||||
def _unavailable(self, error: Exception) -> StorageUnavailable:
|
||||
"""Restate a boto3 failure as something a person can act on."""
|
||||
if isinstance(error, ClientError):
|
||||
code = error.response.get("Error", {}).get("Code", "unknown")
|
||||
return StorageUnavailable(
|
||||
f"对象存储返回错误 {code}({self.endpoint},桶 {self.bucket}):{error}"
|
||||
)
|
||||
return StorageUnavailable(f"连不上对象存储 {self.endpoint}:{error}")
|
||||
|
||||
@contextmanager
|
||||
def _translated(self):
|
||||
"""Let boto3's exception vocabulary stop at this module's boundary."""
|
||||
try:
|
||||
yield
|
||||
except (BotoCoreError, ClientError) as error:
|
||||
raise self._unavailable(error) from error
|
||||
|
||||
# --- operations ------------------------------------------------------
|
||||
def ensure_bucket(self) -> None:
|
||||
"""Create the bucket if it is not there yet.
|
||||
|
||||
The bucket is normally made once by whoever deploys, but a missing one
|
||||
is an easy mistake to make and a tedious one to diagnose. Making it on
|
||||
first use costs one HEAD on an upload and turns that mistake into a
|
||||
working upload.
|
||||
"""
|
||||
client = self.client()
|
||||
try:
|
||||
client.head_bucket(Bucket=self.bucket)
|
||||
return
|
||||
except ClientError as error:
|
||||
code = error.response.get("Error", {}).get("Code", "")
|
||||
if code not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
raise self._unavailable(error) from error
|
||||
|
||||
try:
|
||||
client.create_bucket(Bucket=self.bucket)
|
||||
except ClientError as error:
|
||||
# Two uploads racing on a fresh deployment: one of them creates the
|
||||
# bucket, the other is told it exists. Neither is a failure.
|
||||
code = error.response.get("Error", {}).get("Code", "")
|
||||
if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
|
||||
raise self._unavailable(error) from error
|
||||
|
||||
def save(self, data: bytes) -> tuple[str, str, str]:
|
||||
"""Store ``data`` and return ``(key, content_type, sha256)``.
|
||||
|
||||
Storing is idempotent: the key comes from the bytes, so the same picture
|
||||
uploaded from two paragraphs — or twice by mistake — is one object.
|
||||
"""
|
||||
self._require()
|
||||
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
content_type, extension = IMAGE_FORMATS[sniff(data)]
|
||||
stamp = datetime.now(timezone.utc)
|
||||
key = f"{stamp:%Y/%m}/{digest[:DIGEST_CHARS]}{extension}"
|
||||
|
||||
with self._translated():
|
||||
self.ensure_bucket()
|
||||
self.client().put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=data,
|
||||
ContentType=content_type,
|
||||
# Safe because the key is the content's digest: these bytes can
|
||||
# never be replaced by different bytes under the same URL.
|
||||
CacheControl="public, max-age=31536000, immutable",
|
||||
)
|
||||
return key, content_type, digest
|
||||
|
||||
def stream(self, key: str) -> tuple[Iterator[bytes], str, int]:
|
||||
"""Open ``key`` for serving, as ``(chunks, content_type, length)``."""
|
||||
self._require()
|
||||
|
||||
content_type = content_type_for(key)
|
||||
if content_type is None:
|
||||
raise ImageNotFound(key)
|
||||
|
||||
try:
|
||||
response = self.client().get_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as error:
|
||||
code = error.response.get("Error", {}).get("Code", "")
|
||||
if code in {"NoSuchKey", "NoSuchBucket", "404", "NotFound"}:
|
||||
raise ImageNotFound(key) from error
|
||||
raise self._unavailable(error) from error
|
||||
except BotoCoreError as error:
|
||||
raise self._unavailable(error) from error
|
||||
|
||||
body = response["Body"]
|
||||
return _closing_chunks(body), content_type, int(response.get("ContentLength") or 0)
|
||||
|
||||
|
||||
def _closing_chunks(body, chunk: int = _CHUNK) -> Iterator[bytes]:
|
||||
"""Hand out the object's bytes and release the connection when done.
|
||||
|
||||
boto3's body is a stream on a socket from the pool; a response that is
|
||||
abandoned without closing it leaks that connection until the pool is empty.
|
||||
"""
|
||||
try:
|
||||
for piece in body.iter_chunks(chunk):
|
||||
yield piece
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_store() -> ImageStore:
|
||||
"""Return the process-wide store."""
|
||||
return ImageStore(get_settings())
|
||||
@@ -11,6 +11,7 @@ from fastapi.responses import JSONResponse
|
||||
|
||||
from app.api.router import api_router
|
||||
from app.core.config import get_settings
|
||||
from app.core.storage import StorageUnavailable
|
||||
from app.crud.reference import UnknownReference
|
||||
|
||||
settings = get_settings()
|
||||
@@ -43,3 +44,17 @@ async def unknown_reference_handler(
|
||||
for a rejected request, so a client needs no special case.
|
||||
"""
|
||||
return JSONResponse(status_code=400, content={"detail": str(error)})
|
||||
|
||||
|
||||
@app.exception_handler(StorageUnavailable)
|
||||
async def storage_unavailable_handler(
|
||||
request: Request, error: StorageUnavailable
|
||||
) -> JSONResponse:
|
||||
"""Answer an upload or a figure load when the object store cannot be used.
|
||||
|
||||
``503`` rather than ``500``: nothing about the request was wrong — the
|
||||
service that holds the images is missing or unreachable, and the message
|
||||
says which one so the fix is one configuration away. The writing itself is
|
||||
unaffected, which is why this is not allowed to look like a crash.
|
||||
"""
|
||||
return JSONResponse(status_code=503, content={"detail": str(error)})
|
||||
|
||||
@@ -6,6 +6,7 @@ from app.schemas.common import (
|
||||
PageResult,
|
||||
normalize_hex_color,
|
||||
)
|
||||
from app.schemas.image import ImageRead
|
||||
from app.schemas.paper import (
|
||||
CitationInput,
|
||||
CitationRead,
|
||||
@@ -57,6 +58,7 @@ from app.schemas.template_title import (
|
||||
|
||||
__all__ = [
|
||||
"BatchDeleteRequest",
|
||||
"ImageRead",
|
||||
"ReferenceAuthorInput",
|
||||
"ReferenceAuthorRead",
|
||||
"ReferenceCitationRead",
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
"""The shape an upload answers with."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ImageRead(BaseModel):
|
||||
"""One stored image, as the client needs to know about it.
|
||||
|
||||
The caller gets the URL *and* the Markdown that embeds it, because those are
|
||||
the two things it needs and only one of them is pleasant to assemble by
|
||||
hand. They are produced together here so a client cannot invent a slightly
|
||||
different syntax than the one the reader parses.
|
||||
"""
|
||||
|
||||
key: str = Field(description="对象在桶里的键,也是这个 URL 的唯一来源")
|
||||
url: str = Field(description="句子里的图片地址(同源相对路径,如 /api/images/…)")
|
||||
markdown: str = Field(description="可直接粘进句子的 Markdown:")
|
||||
content_type: str = Field(description="识别出来的图片类型,如 image/png")
|
||||
size: int = Field(description="字节数")
|
||||
sha256: str = Field(description="内容的 SHA-256,键就是它的一部分")
|
||||
@@ -5,6 +5,11 @@ alembic>=1.14
|
||||
pymysql>=1.1
|
||||
cryptography>=43.0
|
||||
pydantic-settings>=2.6
|
||||
# Object storage for the images a paper embeds (app/core/storage.py). boto3 is
|
||||
# the S3 client; python-multipart is what FastAPI needs to parse an upload at
|
||||
# all — without it a multipart route fails when the app is imported.
|
||||
boto3>=1.35
|
||||
python-multipart>=0.0.20
|
||||
# MCP server (app/mcp/) — the same domain layer, exposed as MCP tools for
|
||||
# Claude Code, Codex and the DeepSeek Harness. Only the MCP entry points need
|
||||
# it; the REST API does not import it.
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
"""End-to-end smoke test for images (插图), against a running API.
|
||||
|
||||
Run it while the backend is up, with MinIO reachable::
|
||||
|
||||
.venv/bin/python scripts/smoke_images.py
|
||||
|
||||
It uploads pictures of each accepted format, fetches them back byte for byte,
|
||||
checks the refusals (a text file, an SVG, an empty file, a key we did not
|
||||
write), and then writes a sentence that embeds a figure to prove the paper's
|
||||
own endpoints carry image Markdown through untouched. Exits non-zero on the
|
||||
first failed expectation, so it is usable as a gate.
|
||||
|
||||
Nothing needs cleaning up afterwards, and that is the point of the design being
|
||||
tested: a key is the digest of the picture, so a second run stores the same
|
||||
objects rather than more copies of them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import zlib
|
||||
from typing import Any
|
||||
|
||||
BASE = "http://127.0.0.1:8000/api"
|
||||
#: The origin the API is mounted on. An image URL comes back as `/api/images/…`
|
||||
#: — the path a *browser* uses — so fetching one has to join it to the origin
|
||||
#: rather than to the API prefix, or it asks for `/api/api/images/…`.
|
||||
ROOT = "http://127.0.0.1:8000"
|
||||
|
||||
KEY_PATTERN = re.compile(r"^\d{4}/\d{2}/[0-9a-f]{16}\.(?:png|jpg|gif|webp)$")
|
||||
|
||||
_checks = 0
|
||||
|
||||
|
||||
def call(method: str, path: str, body: Any = None, expect: int = 200) -> Any:
|
||||
"""Perform one JSON API call and assert its status code."""
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
url = BASE + urllib.parse.quote(path, safe="/?&=%")
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
method=method,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
status, payload, _ = _send(request)
|
||||
if status != expect:
|
||||
raise AssertionError(f"{method} {path} -> {status}, expected {expect}: {payload[:300]}")
|
||||
return json.loads(payload) if payload else None
|
||||
|
||||
|
||||
def _send(request: urllib.request.Request):
|
||||
"""Send a request and return ``(status, text body, headers)`` whatever happens.
|
||||
|
||||
The headers are handed over as the live mapping rather than as a ``dict``:
|
||||
uvicorn writes them lower-cased, and a plain dict lookup for
|
||||
``Content-Type`` would miss them.
|
||||
"""
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
return response.status, response.read().decode("utf-8", "replace"), response.headers
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, error.read().decode("utf-8", "replace"), error.headers
|
||||
|
||||
|
||||
def upload(
|
||||
filename: str,
|
||||
data: bytes,
|
||||
content_type: str = "application/octet-stream",
|
||||
expect: int = 201,
|
||||
) -> Any:
|
||||
"""POST one file as ``multipart/form-data``, which is what a browser sends.
|
||||
|
||||
Built by hand rather than with `requests`, so this script needs nothing the
|
||||
project does not already install.
|
||||
"""
|
||||
boundary = "----paperdocsmokeboundary"
|
||||
head = (
|
||||
f"--{boundary}\r\n"
|
||||
f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n'
|
||||
f"Content-Type: {content_type}\r\n\r\n"
|
||||
).encode()
|
||||
body = head + data + f"\r\n--{boundary}--\r\n".encode()
|
||||
request = urllib.request.Request(
|
||||
BASE + "/images",
|
||||
data=body,
|
||||
method="POST",
|
||||
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
|
||||
)
|
||||
status, payload, _ = _send(request)
|
||||
if status != expect:
|
||||
raise AssertionError(f"upload {filename} -> {status}, expected {expect}: {payload[:300]}")
|
||||
return json.loads(payload) if payload else None
|
||||
|
||||
|
||||
def fetch(path: str, expect: int = 200):
|
||||
"""GET an application path (``/api/images/…``) and return its bytes and headers."""
|
||||
request = urllib.request.Request(ROOT + path, method="GET")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
status, headers, body = response.status, response.headers, response.read()
|
||||
except urllib.error.HTTPError as error:
|
||||
status, headers, body = error.code, error.headers, error.read()
|
||||
if status != expect:
|
||||
raise AssertionError(f"GET {path} -> {status}, expected {expect}")
|
||||
return body, headers
|
||||
|
||||
|
||||
def check(label: str, condition: bool, detail: str = "") -> None:
|
||||
"""Assert one expectation and report it."""
|
||||
global _checks
|
||||
_checks += 1
|
||||
if not condition:
|
||||
raise AssertionError(f"{label}: {detail}")
|
||||
print(f" ok {label}")
|
||||
|
||||
|
||||
# --- the files under test ----------------------------------------------------
|
||||
|
||||
|
||||
def make_png(width: int = 240, height: int = 160) -> bytes:
|
||||
"""A real PNG: a gradient, so two of them differ and compression is honest."""
|
||||
|
||||
def chunk(tag: bytes, payload: bytes) -> bytes:
|
||||
body = tag + payload
|
||||
return struct.pack(">I", len(payload)) + body + struct.pack(">I", zlib.crc32(body) & 0xFFFFFFFF)
|
||||
|
||||
rows = bytearray()
|
||||
for y in range(height):
|
||||
rows.append(0) # filter: none
|
||||
for x in range(width):
|
||||
rows += bytes(((x * 255) // max(width - 1, 1), (y * 255) // max(height - 1, 1), 128))
|
||||
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(bytes(rows), 9))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
#: A 1x1 GIF and a 1x1 JPEG: the smallest real files of each format, which is
|
||||
#: all that is needed to prove the format is recognised and stored as itself.
|
||||
GIF_1PX = base64.b64decode("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7")
|
||||
JPEG_1PX = base64.b64decode(
|
||||
"/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0a"
|
||||
"HBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAA"
|
||||
"AAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q=="
|
||||
)
|
||||
|
||||
SVG = b'<svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script></svg>'
|
||||
|
||||
|
||||
def main() -> int:
|
||||
created_papers: list[int] = []
|
||||
|
||||
try:
|
||||
png = make_png()
|
||||
check("the fixture is a PNG", png.startswith(b"\x89PNG"))
|
||||
check("the JPEG fixture starts like a JPEG", JPEG_1PX.startswith(b"\xff\xd8\xff"))
|
||||
check("the GIF fixture starts like a GIF", GIF_1PX.startswith(b"GIF89a"))
|
||||
|
||||
print("1. upload a PNG")
|
||||
image = upload("试验装置.png", png, "image/png")
|
||||
key = image["key"]
|
||||
check("key has the shape we serve", bool(KEY_PATTERN.match(key)), key)
|
||||
check("key ends in .png", key.endswith(".png"), key)
|
||||
check("url is same-origin", image["url"] == f"/api/images/{key}", image["url"])
|
||||
check(
|
||||
"markdown embeds the url, caption from the filename",
|
||||
image["markdown"] == f"",
|
||||
image["markdown"],
|
||||
)
|
||||
check("content type sniffed", image["content_type"] == "image/png", image["content_type"])
|
||||
check("size reported", image["size"] == len(png), f"{image['size']} != {len(png)}")
|
||||
check(
|
||||
"sha256 is the content's",
|
||||
image["sha256"] == hashlib.sha256(png).hexdigest(),
|
||||
image["sha256"],
|
||||
)
|
||||
check("the digest is in the key", image["sha256"][:16] in key, key)
|
||||
|
||||
print("2. fetch it back")
|
||||
body, headers = fetch(image["url"])
|
||||
check("bytes identical", body == png, f"{len(body)} != {len(png)}")
|
||||
check(
|
||||
"served as image/png",
|
||||
headers.get("Content-Type") == "image/png",
|
||||
headers.get("Content-Type", ""),
|
||||
)
|
||||
check(
|
||||
"immutable cache header",
|
||||
"immutable" in headers.get("Cache-Control", ""),
|
||||
headers.get("Cache-Control", ""),
|
||||
)
|
||||
check(
|
||||
"nosniff",
|
||||
headers.get("X-Content-Type-Options") == "nosniff",
|
||||
headers.get("X-Content-Type-Options", ""),
|
||||
)
|
||||
check(
|
||||
"content-length matches",
|
||||
headers.get("Content-Length") == str(len(png)),
|
||||
headers.get("Content-Length", ""),
|
||||
)
|
||||
|
||||
print("3. the same bytes are the same object")
|
||||
again = upload("another-name.png", png, "image/png")
|
||||
check("same key", again["key"] == key, f"{again['key']} != {key}")
|
||||
check(
|
||||
"the caption follows the name given this time",
|
||||
again["markdown"] == f"",
|
||||
again["markdown"],
|
||||
)
|
||||
|
||||
print("4. the other formats")
|
||||
gif = upload("图.gif", GIF_1PX, "image/gif")
|
||||
check("gif key", gif["key"].endswith(".gif"), gif["key"])
|
||||
check("gif type", gif["content_type"] == "image/gif", gif["content_type"])
|
||||
check("gif bytes", fetch(gif["url"])[0] == GIF_1PX)
|
||||
|
||||
jpeg = upload("figure.jpeg", JPEG_1PX, "image/jpeg")
|
||||
check("jpeg key", jpeg["key"].endswith(".jpg"), jpeg["key"])
|
||||
check("jpeg type", jpeg["content_type"] == "image/jpeg", jpeg["content_type"])
|
||||
check("jpeg bytes", fetch(jpeg["url"])[0] == JPEG_1PX)
|
||||
|
||||
print("5. what it refuses")
|
||||
refused = upload("notes.txt", b"this is not a picture\n", "text/plain", expect=415)
|
||||
check("a text file is 415", "格式" in refused["detail"], refused["detail"])
|
||||
|
||||
svg = upload("attack.svg", SVG, "image/svg+xml", expect=415)
|
||||
check("an SVG is 415", "SVG" in svg["detail"], svg["detail"])
|
||||
|
||||
# A file that lies about being a PNG is still refused: the magic bytes
|
||||
# are what count, not the declared type or the extension.
|
||||
liar = upload("fake.png", b"<?php echo 1; ?>", "image/png", expect=415)
|
||||
check("a mislabelled file is 415", "格式" in liar["detail"], liar["detail"])
|
||||
|
||||
empty = upload("empty.png", b"", "image/png", expect=400)
|
||||
check("an empty upload is 400", "空" in empty["detail"], empty["detail"])
|
||||
|
||||
print("6. what it will not serve")
|
||||
missing, _ = fetch(f"/api/images/2026/01/{'0' * 16}.png", expect=404)
|
||||
check("a key with no object is 404", True, missing.decode("utf-8", "replace")[:80])
|
||||
|
||||
fetch("/api/images/mattermost/20260724/logo.png", expect=404)
|
||||
check("a key we did not write is 404 (the bucket is not browsable)", True)
|
||||
|
||||
fetch("/api/images/2026/01/zzzzzzzzzzzzzzzz.png", expect=404)
|
||||
check("a key outside the key space is 404", True)
|
||||
|
||||
print("7. a sentence carries the figure")
|
||||
templates = call("GET", "/templates?page_size=50")["items"]
|
||||
check("a template exists", bool(templates), f"got {len(templates)}")
|
||||
paper = call(
|
||||
"POST",
|
||||
"/papers",
|
||||
{
|
||||
"title": "冒烟测试-插图论文",
|
||||
"template_id": templates[0]["id"],
|
||||
"status": "draft",
|
||||
},
|
||||
expect=201,
|
||||
)
|
||||
created_papers.append(paper["id"])
|
||||
sort = call("GET", f"/papers/{paper['id']}/document")["paragraphs"][0][
|
||||
"paper_template_filed_sort"
|
||||
]
|
||||
|
||||
caption = "试验装置示意图"
|
||||
prose = f"试验装置如 {image['markdown']} 所示,量测点布置在试件中部。"
|
||||
call(
|
||||
"POST",
|
||||
f"/papers/{paper['id']}/sentences",
|
||||
{"paper_template_filed_sort": sort, "content": prose, "sort": 1},
|
||||
expect=201,
|
||||
)
|
||||
call(
|
||||
"POST",
|
||||
f"/papers/{paper['id']}/sentences",
|
||||
{
|
||||
"paper_template_filed_sort": sort,
|
||||
"content": f"{{width=70%}}",
|
||||
"sort": 2,
|
||||
"line_break_before": True,
|
||||
},
|
||||
expect=201,
|
||||
)
|
||||
|
||||
document = call("GET", f"/papers/{paper['id']}/document")
|
||||
paragraph = next(
|
||||
item for item in document["paragraphs"] if item["paper_template_filed_sort"] == sort
|
||||
)
|
||||
contents = [sentence["content"] for sentence in paragraph["sentences"]]
|
||||
check("the inline image survives the round trip", contents[0] == prose, contents[0])
|
||||
check(
|
||||
"the figure sentence survives with its size",
|
||||
contents[1] == f"{{width=70%}}",
|
||||
contents[1],
|
||||
)
|
||||
check("it is on a line of its own", paragraph["sentences"][1]["line_break_before"])
|
||||
check(
|
||||
"the server did not touch the markup",
|
||||
image["url"] in contents[0] and image["url"] in contents[1],
|
||||
)
|
||||
|
||||
print(f"\nall {_checks} checks passed")
|
||||
return 0
|
||||
finally:
|
||||
# Never leave test rows behind, even on a failed expectation.
|
||||
for paper_id in created_papers:
|
||||
try:
|
||||
call("DELETE", f"/papers/{paper_id}", expect=204)
|
||||
except Exception as error: # noqa: BLE001 — cleanup must not mask the failure
|
||||
print(f" !! could not delete paper {paper_id}: {error}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user