6f37120dc4
A paper needs its experiment figures, and a figure is not text: the bytes go to
MinIO and the sentence keeps a Markdown address. That is what keeps a sentence
one editable string — a picture moves between paragraphs by being cut and
pasted, and survives a template switch with the sentence it belongs to.
Three decisions carry the safety, and each is a way to serve something other
than what was uploaded:
* the key is the digest of the bytes, so uploading the same screenshot twice is
one object and the address can never come to mean a different picture — which
is what makes the year-long `immutable` cache header safe;
* the format is sniffed from the magic bytes, not read from the client's
`Content-Type` or filename, and SVG is refused because an uploaded SVG served
from this origin is stored XSS;
* serving is restricted to the key space this application writes, so
`GET /api/images/{key}` cannot become a browser for the rest of the bucket.
Images are served through the API rather than from a public bucket URL so a
sentence never embeds a host name — the LAN in development and the public
address later would otherwise disagree — and the bucket can stay private. The
object is streamed rather than read into memory, and the connection is released
when the reader stops.
Storage unconfigured is a 503 that names the missing variables, not a crash:
the writing path does not depend on the bucket.
`scripts/smoke_images.py` walks the whole thing: each accepted format, the
refusals (text, SVG, a mislabelled file, an empty file, a key we did not
write), and a sentence that carries a figure through the paper endpoints.
299 lines
11 KiB
Python
299 lines
11 KiB
Python
"""Object storage for the images a paper embeds (MinIO, spoken over S3).
|
||
|
||
An image is not text, so it does not live in the database: the sentence keeps a
|
||
short URL and the bytes live in the bucket. That split is what keeps a sentence
|
||
editable as one string — the writer can move a figure between paragraphs by
|
||
cutting and pasting text, and the object itself never has to move.
|
||
|
||
Three decisions are worth stating, because each of them is a way to serve
|
||
something other than what was uploaded:
|
||
|
||
* **The key is content-addressed.** ``2026/09/9f8c1a4b7e2d0c33.png`` is derived
|
||
from the bytes, so uploading the same screenshot twice stores one object, and
|
||
a URL cannot quietly come to mean a *different* picture than the one it was
|
||
written for. That is also what makes the year-long ``immutable`` cache header
|
||
safe to hand out.
|
||
* **The format is sniffed, not trusted.** The upload's ``Content-Type`` and its
|
||
filename are the client's claims; the magic bytes are the fact. Only formats a
|
||
browser draws in an ``<img>`` are accepted, and SVG is deliberately not among
|
||
them — an uploaded SVG served from this origin is stored XSS.
|
||
* **Images are served from our own path.** ``/api/images/{key}`` streams the
|
||
object, so a paper never embeds a host name that differs between the LAN and
|
||
the public address, and the bucket can stay private.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import re
|
||
from contextlib import contextmanager
|
||
from datetime import datetime, timezone
|
||
from functools import lru_cache
|
||
from typing import Iterator
|
||
|
||
import boto3
|
||
from botocore.client import Config
|
||
from botocore.exceptions import BotoCoreError, ClientError
|
||
|
||
from app.core.config import Settings, get_settings
|
||
|
||
|
||
class StorageUnavailable(RuntimeError):
|
||
"""The object store is not configured, or could not be reached."""
|
||
|
||
|
||
class UnsupportedImage(ValueError):
|
||
"""The uploaded bytes are not a picture we are willing to store."""
|
||
|
||
|
||
class ImageNotFound(KeyError):
|
||
"""No image is stored under that key."""
|
||
|
||
|
||
#: Sniffed format -> how the object is stored. Every format listed is one a
|
||
#: browser renders; the extension is what the key ends in, and it is also how
|
||
#: the content type is decided again when the object is served.
|
||
IMAGE_FORMATS: dict[str, tuple[str, str]] = {
|
||
"png": ("image/png", ".png"),
|
||
"jpeg": ("image/jpeg", ".jpg"),
|
||
"gif": ("image/gif", ".gif"),
|
||
"webp": ("image/webp", ".webp"),
|
||
}
|
||
|
||
#: The shape of a key this application writes: ``YYYY/MM/<16 hex><ext>``.
|
||
#:
|
||
#: Serving is restricted to it, so the read endpoint cannot be turned into a
|
||
#: browser for the rest of the bucket — a request for anything this module did
|
||
#: not store is a 404, whatever else happens to be in there.
|
||
KEY_PATTERN = re.compile(r"^\d{4}/\d{2}/[0-9a-f]{16}\.(?:png|jpg|gif|webp)$")
|
||
|
||
#: Content type by extension, for serving. A whitelist rather than a parse of
|
||
#: whatever the object's metadata says.
|
||
CONTENT_TYPE_BY_EXTENSION: dict[str, str] = {
|
||
".png": "image/png",
|
||
".jpg": "image/jpeg",
|
||
".gif": "image/gif",
|
||
".webp": "image/webp",
|
||
}
|
||
|
||
#: How much of a key's digest goes into it. 64 bits of SHA-256: a collision
|
||
#: would need more distinct images than a paper will ever hold, and 16 hex
|
||
#: characters keep the URL short enough to read in a sentence.
|
||
DIGEST_CHARS = 16
|
||
|
||
_CHUNK = 64 * 1024
|
||
|
||
|
||
def sniff(data: bytes) -> str:
|
||
"""Name the format of ``data``, or refuse it.
|
||
|
||
Returns a key of :data:`IMAGE_FORMATS`. The check is on the leading bytes
|
||
because that is the only part of the claim the uploader cannot choose.
|
||
"""
|
||
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||
return "png"
|
||
if data.startswith(b"\xff\xd8\xff"):
|
||
return "jpeg"
|
||
if data.startswith((b"GIF87a", b"GIF89a")):
|
||
return "gif"
|
||
if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||
return "webp"
|
||
|
||
raise UnsupportedImage(
|
||
"只接受 PNG / JPEG / GIF / WebP 图片,这个文件的格式无法识别。"
|
||
"(SVG 不支持:它会在页面上执行脚本。)"
|
||
)
|
||
|
||
|
||
def is_key(key: str) -> bool:
|
||
"""Whether ``key`` has the shape this application stores."""
|
||
return bool(KEY_PATTERN.match(key))
|
||
|
||
|
||
def content_type_for(key: str) -> str | None:
|
||
"""The content type to serve ``key`` as, or ``None`` if we would not serve it."""
|
||
if not is_key(key):
|
||
return None
|
||
return CONTENT_TYPE_BY_EXTENSION.get(f".{key.rsplit('.', 1)[-1].lower()}")
|
||
|
||
|
||
def image_url(key: str, settings: Settings | None = None) -> str:
|
||
"""The URL a sentence stores for ``key``.
|
||
|
||
Relative on purpose: the paper is read from the LAN in development and from
|
||
the public name later, and a sentence that hard-coded either would break on
|
||
the other. The prefix comes from the API settings, so it follows the mount
|
||
point instead of being spelled out twice.
|
||
"""
|
||
settings = settings or get_settings()
|
||
return f"{settings.api_prefix}/images/{key}"
|
||
|
||
|
||
def markdown_for(key: str, caption: str = "", settings: Settings | None = None) -> str:
|
||
"""The Markdown a writer (or a model) pastes into a sentence.
|
||
|
||
Generating it here rather than in each client is the same argument as
|
||
numbering citations on the server: the syntax is one rule, and two
|
||
implementations of a rule eventually disagree.
|
||
"""
|
||
safe = caption.replace("[", "(").replace("]", ")").replace("(", "(").replace(")", ")")
|
||
return f"})"
|
||
|
||
|
||
class ImageStore:
|
||
"""The bucket, seen through the four operations the API needs."""
|
||
|
||
def __init__(self, settings: Settings) -> None:
|
||
self._settings = settings
|
||
self._client = None
|
||
|
||
@property
|
||
def bucket(self) -> str:
|
||
return self._settings.minio_bucket
|
||
|
||
@property
|
||
def endpoint(self) -> str:
|
||
return self._settings.minio_endpoint or "(未配置)"
|
||
|
||
# --- connection ------------------------------------------------------
|
||
def _require(self) -> None:
|
||
"""Refuse early, and in the API's words, when storage is not set up.
|
||
|
||
A missing configuration is an operator's mistake, not the caller's, so
|
||
it must not surface as a stack trace from deep inside boto3.
|
||
"""
|
||
settings = self._settings
|
||
if not (
|
||
settings.minio_endpoint
|
||
and settings.minio_access_key
|
||
and settings.minio_secret_key
|
||
):
|
||
raise StorageUnavailable(
|
||
"图片存储还没有配置好(缺少 MINIO_ENDPOINT / MINIO_ACCESS_KEY / "
|
||
"MINIO_SECRET_KEY),暂时不能上传图片。"
|
||
)
|
||
|
||
def client(self):
|
||
"""The boto3 client, built on first use so importing costs nothing."""
|
||
self._require()
|
||
if self._client is None:
|
||
self._client = boto3.client(
|
||
"s3",
|
||
endpoint_url=self._settings.minio_endpoint,
|
||
aws_access_key_id=self._settings.minio_access_key,
|
||
aws_secret_access_key=self._settings.minio_secret_key,
|
||
region_name=self._settings.minio_region,
|
||
config=Config(signature_version="s3v4", retries={"max_attempts": 3}),
|
||
)
|
||
return self._client
|
||
|
||
def _unavailable(self, error: Exception) -> StorageUnavailable:
|
||
"""Restate a boto3 failure as something a person can act on."""
|
||
if isinstance(error, ClientError):
|
||
code = error.response.get("Error", {}).get("Code", "unknown")
|
||
return StorageUnavailable(
|
||
f"对象存储返回错误 {code}({self.endpoint},桶 {self.bucket}):{error}"
|
||
)
|
||
return StorageUnavailable(f"连不上对象存储 {self.endpoint}:{error}")
|
||
|
||
@contextmanager
|
||
def _translated(self):
|
||
"""Let boto3's exception vocabulary stop at this module's boundary."""
|
||
try:
|
||
yield
|
||
except (BotoCoreError, ClientError) as error:
|
||
raise self._unavailable(error) from error
|
||
|
||
# --- operations ------------------------------------------------------
|
||
def ensure_bucket(self) -> None:
|
||
"""Create the bucket if it is not there yet.
|
||
|
||
The bucket is normally made once by whoever deploys, but a missing one
|
||
is an easy mistake to make and a tedious one to diagnose. Making it on
|
||
first use costs one HEAD on an upload and turns that mistake into a
|
||
working upload.
|
||
"""
|
||
client = self.client()
|
||
try:
|
||
client.head_bucket(Bucket=self.bucket)
|
||
return
|
||
except ClientError as error:
|
||
code = error.response.get("Error", {}).get("Code", "")
|
||
if code not in {"404", "NoSuchBucket", "NotFound"}:
|
||
raise self._unavailable(error) from error
|
||
|
||
try:
|
||
client.create_bucket(Bucket=self.bucket)
|
||
except ClientError as error:
|
||
# Two uploads racing on a fresh deployment: one of them creates the
|
||
# bucket, the other is told it exists. Neither is a failure.
|
||
code = error.response.get("Error", {}).get("Code", "")
|
||
if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
|
||
raise self._unavailable(error) from error
|
||
|
||
def save(self, data: bytes) -> tuple[str, str, str]:
|
||
"""Store ``data`` and return ``(key, content_type, sha256)``.
|
||
|
||
Storing is idempotent: the key comes from the bytes, so the same picture
|
||
uploaded from two paragraphs — or twice by mistake — is one object.
|
||
"""
|
||
self._require()
|
||
|
||
digest = hashlib.sha256(data).hexdigest()
|
||
content_type, extension = IMAGE_FORMATS[sniff(data)]
|
||
stamp = datetime.now(timezone.utc)
|
||
key = f"{stamp:%Y/%m}/{digest[:DIGEST_CHARS]}{extension}"
|
||
|
||
with self._translated():
|
||
self.ensure_bucket()
|
||
self.client().put_object(
|
||
Bucket=self.bucket,
|
||
Key=key,
|
||
Body=data,
|
||
ContentType=content_type,
|
||
# Safe because the key is the content's digest: these bytes can
|
||
# never be replaced by different bytes under the same URL.
|
||
CacheControl="public, max-age=31536000, immutable",
|
||
)
|
||
return key, content_type, digest
|
||
|
||
def stream(self, key: str) -> tuple[Iterator[bytes], str, int]:
|
||
"""Open ``key`` for serving, as ``(chunks, content_type, length)``."""
|
||
self._require()
|
||
|
||
content_type = content_type_for(key)
|
||
if content_type is None:
|
||
raise ImageNotFound(key)
|
||
|
||
try:
|
||
response = self.client().get_object(Bucket=self.bucket, Key=key)
|
||
except ClientError as error:
|
||
code = error.response.get("Error", {}).get("Code", "")
|
||
if code in {"NoSuchKey", "NoSuchBucket", "404", "NotFound"}:
|
||
raise ImageNotFound(key) from error
|
||
raise self._unavailable(error) from error
|
||
except BotoCoreError as error:
|
||
raise self._unavailable(error) from error
|
||
|
||
body = response["Body"]
|
||
return _closing_chunks(body), content_type, int(response.get("ContentLength") or 0)
|
||
|
||
|
||
def _closing_chunks(body, chunk: int = _CHUNK) -> Iterator[bytes]:
|
||
"""Hand out the object's bytes and release the connection when done.
|
||
|
||
boto3's body is a stream on a socket from the pool; a response that is
|
||
abandoned without closing it leaks that connection until the pool is empty.
|
||
"""
|
||
try:
|
||
for piece in body.iter_chunks(chunk):
|
||
yield piece
|
||
finally:
|
||
body.close()
|
||
|
||
|
||
@lru_cache
|
||
def get_store() -> ImageStore:
|
||
"""Return the process-wide store."""
|
||
return ImageStore(get_settings())
|