"""Object storage for the images a paper embeds (MinIO, spoken over S3). An image is not text, so it does not live in the database: the sentence keeps a short URL and the bytes live in the bucket. That split is what keeps a sentence editable as one string — the writer can move a figure between paragraphs by cutting and pasting text, and the object itself never has to move. Three decisions are worth stating, because each of them is a way to serve something other than what was uploaded: * **The key is content-addressed.** ``2026/09/9f8c1a4b7e2d0c33.png`` is derived from the bytes, so uploading the same screenshot twice stores one object, and a URL cannot quietly come to mean a *different* picture than the one it was written for. That is also what makes the year-long ``immutable`` cache header safe to hand out. * **The format is sniffed, not trusted.** The upload's ``Content-Type`` and its filename are the client's claims; the magic bytes are the fact. Only formats a browser draws in an ```` are accepted, and SVG is deliberately not among them — an uploaded SVG served from this origin is stored XSS. * **Images are served from our own path.** ``/api/images/{key}`` streams the object, so a paper never embeds a host name that differs between the LAN and the public address, and the bucket can stay private. """ from __future__ import annotations import hashlib import re from contextlib import contextmanager from datetime import datetime, timezone from functools import lru_cache from typing import Iterator import boto3 from botocore.client import Config from botocore.exceptions import BotoCoreError, ClientError from app.core.config import Settings, get_settings class StorageUnavailable(RuntimeError): """The object store is not configured, or could not be reached.""" class UnsupportedImage(ValueError): """The uploaded bytes are not a picture we are willing to store.""" class ImageNotFound(KeyError): """No image is stored under that key.""" #: Sniffed format -> how the object is stored. Every format listed is one a #: browser renders; the extension is what the key ends in, and it is also how #: the content type is decided again when the object is served. IMAGE_FORMATS: dict[str, tuple[str, str]] = { "png": ("image/png", ".png"), "jpeg": ("image/jpeg", ".jpg"), "gif": ("image/gif", ".gif"), "webp": ("image/webp", ".webp"), } #: The shape of a key this application writes: ``YYYY/MM/<16 hex>``. #: #: Serving is restricted to it, so the read endpoint cannot be turned into a #: browser for the rest of the bucket — a request for anything this module did #: not store is a 404, whatever else happens to be in there. KEY_PATTERN = re.compile(r"^\d{4}/\d{2}/[0-9a-f]{16}\.(?:png|jpg|gif|webp)$") #: Content type by extension, for serving. A whitelist rather than a parse of #: whatever the object's metadata says. CONTENT_TYPE_BY_EXTENSION: dict[str, str] = { ".png": "image/png", ".jpg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp", } #: How much of a key's digest goes into it. 64 bits of SHA-256: a collision #: would need more distinct images than a paper will ever hold, and 16 hex #: characters keep the URL short enough to read in a sentence. DIGEST_CHARS = 16 _CHUNK = 64 * 1024 def sniff(data: bytes) -> str: """Name the format of ``data``, or refuse it. Returns a key of :data:`IMAGE_FORMATS`. The check is on the leading bytes because that is the only part of the claim the uploader cannot choose. """ if data.startswith(b"\x89PNG\r\n\x1a\n"): return "png" if data.startswith(b"\xff\xd8\xff"): return "jpeg" if data.startswith((b"GIF87a", b"GIF89a")): return "gif" if data[:4] == b"RIFF" and data[8:12] == b"WEBP": return "webp" raise UnsupportedImage( "只接受 PNG / JPEG / GIF / WebP 图片,这个文件的格式无法识别。" "(SVG 不支持:它会在页面上执行脚本。)" ) def is_key(key: str) -> bool: """Whether ``key`` has the shape this application stores.""" return bool(KEY_PATTERN.match(key)) def content_type_for(key: str) -> str | None: """The content type to serve ``key`` as, or ``None`` if we would not serve it.""" if not is_key(key): return None return CONTENT_TYPE_BY_EXTENSION.get(f".{key.rsplit('.', 1)[-1].lower()}") def image_url(key: str, settings: Settings | None = None) -> str: """The URL a sentence stores for ``key``. Relative on purpose: the paper is read from the LAN in development and from the public name later, and a sentence that hard-coded either would break on the other. The prefix comes from the API settings, so it follows the mount point instead of being spelled out twice. """ settings = settings or get_settings() return f"{settings.api_prefix}/images/{key}" def markdown_for(key: str, caption: str = "", settings: Settings | None = None) -> str: """The Markdown a writer (or a model) pastes into a sentence. Generating it here rather than in each client is the same argument as numbering citations on the server: the syntax is one rule, and two implementations of a rule eventually disagree. """ safe = caption.replace("[", "(").replace("]", ")").replace("(", "(").replace(")", ")") return f"![{safe.strip()}]({image_url(key, settings)})" class ImageStore: """The bucket, seen through the four operations the API needs.""" def __init__(self, settings: Settings) -> None: self._settings = settings self._client = None @property def bucket(self) -> str: return self._settings.minio_bucket @property def endpoint(self) -> str: return self._settings.minio_endpoint or "(未配置)" # --- connection ------------------------------------------------------ def _require(self) -> None: """Refuse early, and in the API's words, when storage is not set up. A missing configuration is an operator's mistake, not the caller's, so it must not surface as a stack trace from deep inside boto3. """ settings = self._settings if not ( settings.minio_endpoint and settings.minio_access_key and settings.minio_secret_key ): raise StorageUnavailable( "图片存储还没有配置好(缺少 MINIO_ENDPOINT / MINIO_ACCESS_KEY / " "MINIO_SECRET_KEY),暂时不能上传图片。" ) def client(self): """The boto3 client, built on first use so importing costs nothing.""" self._require() if self._client is None: self._client = boto3.client( "s3", endpoint_url=self._settings.minio_endpoint, aws_access_key_id=self._settings.minio_access_key, aws_secret_access_key=self._settings.minio_secret_key, region_name=self._settings.minio_region, config=Config(signature_version="s3v4", retries={"max_attempts": 3}), ) return self._client def _unavailable(self, error: Exception) -> StorageUnavailable: """Restate a boto3 failure as something a person can act on.""" if isinstance(error, ClientError): code = error.response.get("Error", {}).get("Code", "unknown") return StorageUnavailable( f"对象存储返回错误 {code}({self.endpoint},桶 {self.bucket}):{error}" ) return StorageUnavailable(f"连不上对象存储 {self.endpoint}:{error}") @contextmanager def _translated(self): """Let boto3's exception vocabulary stop at this module's boundary.""" try: yield except (BotoCoreError, ClientError) as error: raise self._unavailable(error) from error # --- operations ------------------------------------------------------ def ensure_bucket(self) -> None: """Create the bucket if it is not there yet. The bucket is normally made once by whoever deploys, but a missing one is an easy mistake to make and a tedious one to diagnose. Making it on first use costs one HEAD on an upload and turns that mistake into a working upload. """ client = self.client() try: client.head_bucket(Bucket=self.bucket) return except ClientError as error: code = error.response.get("Error", {}).get("Code", "") if code in {"404", "NoSuchBucket", "NotFound"}: pass # not there — fall through and make it elif code in {"403", "AccessDenied"}: # Being refused a look at the bucket is not the same as the # bucket being absent. A key scoped to objects alone (PutObject / # GetObject, no ListBucket) gets exactly this answer, and the # upload that follows is the honest test of whether it works — # so return rather than turning a working key into a 503. return else: raise self._unavailable(error) from error try: client.create_bucket(Bucket=self.bucket) except ClientError as error: # Two uploads racing on a fresh deployment: one of them creates the # bucket, the other is told it exists. Neither is a failure. code = error.response.get("Error", {}).get("Code", "") if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}: raise self._unavailable(error) from error def save(self, data: bytes) -> tuple[str, str, str]: """Store ``data`` and return ``(key, content_type, sha256)``. Storing is idempotent: the key comes from the bytes, so the same picture uploaded from two paragraphs — or twice by mistake — is one object. """ self._require() digest = hashlib.sha256(data).hexdigest() content_type, extension = IMAGE_FORMATS[sniff(data)] stamp = datetime.now(timezone.utc) key = f"{stamp:%Y/%m}/{digest[:DIGEST_CHARS]}{extension}" with self._translated(): self.ensure_bucket() self.client().put_object( Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, # Safe because the key is the content's digest: these bytes can # never be replaced by different bytes under the same URL. CacheControl="public, max-age=31536000, immutable", ) return key, content_type, digest def stream(self, key: str) -> tuple[Iterator[bytes], str, int]: """Open ``key`` for serving, as ``(chunks, content_type, length)``.""" self._require() content_type = content_type_for(key) if content_type is None: raise ImageNotFound(key) try: response = self.client().get_object(Bucket=self.bucket, Key=key) except ClientError as error: code = error.response.get("Error", {}).get("Code", "") if code in {"NoSuchKey", "NoSuchBucket", "404", "NotFound"}: raise ImageNotFound(key) from error raise self._unavailable(error) from error except BotoCoreError as error: raise self._unavailable(error) from error body = response["Body"] return _closing_chunks(body), content_type, int(response.get("ContentLength") or 0) def _closing_chunks(body, chunk: int = _CHUNK) -> Iterator[bytes]: """Hand out the object's bytes and release the connection when done. boto3's body is a stream on a socket from the pool; a response that is abandoned without closing it leaks that connection until the pool is empty. """ try: for piece in body.iter_chunks(chunk): yield piece finally: body.close() @lru_cache def get_store() -> ImageStore: """Return the process-wide store.""" return ImageStore(get_settings())