backend: check the image rules without a bucket, and accept a scoped key
A least-privilege key — PutObject / GetObject and no ListBucket — gets a 403
from HeadBucket, and reading that as "the bucket is missing" turned a working
key into a 503. The bucket may well be there and simply not be the key's
business, so a refusal to look at it is no longer a failure; the PUT that
follows is the honest test.
`scripts/check_storage.py` covers what a smoke test cannot, because these are
the rules whose failure is silent: the format read from the magic bytes rather
than the client's word, the key pattern the read route serves and nothing
outside it, a caption that cannot break out of the Markdown it is written into,
what is actually asked of MinIO on a write, and the refusals — unconfigured
storage, a key we never wrote, an object that is gone. boto3's client is a stub,
so it needs no credentials, no bucket and no network.
`frontend/scripts/check-image.mjs` is the third layer: bytes can be stored and
returned perfectly while the `<img>` draws nothing. It builds its own PNG
fixture, drives the page and measures what the browser did — the picture
decoded, the figure is centred on the column, `{width=50%}` is half the column,
the 图注 prints under the picture, an image inside a sentence stays inline, and
「插入图片」 ends with the picture in the paper. Run with `pnpm check:image`; it
needs Playwright, which stays out of the app's dependencies.
This commit is contained in:
@@ -0,0 +1,301 @@
|
||||
"""Check the image storage rules without a bucket.
|
||||
|
||||
Run it with no server and no object store::
|
||||
|
||||
.venv/bin/python scripts/check_storage.py
|
||||
|
||||
``smoke_images.py`` proves the endpoints work against a real MinIO; this one
|
||||
checks the rules that decide *what may be stored and served at all*, which are
|
||||
the rules that fail silently. A format sniffed from the wrong offset, a key
|
||||
pattern that lets one more shape through, a `Cache-Control` that promises
|
||||
immutability the key does not provide — none of those break the happy path a
|
||||
smoke test walks, and each of them is a way to serve something other than what
|
||||
was uploaded.
|
||||
|
||||
boto3's client is replaced by a stub here, so nothing here needs credentials or
|
||||
network. It exits non-zero on the first failed expectation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Running this as a plain script puts scripts/ on sys.path, not backend/, so
|
||||
# `import app` would fail. Anchor to the backend directory instead.
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
from botocore.exceptions import ClientError # noqa: E402
|
||||
|
||||
from app.core.config import Settings # noqa: E402
|
||||
from app.core.storage import ( # noqa: E402
|
||||
IMAGE_FORMATS,
|
||||
ImageNotFound,
|
||||
ImageStore,
|
||||
StorageUnavailable,
|
||||
UnsupportedImage,
|
||||
content_type_for,
|
||||
image_url,
|
||||
is_key,
|
||||
markdown_for,
|
||||
sniff,
|
||||
)
|
||||
|
||||
_checks = 0
|
||||
|
||||
|
||||
def check(label: str, condition: bool, detail: str = "") -> None:
|
||||
"""Assert one expectation and report it."""
|
||||
global _checks
|
||||
_checks += 1
|
||||
if not condition:
|
||||
raise AssertionError(f"{label}: {detail}")
|
||||
print(f" ok {label}")
|
||||
|
||||
|
||||
def equal(label: str, actual: object, expected: object) -> None:
|
||||
"""Assert one value, showing both sides when it differs."""
|
||||
check(label, actual == expected, f"{actual!r} != {expected!r}")
|
||||
|
||||
|
||||
# --- stubs -------------------------------------------------------------------
|
||||
|
||||
|
||||
class FakeBody:
|
||||
"""A boto3 streaming body that says whether it was closed."""
|
||||
|
||||
def __init__(self, data: bytes) -> None:
|
||||
self.data = data
|
||||
self.closed = False
|
||||
|
||||
def iter_chunks(self, size: int):
|
||||
for start in range(0, len(self.data), size):
|
||||
yield self.data[start : start + size]
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class FakeClient:
|
||||
"""The four calls :class:`ImageStore` makes, with the answers we choose."""
|
||||
|
||||
def __init__(self, *, head_error: str | None = None, get_error: str | None = None) -> None:
|
||||
self.head_error = head_error
|
||||
self.get_error = get_error
|
||||
self.created: list[str] = []
|
||||
self.puts: list[dict] = []
|
||||
self.body = FakeBody(b"stored bytes")
|
||||
self.heads: list[str] = []
|
||||
|
||||
def head_bucket(self, Bucket: str): # noqa: N803 — boto3's own keyword names
|
||||
self.heads.append(Bucket)
|
||||
if self.head_error:
|
||||
raise ClientError({"Error": {"Code": self.head_error, "Message": "stub"}}, "HeadBucket")
|
||||
return {}
|
||||
|
||||
def create_bucket(self, Bucket: str): # noqa: N803
|
||||
self.created.append(Bucket)
|
||||
return {}
|
||||
|
||||
def put_object(self, **kwargs):
|
||||
self.puts.append(kwargs)
|
||||
return {}
|
||||
|
||||
def get_object(self, Bucket: str, Key: str): # noqa: N803
|
||||
if self.get_error:
|
||||
raise ClientError({"Error": {"Code": self.get_error, "Message": "stub"}}, "GetObject")
|
||||
return {"Body": self.body, "ContentLength": len(self.body.data)}
|
||||
|
||||
|
||||
def store(client: FakeClient | None = None, **overrides) -> ImageStore:
|
||||
"""A store whose boto3 client is the stub, so nothing leaves this process."""
|
||||
settings = Settings(
|
||||
minio_endpoint="http://minio.test:9000",
|
||||
minio_access_key="test-key",
|
||||
minio_secret_key="test-secret",
|
||||
minio_bucket="paper-images",
|
||||
**overrides,
|
||||
)
|
||||
instance = ImageStore(settings)
|
||||
if client is not None:
|
||||
instance._client = client # noqa: SLF001 — the seam this script exists to use
|
||||
return instance
|
||||
|
||||
|
||||
# --- the formats -------------------------------------------------------------
|
||||
|
||||
PNG = b"\x89PNG\r\n\x1a\n" + b"0" * 32
|
||||
JPEG = b"\xff\xd8\xff\xe0" + b"0" * 32
|
||||
GIF87 = b"GIF87a" + b"0" * 32
|
||||
GIF89 = b"GIF89a" + b"0" * 32
|
||||
WEBP = b"RIFF" + b"\x00\x00\x00\x00" + b"WEBP" + b"0" * 32
|
||||
|
||||
|
||||
def main() -> int:
|
||||
print("1. the format is read from the bytes")
|
||||
equal("a PNG is a PNG", sniff(PNG), "png")
|
||||
equal("a JPEG is a JPEG", sniff(JPEG), "jpeg")
|
||||
equal("GIF87a", sniff(GIF87), "gif")
|
||||
equal("GIF89a", sniff(GIF89), "gif")
|
||||
equal("RIFF….WEBP", sniff(WEBP), "webp")
|
||||
equal("every accepted format has a type and an extension", len(IMAGE_FORMATS), 4)
|
||||
|
||||
print("2. what is refused, whatever it claims to be")
|
||||
for label, data in [
|
||||
("an SVG", b'<svg xmlns="http://www.w3.org/2000/svg"></svg>'),
|
||||
("a PHP file renamed to .png", b"<?php echo 1; ?>"),
|
||||
("HTML", b"<!DOCTYPE html><html></html>"),
|
||||
("a PDF", b"%PDF-1.7\n"),
|
||||
("an empty file", b""),
|
||||
("four bytes that begin like a RIFF but are not WEBP", b"RIFF\x00\x00\x00\x00AVI "),
|
||||
]:
|
||||
try:
|
||||
sniff(data)
|
||||
check(f"{label} is refused", False, "it was accepted")
|
||||
except UnsupportedImage as error:
|
||||
check(f"{label} is refused", "SVG" in str(error) or "格式" in str(error), str(error))
|
||||
|
||||
print("3. the served key space is exactly what we write")
|
||||
ours = "2026/09/9f8c1a4b7e2d0c33.png"
|
||||
check("our own key is servable", is_key(ours), ours)
|
||||
equal("…and its content type comes from the extension", content_type_for(ours), "image/png")
|
||||
equal("a .jpg key", content_type_for("2026/09/9f8c1a4b7e2d0c33.jpg"), "image/jpeg")
|
||||
|
||||
for label, key in [
|
||||
("another bucket's layout", "mattermost/20260724/teams/logo.png"),
|
||||
("a key with no date", "9f8c1a4b7e2d0c33.png"),
|
||||
("a traversal attempt", "2026/09/../../../etc/passwd"),
|
||||
("a nested path", "2026/09/extra/9f8c1a4b7e2d0c33.png"),
|
||||
("an uppercase digest", "2026/09/9F8C1A4B7E2D0C33.png"),
|
||||
("a digest that is too short", "2026/09/9f8c.png"),
|
||||
("a format we would not serve", "2026/09/9f8c1a4b7e2d0c33.svg"),
|
||||
("an extension we would not serve", "2026/09/9f8c1a4b7e2d0c33.bmp"),
|
||||
("a double extension", "2026/09/9f8c1a4b7e2d0c33.png.html"),
|
||||
("a month that is not a month", "2026/9/9f8c1a4b7e2d0c33.png"),
|
||||
]:
|
||||
check(f"{label} is refused", not is_key(key), key)
|
||||
equal(f"…and {label} has no content type", content_type_for(key), None)
|
||||
|
||||
print("4. the URL and the Markdown a sentence keeps")
|
||||
settings = Settings(api_prefix="/api", minio_bucket="paper-images")
|
||||
equal("the URL is the API's own path", image_url(ours, settings), f"/api/images/{ours}")
|
||||
equal(
|
||||
"the Markdown carries the caption",
|
||||
markdown_for(ours, "试验装置", settings),
|
||||
f"",
|
||||
)
|
||||
equal(
|
||||
"a caption cannot close the brackets early",
|
||||
markdown_for(ours, "a]b(c)", settings),
|
||||
f"",
|
||||
)
|
||||
equal("an empty caption is allowed", markdown_for(ours, "", settings), f"")
|
||||
equal(
|
||||
"the prefix follows the settings",
|
||||
image_url(ours, Settings(api_prefix="/v2")),
|
||||
f"/v2/images/{ours}",
|
||||
)
|
||||
|
||||
print("5. unconfigured storage refuses in the API's words")
|
||||
bare = ImageStore(Settings(minio_endpoint="", minio_access_key="", minio_secret_key=""))
|
||||
for label, call in [
|
||||
("save", lambda: bare.save(PNG)),
|
||||
("stream", lambda: bare.stream(ours)),
|
||||
]:
|
||||
try:
|
||||
call()
|
||||
check(f"{label} without credentials is refused", False, "it went ahead")
|
||||
except StorageUnavailable as error:
|
||||
check(
|
||||
f"{label} without credentials names the missing variables",
|
||||
"MINIO_ENDPOINT" in str(error),
|
||||
str(error),
|
||||
)
|
||||
|
||||
print("6. storing asks MinIO for exactly what we intend")
|
||||
client = FakeClient(head_error="404")
|
||||
saved = store(client)
|
||||
key, content_type, digest = saved.save(PNG)
|
||||
check("the key is the digest", digest[:16] in key, key)
|
||||
check("…with the right extension", key.endswith(".png"), key)
|
||||
check("…and a servable shape", is_key(key), key)
|
||||
equal("the type is the sniffed one", content_type, "image/png")
|
||||
equal("it looked for the bucket once", len(client.heads), 1)
|
||||
equal("and created it, because it was missing", client.created, ["paper-images"])
|
||||
put = client.puts[0]
|
||||
equal("the bucket", put["Bucket"], "paper-images")
|
||||
equal("the key", put["Key"], key)
|
||||
equal("the bytes", put["Body"], PNG)
|
||||
equal("the content type", put["ContentType"], "image/png")
|
||||
check(
|
||||
"the cache promise matches the content-addressed key",
|
||||
"immutable" in put["CacheControl"],
|
||||
put["CacheControl"],
|
||||
)
|
||||
|
||||
print("7. the same bytes are the same object")
|
||||
again = FakeClient()
|
||||
key_again, _, digest_again = store(again).save(PNG)
|
||||
equal("same key", key_again, key)
|
||||
equal("same digest", digest_again, digest)
|
||||
other = FakeClient()
|
||||
key_other, _, _ = store(other).save(PNG + b"x")
|
||||
check("different bytes are a different object", key_other != key, key_other)
|
||||
|
||||
print("8. a key that may not look at the bucket still works")
|
||||
# A least-privilege key (PutObject / GetObject, no ListBucket) gets a 403
|
||||
# from HeadBucket. Treating that as "the bucket is missing" would turn a
|
||||
# working key into a 503, so the upload has to carry on.
|
||||
denied = FakeClient(head_error="403")
|
||||
store(denied).save(PNG)
|
||||
equal("it did not try to create the bucket", denied.created, [])
|
||||
equal("but it did store the object", len(denied.puts), 1)
|
||||
|
||||
print("9. reading refuses first, translates second")
|
||||
for label, key in [("a key we did not write", "mattermost/logo.png")]:
|
||||
try:
|
||||
store(FakeClient()).stream(key)
|
||||
check(f"{label} is refused", False, "it was served")
|
||||
except ImageNotFound:
|
||||
check(f"{label} is refused before boto3 is asked", True)
|
||||
|
||||
try:
|
||||
store(FakeClient(get_error="NoSuchKey")).stream(ours)
|
||||
check("a missing object is ImageNotFound", False, "it was served")
|
||||
except ImageNotFound:
|
||||
check("a missing object is ImageNotFound", True)
|
||||
|
||||
streaming_client = FakeClient()
|
||||
chunks, stream_type, length = store(streaming_client).stream(ours)
|
||||
body = b"".join(chunks)
|
||||
equal("the bytes come back", body, b"stored bytes")
|
||||
equal("with their type", stream_type, "image/png")
|
||||
equal("and their length", length, len(b"stored bytes"))
|
||||
check("the connection is released", streaming_client.body.closed)
|
||||
|
||||
print("10. a wrong endpoint is reported, not raised as a stack trace")
|
||||
try:
|
||||
ImageStore(
|
||||
Settings(
|
||||
minio_endpoint="http://127.0.0.1:1",
|
||||
minio_access_key="k",
|
||||
minio_secret_key="s",
|
||||
minio_bucket="paper-images",
|
||||
)
|
||||
).save(PNG)
|
||||
check("an unreachable endpoint is StorageUnavailable", False, "it went ahead")
|
||||
except StorageUnavailable as error:
|
||||
check(
|
||||
"an unreachable endpoint is StorageUnavailable",
|
||||
"连不上" in str(error) or "对象存储" in str(error),
|
||||
str(error),
|
||||
)
|
||||
|
||||
print(f"\nall {_checks} checks passed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user