backend: keep a figure in the object store, and serve it back by key
A paper needs its experiment figures, and a figure is not text: the bytes go to
MinIO and the sentence keeps a Markdown address. That is what keeps a sentence
one editable string — a picture moves between paragraphs by being cut and
pasted, and survives a template switch with the sentence it belongs to.
Three decisions carry the safety, and each is a way to serve something other
than what was uploaded:
* the key is the digest of the bytes, so uploading the same screenshot twice is
one object and the address can never come to mean a different picture — which
is what makes the year-long `immutable` cache header safe;
* the format is sniffed from the magic bytes, not read from the client's
`Content-Type` or filename, and SVG is refused because an uploaded SVG served
from this origin is stored XSS;
* serving is restricted to the key space this application writes, so
`GET /api/images/{key}` cannot become a browser for the rest of the bucket.
Images are served through the API rather than from a public bucket URL so a
sentence never embeds a host name — the LAN in development and the public
address later would otherwise disagree — and the bucket can stay private. The
object is streamed rather than read into memory, and the connection is released
when the reader stops.
Storage unconfigured is a 503 that names the missing variables, not a crash:
the writing path does not depend on the bucket.
`scripts/smoke_images.py` walks the whole thing: each accepted format, the
refusals (text, SVG, a mislabelled file, an empty file, a key we did not
write), and a sentence that carries a figure through the paper endpoints.
This commit is contained in:
@@ -0,0 +1,327 @@
|
||||
"""End-to-end smoke test for images (插图), against a running API.
|
||||
|
||||
Run it while the backend is up, with MinIO reachable::
|
||||
|
||||
.venv/bin/python scripts/smoke_images.py
|
||||
|
||||
It uploads pictures of each accepted format, fetches them back byte for byte,
|
||||
checks the refusals (a text file, an SVG, an empty file, a key we did not
|
||||
write), and then writes a sentence that embeds a figure to prove the paper's
|
||||
own endpoints carry image Markdown through untouched. Exits non-zero on the
|
||||
first failed expectation, so it is usable as a gate.
|
||||
|
||||
Nothing needs cleaning up afterwards, and that is the point of the design being
|
||||
tested: a key is the digest of the picture, so a second run stores the same
|
||||
objects rather than more copies of them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import zlib
|
||||
from typing import Any
|
||||
|
||||
BASE = "http://127.0.0.1:8000/api"
|
||||
#: The origin the API is mounted on. An image URL comes back as `/api/images/…`
|
||||
#: — the path a *browser* uses — so fetching one has to join it to the origin
|
||||
#: rather than to the API prefix, or it asks for `/api/api/images/…`.
|
||||
ROOT = "http://127.0.0.1:8000"
|
||||
|
||||
KEY_PATTERN = re.compile(r"^\d{4}/\d{2}/[0-9a-f]{16}\.(?:png|jpg|gif|webp)$")
|
||||
|
||||
_checks = 0
|
||||
|
||||
|
||||
def call(method: str, path: str, body: Any = None, expect: int = 200) -> Any:
|
||||
"""Perform one JSON API call and assert its status code."""
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
url = BASE + urllib.parse.quote(path, safe="/?&=%")
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
method=method,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
status, payload, _ = _send(request)
|
||||
if status != expect:
|
||||
raise AssertionError(f"{method} {path} -> {status}, expected {expect}: {payload[:300]}")
|
||||
return json.loads(payload) if payload else None
|
||||
|
||||
|
||||
def _send(request: urllib.request.Request):
|
||||
"""Send a request and return ``(status, text body, headers)`` whatever happens.
|
||||
|
||||
The headers are handed over as the live mapping rather than as a ``dict``:
|
||||
uvicorn writes them lower-cased, and a plain dict lookup for
|
||||
``Content-Type`` would miss them.
|
||||
"""
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
return response.status, response.read().decode("utf-8", "replace"), response.headers
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, error.read().decode("utf-8", "replace"), error.headers
|
||||
|
||||
|
||||
def upload(
|
||||
filename: str,
|
||||
data: bytes,
|
||||
content_type: str = "application/octet-stream",
|
||||
expect: int = 201,
|
||||
) -> Any:
|
||||
"""POST one file as ``multipart/form-data``, which is what a browser sends.
|
||||
|
||||
Built by hand rather than with `requests`, so this script needs nothing the
|
||||
project does not already install.
|
||||
"""
|
||||
boundary = "----paperdocsmokeboundary"
|
||||
head = (
|
||||
f"--{boundary}\r\n"
|
||||
f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n'
|
||||
f"Content-Type: {content_type}\r\n\r\n"
|
||||
).encode()
|
||||
body = head + data + f"\r\n--{boundary}--\r\n".encode()
|
||||
request = urllib.request.Request(
|
||||
BASE + "/images",
|
||||
data=body,
|
||||
method="POST",
|
||||
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
|
||||
)
|
||||
status, payload, _ = _send(request)
|
||||
if status != expect:
|
||||
raise AssertionError(f"upload {filename} -> {status}, expected {expect}: {payload[:300]}")
|
||||
return json.loads(payload) if payload else None
|
||||
|
||||
|
||||
def fetch(path: str, expect: int = 200):
|
||||
"""GET an application path (``/api/images/…``) and return its bytes and headers."""
|
||||
request = urllib.request.Request(ROOT + path, method="GET")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
status, headers, body = response.status, response.headers, response.read()
|
||||
except urllib.error.HTTPError as error:
|
||||
status, headers, body = error.code, error.headers, error.read()
|
||||
if status != expect:
|
||||
raise AssertionError(f"GET {path} -> {status}, expected {expect}")
|
||||
return body, headers
|
||||
|
||||
|
||||
def check(label: str, condition: bool, detail: str = "") -> None:
|
||||
"""Assert one expectation and report it."""
|
||||
global _checks
|
||||
_checks += 1
|
||||
if not condition:
|
||||
raise AssertionError(f"{label}: {detail}")
|
||||
print(f" ok {label}")
|
||||
|
||||
|
||||
# --- the files under test ----------------------------------------------------
|
||||
|
||||
|
||||
def make_png(width: int = 240, height: int = 160) -> bytes:
|
||||
"""A real PNG: a gradient, so two of them differ and compression is honest."""
|
||||
|
||||
def chunk(tag: bytes, payload: bytes) -> bytes:
|
||||
body = tag + payload
|
||||
return struct.pack(">I", len(payload)) + body + struct.pack(">I", zlib.crc32(body) & 0xFFFFFFFF)
|
||||
|
||||
rows = bytearray()
|
||||
for y in range(height):
|
||||
rows.append(0) # filter: none
|
||||
for x in range(width):
|
||||
rows += bytes(((x * 255) // max(width - 1, 1), (y * 255) // max(height - 1, 1), 128))
|
||||
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(bytes(rows), 9))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
#: A 1x1 GIF and a 1x1 JPEG: the smallest real files of each format, which is
|
||||
#: all that is needed to prove the format is recognised and stored as itself.
|
||||
GIF_1PX = base64.b64decode("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7")
|
||||
JPEG_1PX = base64.b64decode(
|
||||
"/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0a"
|
||||
"HBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAA"
|
||||
"AAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q=="
|
||||
)
|
||||
|
||||
SVG = b'<svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script></svg>'
|
||||
|
||||
|
||||
def main() -> int:
|
||||
created_papers: list[int] = []
|
||||
|
||||
try:
|
||||
png = make_png()
|
||||
check("the fixture is a PNG", png.startswith(b"\x89PNG"))
|
||||
check("the JPEG fixture starts like a JPEG", JPEG_1PX.startswith(b"\xff\xd8\xff"))
|
||||
check("the GIF fixture starts like a GIF", GIF_1PX.startswith(b"GIF89a"))
|
||||
|
||||
print("1. upload a PNG")
|
||||
image = upload("试验装置.png", png, "image/png")
|
||||
key = image["key"]
|
||||
check("key has the shape we serve", bool(KEY_PATTERN.match(key)), key)
|
||||
check("key ends in .png", key.endswith(".png"), key)
|
||||
check("url is same-origin", image["url"] == f"/api/images/{key}", image["url"])
|
||||
check(
|
||||
"markdown embeds the url, caption from the filename",
|
||||
image["markdown"] == f"",
|
||||
image["markdown"],
|
||||
)
|
||||
check("content type sniffed", image["content_type"] == "image/png", image["content_type"])
|
||||
check("size reported", image["size"] == len(png), f"{image['size']} != {len(png)}")
|
||||
check(
|
||||
"sha256 is the content's",
|
||||
image["sha256"] == hashlib.sha256(png).hexdigest(),
|
||||
image["sha256"],
|
||||
)
|
||||
check("the digest is in the key", image["sha256"][:16] in key, key)
|
||||
|
||||
print("2. fetch it back")
|
||||
body, headers = fetch(image["url"])
|
||||
check("bytes identical", body == png, f"{len(body)} != {len(png)}")
|
||||
check(
|
||||
"served as image/png",
|
||||
headers.get("Content-Type") == "image/png",
|
||||
headers.get("Content-Type", ""),
|
||||
)
|
||||
check(
|
||||
"immutable cache header",
|
||||
"immutable" in headers.get("Cache-Control", ""),
|
||||
headers.get("Cache-Control", ""),
|
||||
)
|
||||
check(
|
||||
"nosniff",
|
||||
headers.get("X-Content-Type-Options") == "nosniff",
|
||||
headers.get("X-Content-Type-Options", ""),
|
||||
)
|
||||
check(
|
||||
"content-length matches",
|
||||
headers.get("Content-Length") == str(len(png)),
|
||||
headers.get("Content-Length", ""),
|
||||
)
|
||||
|
||||
print("3. the same bytes are the same object")
|
||||
again = upload("another-name.png", png, "image/png")
|
||||
check("same key", again["key"] == key, f"{again['key']} != {key}")
|
||||
check(
|
||||
"the caption follows the name given this time",
|
||||
again["markdown"] == f"",
|
||||
again["markdown"],
|
||||
)
|
||||
|
||||
print("4. the other formats")
|
||||
gif = upload("图.gif", GIF_1PX, "image/gif")
|
||||
check("gif key", gif["key"].endswith(".gif"), gif["key"])
|
||||
check("gif type", gif["content_type"] == "image/gif", gif["content_type"])
|
||||
check("gif bytes", fetch(gif["url"])[0] == GIF_1PX)
|
||||
|
||||
jpeg = upload("figure.jpeg", JPEG_1PX, "image/jpeg")
|
||||
check("jpeg key", jpeg["key"].endswith(".jpg"), jpeg["key"])
|
||||
check("jpeg type", jpeg["content_type"] == "image/jpeg", jpeg["content_type"])
|
||||
check("jpeg bytes", fetch(jpeg["url"])[0] == JPEG_1PX)
|
||||
|
||||
print("5. what it refuses")
|
||||
refused = upload("notes.txt", b"this is not a picture\n", "text/plain", expect=415)
|
||||
check("a text file is 415", "格式" in refused["detail"], refused["detail"])
|
||||
|
||||
svg = upload("attack.svg", SVG, "image/svg+xml", expect=415)
|
||||
check("an SVG is 415", "SVG" in svg["detail"], svg["detail"])
|
||||
|
||||
# A file that lies about being a PNG is still refused: the magic bytes
|
||||
# are what count, not the declared type or the extension.
|
||||
liar = upload("fake.png", b"<?php echo 1; ?>", "image/png", expect=415)
|
||||
check("a mislabelled file is 415", "格式" in liar["detail"], liar["detail"])
|
||||
|
||||
empty = upload("empty.png", b"", "image/png", expect=400)
|
||||
check("an empty upload is 400", "空" in empty["detail"], empty["detail"])
|
||||
|
||||
print("6. what it will not serve")
|
||||
missing, _ = fetch(f"/api/images/2026/01/{'0' * 16}.png", expect=404)
|
||||
check("a key with no object is 404", True, missing.decode("utf-8", "replace")[:80])
|
||||
|
||||
fetch("/api/images/mattermost/20260724/logo.png", expect=404)
|
||||
check("a key we did not write is 404 (the bucket is not browsable)", True)
|
||||
|
||||
fetch("/api/images/2026/01/zzzzzzzzzzzzzzzz.png", expect=404)
|
||||
check("a key outside the key space is 404", True)
|
||||
|
||||
print("7. a sentence carries the figure")
|
||||
templates = call("GET", "/templates?page_size=50")["items"]
|
||||
check("a template exists", bool(templates), f"got {len(templates)}")
|
||||
paper = call(
|
||||
"POST",
|
||||
"/papers",
|
||||
{
|
||||
"title": "冒烟测试-插图论文",
|
||||
"template_id": templates[0]["id"],
|
||||
"status": "draft",
|
||||
},
|
||||
expect=201,
|
||||
)
|
||||
created_papers.append(paper["id"])
|
||||
sort = call("GET", f"/papers/{paper['id']}/document")["paragraphs"][0][
|
||||
"paper_template_filed_sort"
|
||||
]
|
||||
|
||||
caption = "试验装置示意图"
|
||||
prose = f"试验装置如 {image['markdown']} 所示,量测点布置在试件中部。"
|
||||
call(
|
||||
"POST",
|
||||
f"/papers/{paper['id']}/sentences",
|
||||
{"paper_template_filed_sort": sort, "content": prose, "sort": 1},
|
||||
expect=201,
|
||||
)
|
||||
call(
|
||||
"POST",
|
||||
f"/papers/{paper['id']}/sentences",
|
||||
{
|
||||
"paper_template_filed_sort": sort,
|
||||
"content": f"{{width=70%}}",
|
||||
"sort": 2,
|
||||
"line_break_before": True,
|
||||
},
|
||||
expect=201,
|
||||
)
|
||||
|
||||
document = call("GET", f"/papers/{paper['id']}/document")
|
||||
paragraph = next(
|
||||
item for item in document["paragraphs"] if item["paper_template_filed_sort"] == sort
|
||||
)
|
||||
contents = [sentence["content"] for sentence in paragraph["sentences"]]
|
||||
check("the inline image survives the round trip", contents[0] == prose, contents[0])
|
||||
check(
|
||||
"the figure sentence survives with its size",
|
||||
contents[1] == f"{{width=70%}}",
|
||||
contents[1],
|
||||
)
|
||||
check("it is on a line of its own", paragraph["sentences"][1]["line_break_before"])
|
||||
check(
|
||||
"the server did not touch the markup",
|
||||
image["url"] in contents[0] and image["url"] in contents[1],
|
||||
)
|
||||
|
||||
print(f"\nall {_checks} checks passed")
|
||||
return 0
|
||||
finally:
|
||||
# Never leave test rows behind, even on a failed expectation.
|
||||
for paper_id in created_papers:
|
||||
try:
|
||||
call("DELETE", f"/papers/{paper_id}", expect=204)
|
||||
except Exception as error: # noqa: BLE001 — cleanup must not mask the failure
|
||||
print(f" !! could not delete paper {paper_id}: {error}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user