backend: decide the seam between sentences instead of gluing them

A paragraph is its sentences concatenated. Nothing was put between them, which is right for Chinese — 「。」 already separates — and wrong for English, where `Adaptive capacity rises.` followed by `Relocation follows.` printed as `...rises.Relocation...`: a sentence boundary the reader cannot see. CJK output hid it, so it would have surfaced as an English bug later rather than now. `sentence_separator` now returns "" or a single space per seam, and the document carries it as `SentenceRead.separator_before`. The rule is about the seam, not the language: a space goes in unless both sides are CJK. Mixed seams take the space. Empty sentences take none. It is derived on every read and never stored, so it cannot drift from the text, and the client prints `separator_before + content` and adds no spacing of its own. No splitting was added anywhere, and none exists: a sentence is one line in the editor and nothing parses it. The single split this project has ever performed was the one-time move of the old abstract column, which cut after 「。」 only — conservative on purpose, since an English abstract is better left in one row than cut at the first `et al.`. Five smoke checks cover the seam (Chinese, English, the assembled paragraph, and that no separator is stored inside the content). 45 checks pass.
2026-09-18 19:01:31 +08:00
parent 898f519773
commit e582198cc8
6 changed files with 181 additions and 13 deletions
@@ -227,6 +227,60 @@ def read(paper: Paper) -> PaperRead:
    return PaperRead(**item.model_dump())


+# --- how a paragraph reads ---------------------------------------------------
+
+#: Code-point ranges that count as CJK when deciding what belongs at the seam
+#: between two sentences: ideographs, kana, CJK punctuation (。、「」…) and the
+#: fullwidth forms.
+_CJK_RANGES: tuple[tuple[int, int], ...] = (
+    (0x3000, 0x303F),
+    (0x3040, 0x30FF),
+    (0x3400, 0x4DBF),
+    (0x4E00, 0x9FFF),
+    (0xF900, 0xFAFF),
+    (0xFF00, 0xFF60),
+    (0x20000, 0x2FA1F),
+)
+
+
+def _is_cjk(character: str) -> bool:
+    """Whether ``character`` is CJK, by code point."""
+    code = ord(character)
+    return any(low <= code <= high for low, high in _CJK_RANGES)
+
+
+def sentence_separator(previous: str, current: str) -> str:
+    """What belongs between two sentences when a paragraph is put back together.
+
+    A paragraph is printed by concatenating its sentences in ``sort`` order, so
+    something has to decide what goes at the seam. Chinese needs nothing: the
+    full stop already separates, and a space between 「。」 and the next
+    character is wrong. English needs a space, or ``"First."`` followed by
+    ``"Second."`` prints as ``First.Second.`` — a boundary the reader cannot
+    see, and the one way this model could actually break an English paper.
+
+    So the rule is about the seam, not about the language: a space goes in
+    unless **both** sides are CJK, where adjacency is the convention. A mixed
+    seam — 「……缺口。」 + ``This study…``, ``test.`` + 「本研究……」 — takes the
+    space, which is what a bilingual manuscript wants.
+
+    Nothing about this is stored. The separator is derived on every read, so it
+    cannot drift from the text it separates, and it is recomputed for free when
+    a sentence is edited.
+
+    Empty sentences take no separator: an empty line that carries only a
+    citation has no text to separate from.
+    """
+    if not previous or not current:
+        return ""
+    if previous[-1].isspace() or current[0].isspace():
+        # The writer's own spacing wins, though trimming makes this rare.
+        return ""
+    if _is_cjk(previous[-1]) and _is_cjk(current[0]):
+        return ""
+    return " "
+
+
 # --- document assembly -------------------------------------------------------


@@ -290,6 +344,12 @@ def build_document(db: Session, paper: Paper) -> PaperDocumentRead:
        )
        sentences = [SentenceRead.model_validate(row) for row in rows]

+        # What goes between one sentence and the next is decided here rather
+        # than by whoever prints them, so a reader, an export and a preview
+        # cannot each invent their own spacing.
+        for earlier, later in zip(sentences, sentences[1:]):
+            later.separator_before = sentence_separator(earlier.content, later.content)
+
        paragraph = ParagraphRead(
            paper_template_filed_sort=position,
            template_field_id=placement.id if placement is not None else None,
@@ -167,6 +167,11 @@ class SentenceRead(BaseModel):
    sort: int
    content: str
    citations: list[CitationRead]
+    #: Text to print before this sentence when the paragraph is put back
+    #: together — ``""`` or a single space, never anything else. Derived on
+    #: read by :func:`app.crud.paper.sentence_separator`, never stored, because
+    #: a separator that is stored is a separator that can go stale.
+    separator_before: str = ""
    created_at: datetime
    updated_at: datetime

@@ -183,7 +183,49 @@ def main() -> int:
            {"sentences": []},
        )

-        print("6. reject a citation with no quote")
+        print("6. the seam between sentences depends on the language")
+        # Chinese needs nothing after 。; English needs a space, or two
+        # sentences print as `First.Second.`
+        mixed = call(
+            "PUT",
+            f"/papers/{paper['id']}/paragraphs/{second_sort}",
+            {
+                "sentences": [
+                    {"content": "第一句。"},
+                    {"content": "第二句。"},
+                    {"content": "This is English."},
+                    {"content": "So is this."},
+                ]
+            },
+        )
+        paragraph_mixed = next(
+            p
+            for p in mixed["paragraphs"]
+            if p["paper_template_filed_sort"] == second_sort
+        )
+        separators = [s["separator_before"] for s in paragraph_mixed["sentences"]]
+        check("no space between Chinese sentences", separators[:2] == ["", ""], str(separators))
+        check("a space before an English sentence", separators[2] == " ", str(separators))
+        check("a space between English sentences", separators[3] == " ", str(separators))
+        assembled = "".join(
+            s["separator_before"] + s["content"] for s in paragraph_mixed["sentences"]
+        )
+        check(
+            "the assembled paragraph reads correctly",
+            assembled == "第一句。第二句。 This is English. So is this.",
+            assembled,
+        )
+        check(
+            "the separator is not part of the stored text",
+            all(
+                not s["content"].startswith(" ") and not s["content"].endswith(" ")
+                for s in paragraph_mixed["sentences"]
+            ),
+        )
+        # Clear it again so the counts asserted below stay meaningful.
+        call("PUT", f"/papers/{paper['id']}/paragraphs/{second_sort}", {"sentences": []})
+
+        print("7. reject a citation with no quote")
        bad = call(
            "PUT",
            f"/papers/{paper['id']}/paragraphs/{first_sort}",
@@ -192,7 +234,7 @@ def main() -> int:
        )
        check("citation without content refused", "detail" in bad)

-        print("7. a sentence with no paragraph in the template survives a switch")
+        print("8. a sentence with no paragraph in the template survives a switch")
        # A position neither template defines, so the check means the same
        # thing before and after the switch below.
        orphan_sort = (
@@ -216,7 +258,7 @@ def main() -> int:
            p for p in document["paragraphs"] if p["paper_template_filed_sort"] == orphan_sort
        )["name"] is None)

-        print("8. switch the template")
+        print("9. switch the template")
        switched = call("PATCH", f"/papers/{paper['id']}", {"template_id": narrow["id"]})
        check("template changed", switched["template_id"] == narrow["id"])
        document = call("GET", f"/papers/{paper['id']}/document")
@@ -246,7 +288,7 @@ def main() -> int:
            ),
        )

-        print("9. move one paragraph onto another position")
+        print("10. move one paragraph onto another position")
        document = call(
            "PUT",
            f"/papers/{paper['id']}/paragraphs/{orphan_sort}",
@@ -261,18 +303,18 @@ def main() -> int:
            all(p["paper_template_filed_sort"] != orphan_sort for p in document["paragraphs"]),
        )

-        print("10. paper list reports progress")
+        print("11. paper list reports progress")
        listing = call("GET", f"/papers?keyword=冒烟&status=writing")
        check("found by keyword and status", listing["total"] == 1, str(listing["total"]))
        row = listing["items"][0]
        check("paragraph count", row["paragraph_count"] == 1, str(row["paragraph_count"]))
        check("sentence count", row["sentence_count"] == 3, str(row["sentence_count"]))

-        print("11. a template in use cannot be deleted")
+        print("12. a template in use cannot be deleted")
        conflict = call("DELETE", f"/templates/{narrow['id']}", expect=409)
        check("template delete refused", "正被论文使用" in conflict["detail"], conflict["detail"])

-        print("12. single sentence edit and delete")
+        print("13. single sentence edit and delete")
        sentence_id = target["sentences"][0]["id"]
        updated = call(
            "PATCH",
@@ -288,7 +330,7 @@ def main() -> int:
            sum(len(p["sentences"]) for p in document["paragraphs"]) == 2,
        )

-        print("13. a paper with no template still works")
+        print("14. a paper with no template still works")
        bare = call(
            "POST",
            "/papers",
@@ -311,7 +353,7 @@ def main() -> int:
        call("DELETE", f"/papers/{bare['id']}", expect=204)
        created_papers.remove(bare["id"])

-        print("14. delete the paper")
+        print("15. delete the paper")
        call("DELETE", f"/papers/{paper['id']}", expect=204)
        created_papers.clear()
        check(