diff --git a/backend/app/crud/paper.py b/backend/app/crud/paper.py index 5416c9d..b617942 100644 --- a/backend/app/crud/paper.py +++ b/backend/app/crud/paper.py @@ -227,6 +227,60 @@ def read(paper: Paper) -> PaperRead: return PaperRead(**item.model_dump()) +# --- how a paragraph reads --------------------------------------------------- + +#: Code-point ranges that count as CJK when deciding what belongs at the seam +#: between two sentences: ideographs, kana, CJK punctuation (。、「」…) and the +#: fullwidth forms. +_CJK_RANGES: tuple[tuple[int, int], ...] = ( + (0x3000, 0x303F), + (0x3040, 0x30FF), + (0x3400, 0x4DBF), + (0x4E00, 0x9FFF), + (0xF900, 0xFAFF), + (0xFF00, 0xFF60), + (0x20000, 0x2FA1F), +) + + +def _is_cjk(character: str) -> bool: + """Whether ``character`` is CJK, by code point.""" + code = ord(character) + return any(low <= code <= high for low, high in _CJK_RANGES) + + +def sentence_separator(previous: str, current: str) -> str: + """What belongs between two sentences when a paragraph is put back together. + + A paragraph is printed by concatenating its sentences in ``sort`` order, so + something has to decide what goes at the seam. Chinese needs nothing: the + full stop already separates, and a space between 「。」 and the next + character is wrong. English needs a space, or ``"First."`` followed by + ``"Second."`` prints as ``First.Second.`` — a boundary the reader cannot + see, and the one way this model could actually break an English paper. + + So the rule is about the seam, not about the language: a space goes in + unless **both** sides are CJK, where adjacency is the convention. A mixed + seam — 「……缺口。」 + ``This study…``, ``test.`` + 「本研究……」 — takes the + space, which is what a bilingual manuscript wants. + + Nothing about this is stored. The separator is derived on every read, so it + cannot drift from the text it separates, and it is recomputed for free when + a sentence is edited. + + Empty sentences take no separator: an empty line that carries only a + citation has no text to separate from. + """ + if not previous or not current: + return "" + if previous[-1].isspace() or current[0].isspace(): + # The writer's own spacing wins, though trimming makes this rare. + return "" + if _is_cjk(previous[-1]) and _is_cjk(current[0]): + return "" + return " " + + # --- document assembly ------------------------------------------------------- @@ -290,6 +344,12 @@ def build_document(db: Session, paper: Paper) -> PaperDocumentRead: ) sentences = [SentenceRead.model_validate(row) for row in rows] + # What goes between one sentence and the next is decided here rather + # than by whoever prints them, so a reader, an export and a preview + # cannot each invent their own spacing. + for earlier, later in zip(sentences, sentences[1:]): + later.separator_before = sentence_separator(earlier.content, later.content) + paragraph = ParagraphRead( paper_template_filed_sort=position, template_field_id=placement.id if placement is not None else None, diff --git a/backend/app/schemas/paper.py b/backend/app/schemas/paper.py index 8dc5937..92e5765 100644 --- a/backend/app/schemas/paper.py +++ b/backend/app/schemas/paper.py @@ -167,6 +167,11 @@ class SentenceRead(BaseModel): sort: int content: str citations: list[CitationRead] + #: Text to print before this sentence when the paragraph is put back + #: together — ``""`` or a single space, never anything else. Derived on + #: read by :func:`app.crud.paper.sentence_separator`, never stored, because + #: a separator that is stored is a separator that can go stale. + separator_before: str = "" created_at: datetime updated_at: datetime diff --git a/backend/scripts/smoke_papers.py b/backend/scripts/smoke_papers.py index 9de0d06..e51cb97 100644 --- a/backend/scripts/smoke_papers.py +++ b/backend/scripts/smoke_papers.py @@ -183,7 +183,49 @@ def main() -> int: {"sentences": []}, ) - print("6. reject a citation with no quote") + print("6. the seam between sentences depends on the language") + # Chinese needs nothing after 。; English needs a space, or two + # sentences print as `First.Second.` + mixed = call( + "PUT", + f"/papers/{paper['id']}/paragraphs/{second_sort}", + { + "sentences": [ + {"content": "第一句。"}, + {"content": "第二句。"}, + {"content": "This is English."}, + {"content": "So is this."}, + ] + }, + ) + paragraph_mixed = next( + p + for p in mixed["paragraphs"] + if p["paper_template_filed_sort"] == second_sort + ) + separators = [s["separator_before"] for s in paragraph_mixed["sentences"]] + check("no space between Chinese sentences", separators[:2] == ["", ""], str(separators)) + check("a space before an English sentence", separators[2] == " ", str(separators)) + check("a space between English sentences", separators[3] == " ", str(separators)) + assembled = "".join( + s["separator_before"] + s["content"] for s in paragraph_mixed["sentences"] + ) + check( + "the assembled paragraph reads correctly", + assembled == "第一句。第二句。 This is English. So is this.", + assembled, + ) + check( + "the separator is not part of the stored text", + all( + not s["content"].startswith(" ") and not s["content"].endswith(" ") + for s in paragraph_mixed["sentences"] + ), + ) + # Clear it again so the counts asserted below stay meaningful. + call("PUT", f"/papers/{paper['id']}/paragraphs/{second_sort}", {"sentences": []}) + + print("7. reject a citation with no quote") bad = call( "PUT", f"/papers/{paper['id']}/paragraphs/{first_sort}", @@ -192,7 +234,7 @@ def main() -> int: ) check("citation without content refused", "detail" in bad) - print("7. a sentence with no paragraph in the template survives a switch") + print("8. a sentence with no paragraph in the template survives a switch") # A position neither template defines, so the check means the same # thing before and after the switch below. orphan_sort = ( @@ -216,7 +258,7 @@ def main() -> int: p for p in document["paragraphs"] if p["paper_template_filed_sort"] == orphan_sort )["name"] is None) - print("8. switch the template") + print("9. switch the template") switched = call("PATCH", f"/papers/{paper['id']}", {"template_id": narrow["id"]}) check("template changed", switched["template_id"] == narrow["id"]) document = call("GET", f"/papers/{paper['id']}/document") @@ -246,7 +288,7 @@ def main() -> int: ), ) - print("9. move one paragraph onto another position") + print("10. move one paragraph onto another position") document = call( "PUT", f"/papers/{paper['id']}/paragraphs/{orphan_sort}", @@ -261,18 +303,18 @@ def main() -> int: all(p["paper_template_filed_sort"] != orphan_sort for p in document["paragraphs"]), ) - print("10. paper list reports progress") + print("11. paper list reports progress") listing = call("GET", f"/papers?keyword=冒烟&status=writing") check("found by keyword and status", listing["total"] == 1, str(listing["total"])) row = listing["items"][0] check("paragraph count", row["paragraph_count"] == 1, str(row["paragraph_count"])) check("sentence count", row["sentence_count"] == 3, str(row["sentence_count"])) - print("11. a template in use cannot be deleted") + print("12. a template in use cannot be deleted") conflict = call("DELETE", f"/templates/{narrow['id']}", expect=409) check("template delete refused", "正被论文使用" in conflict["detail"], conflict["detail"]) - print("12. single sentence edit and delete") + print("13. single sentence edit and delete") sentence_id = target["sentences"][0]["id"] updated = call( "PATCH", @@ -288,7 +330,7 @@ def main() -> int: sum(len(p["sentences"]) for p in document["paragraphs"]) == 2, ) - print("13. a paper with no template still works") + print("14. a paper with no template still works") bare = call( "POST", "/papers", @@ -311,7 +353,7 @@ def main() -> int: call("DELETE", f"/papers/{bare['id']}", expect=204) created_papers.remove(bare["id"]) - print("14. delete the paper") + print("15. delete the paper") call("DELETE", f"/papers/{paper['id']}", expect=204) created_papers.clear() check( diff --git a/docs/OVERVIEW.md b/docs/OVERVIEW.md index 21827cf..2a993f9 100644 --- a/docs/OVERVIEW.md +++ b/docs/OVERVIEW.md @@ -169,6 +169,54 @@ written against. It is kept in step with the paper's current template and is provenance rather than a lookup key: rendering never filters on it, which is precisely why content survives a switch. +### A sentence is a line, not parsed text + +Nothing in the writing path splits text. There is no sentence detector, no +punctuation rules and no comma handling: **one line in the editor is one +sentence**, and what the writer types is what is stored, minus runs of +whitespace (leading, trailing, and any newline pasted inside a line, which fold +to a single space so a sentence really is one line). Commas never break +anything — the only place a comma separates anything in this project is the +keyword field. + +That is deliberate. A splitter that guesses wrong corrupts content, and the +guess would have to be right for every abbreviation (`et al.`, `i.e.`), every +decimal (`3.14`), every numbered list and every language the tool is used in. +The writer decides where a sentence ends; the software only decides what goes at +the seam between two of them. + +Exactly one split has ever been performed, in revision `a83f5c21d7b6`, to move +the old `paper.abstract` column into the body. It cut *after* a Chinese full +stop (`。`) and nothing else — deliberately conservative: an English abstract +survives that migration as a single row rather than being cut at the first +`et al.`, and one long line is a line the writer can split by hand. + +#### What goes at the seam + +A paragraph is printed by concatenating its sentences in `sort` order, so the +seam between two of them has to be spelled out: +`app.crud.paper.sentence_separator` returns `""` or a single space, and the +document response carries it as `SentenceRead.separator_before`. + +| Seam | Separator | Why | +|---|---|---| +| `……缺口。` + `本研究……` | `""` | Chinese needs nothing; the full stop separates | +| `First sentence.` + `Second.` | `" "` | without it the two print as `First sentence.Second.` | +| `缺口。` + `This study` | `" "` | mixed text takes the space | +| `test.` + `本研究……` | `" "` | same, in the other direction | +| anything + an empty sentence | `""` | an empty line carries citations, not text | + +So the rule is about the seam rather than the language: a space goes in unless +**both** sides are CJK, where adjacency is already the convention. It is derived +on every read and never stored — a stored separator is one that can go stale +against the text it separates — and a client prints `separator_before + +content` and adds no spacing of its own. + +`separator_before` was added because the alternative was already a bug: CJK +sentences printed correctly by accident (the full stop hides the missing space), +so nothing looked wrong until the first English paper, whose sentences would +have run together. + ### Citations are a table, not a column One sentence may quote several references, so diff --git a/frontend/src/api/papers.ts b/frontend/src/api/papers.ts index 643acd0..75d1e36 100644 --- a/frontend/src/api/papers.ts +++ b/frontend/src/api/papers.ts @@ -85,6 +85,16 @@ export interface Sentence { sort: number content: string citations: Citation[] + /** + * What to print before this sentence when the paragraph is put back + * together — `''` or a single space, decided by the server. + * + * A paragraph is its sentences concatenated, and the seam matters: Chinese + * needs nothing after 「。」 while English needs a space, or `First.` followed + * by `Second.` reads as `First.Second.`. The server owns the rule so that a + * reader, a preview and a future export cannot each space it differently. + */ + separator_before: string created_at: string updated_at: string } diff --git a/frontend/src/components/papers/PaperParagraph.vue b/frontend/src/components/papers/PaperParagraph.vue index 9338b35..26201e2 100644 --- a/frontend/src/components/papers/PaperParagraph.vue +++ b/frontend/src/components/papers/PaperParagraph.vue @@ -8,9 +8,10 @@ * renders its heading and a quiet placeholder, because the paper's shape * comes from its template and an unwritten paragraph is still a paragraph. * 2. **A paragraph is reassembled from its sentences.** They are printed in - * `sort` order, one after another, with no separator added — Chinese - * sentences already end in punctuation, and inserting anything between them - * would show up in the text. + * `sort` order, and the only thing between them is the server's + * `separator_before` — a space for English, nothing for Chinese. This + * component adds no spacing of its own, because a client that guessed would + * space one paragraph differently from the next reader. * 3. **Nothing but the writing is printed.** No position numbers, no ids, no * counts — this is the paper, not the editor. `sort` is a writing detail and * the places that show it are the paragraph dialog (its title and every @@ -92,7 +93,9 @@ const anchor = computed(() => `paragraph-${props.paragraph.paper_template_filed_