diff --git a/app.py b/app.py index 37441d5..5d3f599 100644 --- a/app.py +++ b/app.py @@ -3,7 +3,7 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request, HTTPExcept from fastapi.responses import HTMLResponse, RedirectResponse, PlainTextResponse, FileResponse, JSONResponse from fastapi.staticfiles import StaticFiles import json, re, secrets, string, time, os, threading, asyncio, hashlib -from collections import defaultdict +from collections import defaultdict, deque from typing import Optional app = FastAPI() @@ -45,6 +45,111 @@ DOC_ID_RE = re.compile(r"^[a-zA-Z0-9_-]{1,64}$") def is_valid_doc_id(doc_id: str) -> bool: return bool(DOC_ID_RE.match(doc_id)) +# --- Operational transform core ------------------------------------------------ +# An op is a single splice: at `pos`, remove `dele` code units, insert `ins`. Every +# textarea interaction - typing, deleting, pasting, Tab, autocorrect - reduces to one +# splice, so this is the whole edit vocabulary. Ops travel over the wire instead of +# document snapshots because two snapshots taken concurrently cannot be combined: one +# has to win, which is what used to drop a peer's text and yank carets between lines. +# +# Offsets are UTF-16 code units, because that is what a textarea reports and what JS +# string indexing uses. Python indexes by code point; the two diverge only on pads +# holding astral characters (emoji), which is what splice()'s second path is for. +# +# diffOp/applyOp/transform in the client mirror this. The two copies MUST agree: +# test_ot.py fuzzes convergence, and every update carries the server's length so a +# disagreement resyncs instead of silently drifting. + +MAX_OP_LOG = 256 # ops kept per room, for rebasing edits from stale clients +MAX_OP_LOG_BYTES = 262144 # ...and a size ceiling, so one big paste cannot pin memory + + +def count_astral(s: str) -> int: + """Chars needing two UTF-16 code units. isascii() is a flag on the string object, + so the common case is O(1) and never walks the document.""" + if s.isascii(): + return 0 + return sum(1 for ch in s if ord(ch) > 0xFFFF) + + +def u16len(s: str) -> int: + return len(s) + count_astral(s) + + +def splice(text: str, op, u16: bool): + """Apply an op; returns (new_text, removed_text). + + `u16` says the text holds astral characters, so code-unit offsets no longer line up + with Python's code-point indexing and the splice has to happen in UTF-16 space. That + path costs an encode/decode of the document, which is why it is gated rather than + unconditional.""" + pos, dele, ins = op + if not u16: + return text[:pos] + ins + text[pos + dele:], text[pos:pos + dele] + b = text.encode("utf-16-le") + lo, hi = pos * 2, (pos + dele) * 2 + return ((b[:lo] + ins.encode("utf-16-le") + b[hi:]).decode("utf-16-le"), + b[lo:hi].decode("utf-16-le")) + + +def transform(a, b, a_first: bool): + """Rewrite op `a` so it applies to a text that already has op `b` applied. + + `a_first` says whether `a` precedes `b` in the server's total order - which is not the + same question as which one is already applied. The server rebases an incoming op onto + ops it has already ordered (a_first=False); a client rebases an op the server has + already ordered onto its own uncommitted edits (a_first=True). Both sides must agree + on the resulting order or their documents drift apart, so the flag is required rather + than defaulted.""" + apos, adel, ains = a + bpos, bdel, bins = b + aend, bend = apos + adel, bpos + bdel + # Two pure inserts at one position do not overlap - only the order of the two + # insertions is ambiguous - so settle it on the server's ordering. Without this each + # side puts the other's text first and two people typing at the same spot diverge + # immediately. + if adel == 0 and bdel == 0 and apos == bpos: + return a if a_first else (apos + u16len(bins), 0, ains) + if bend <= apos: # b entirely before a + return (apos + u16len(bins) - bdel, adel, ains) + if aend <= bpos: # b entirely after a + return a + # The two edits touch the same characters. A single splice cannot express the honest + # result - a's surviving fragments are no longer contiguous - so clamp, identically on + # both sides so they still converge. Only reachable when two people edit the same + # characters inside one round trip, where any answer is arbitrary. + if apos < bpos: + return (apos, bpos - apos, ains) + return (bpos + u16len(bins), max(0, aend - bend), ains) + + +def parse_op(data): + """Validate a wire op into (pos, dele, ins), or None. Structural checks only: the + range is checked after rebasing, since these offsets describe the client's older + version of the text, and the size limit is checked against the resulting document so + that an oversize paste gets told it was too large rather than malformed.""" + if not isinstance(data, dict): + return None + pos, dele, ins = data.get("pos"), data.get("del"), data.get("ins") + # bool subclasses int, so True would otherwise pass as position 1 + if isinstance(pos, bool) or isinstance(dele, bool): + return None + if not isinstance(pos, int) or not isinstance(dele, int) or not isinstance(ins, str): + return None + if pos < 0 or dele < 0: + return None + if "\x00" in ins: # matches the POST / check + return None + try: + ins.encode("utf-8") # rejects lone surrogates, which cannot be stored + except UnicodeEncodeError: + return None + return (pos, dele, ins) + +# --- end operational transform core -------------------------------------------- +# test_ot.py slices the block between these two markers and execs it, so that the +# fuzzer can run on a bare python with no fastapi installed. Keep them in place. + def get_client_ip(conn) -> str: # Only honor proxy headers when explicitly opted in — otherwise attackers # can spoof them to bypass per-IP limits. @@ -510,9 +615,95 @@ document.title = docId + " - aukpad"; let ws, ver = 0, clientId = Math.random().toString(36).slice(2), debounce; let isProtected = false, isAuthed = false; -// dirty: local edits the server has not acknowledged. Guards the reconnect path -// from overwriting text typed while the socket was down. -let dirty = false, reconnectTimer = null, reconnectDelay = 500; +let reconnectTimer = null, reconnectDelay = 500; + +// --- Sync state --- +// Invariant: shadow -> sent -> ta.value, each step being one splice. +// shadow the server text this client knows about, at version `ver` +// sent shadow with the op currently in flight applied, i.e. what the server will +// hold once it acks. Equal to shadow when nothing is in flight. +// Ops are never stored, only derived by diffing these three strings. That is what keeps +// the state machine small enough to reason about: there is no operation queue. +let shadow = "", sent = "", inflight = false, resyncing = false; + +// --- Operational transform core (mirror of the Python side; the two MUST agree) --- +// An op is one splice {pos, del, ins} in UTF-16 code units, which is what textarea +// offsets and JS string indices already use, so nothing is converted on this side. +const isHigh = c => c >= 0xD800 && c <= 0xDBFF; +const isLow = c => c >= 0xDC00 && c <= 0xDFFF; + +// The single changed region between two texts: common prefix, common suffix, and +// whatever is left in the middle. Every textarea interaction produces exactly one. +function diffOp(a, b) { + if (a === b) return null; + let s = 0; + const min = Math.min(a.length, b.length); + while (s < min && a[s] === b[s]) s++; + let ae = a.length, be = b.length; + while (ae > s && be > s && a[ae-1] === b[be-1]) { ae--; be--; } + // Never split a surrogate pair: a lone surrogate has no UTF-8 encoding, so the server + // would reject the op instead of syncing it. Nudge the boundaries outward instead. + if (s > 0 && isLow(a.charCodeAt(s)) && isHigh(a.charCodeAt(s-1))) s--; + if (ae < a.length && isLow(a.charCodeAt(ae)) && isHigh(a.charCodeAt(ae-1))) { ae++; be++; } + return {pos: s, del: ae - s, ins: b.slice(s, be)}; +} + +function applyOp(text, op) { + return text.slice(0, op.pos) + op.ins + text.slice(op.pos + op.del); +} + +// Rewrite op `a` so it applies to a text that already has op `b` applied. `aFirst` says +// whether `a` precedes `b` in the server's total order, which is a different question +// from which of the two is already applied: here it is always the op the server has +// ordered that comes first, and the server applies the identical rule. +function transform(a, b, aFirst) { + const aEnd = a.pos + a.del, bEnd = b.pos + b.del; + // Two pure inserts at one position do not overlap - only the order of the two + // insertions is ambiguous - so settle it on the server's ordering. Without this each + // side puts the other's text first and two people typing at the same spot diverge + // immediately. + if (!a.del && !b.del && a.pos === b.pos) + return aFirst ? a : {pos: a.pos + b.ins.length, del: 0, ins: a.ins}; + if (bEnd <= a.pos) return {pos: a.pos + b.ins.length - b.del, del: a.del, ins: a.ins}; + if (aEnd <= b.pos) return a; + // Same-characters overlap, which one splice cannot express faithfully. Clamp exactly + // as the server does; the length tripwire below catches anything this gets wrong. + if (a.pos < b.pos) return {pos: a.pos, del: b.pos - a.pos, ins: a.ins}; + return {pos: b.pos + b.ins.length, del: Math.max(0, aEnd - bEnd), ins: a.ins}; +} + +// Our picture of the server is wrong, so stop guessing and ask for a snapshot. Cheap to +// trigger and self-healing, which is what lets transform() clamp overlaps crudely rather +// than carry the machinery needed to always be exactly right. Blocks sending meanwhile: +// an op diffed against a bad shadow would be nonsense. +function resync() { + if (resyncing || ws?.readyState !== 1) return; + resyncing = true; + ws.send(JSON.stringify({type: "resync"})); +} + +// Send edits: one op in flight at a time, so the server always has a base version to +// rebase against. Anything typed while it is in flight is picked up by the next diff. +function flush() { + if (inflight || resyncing || ws?.readyState !== 1 || !isAuthed) return; + const op = diffOp(shadow, ta.value); + if (!op) return; + ws.send(JSON.stringify({type: "edit", base: ver, op, clientId})); + sent = ta.value; + inflight = true; +} + +// Transient message in the status slot. There is no error surface in the editor, which +// is how a refused edit used to pass unnoticed. +let notifyTimer = null; +function notify(text) { + const el = $("#status"); + el.textContent = text; + clearTimeout(notifyTimer); + notifyTimer = setTimeout(() => { + el.textContent = ws?.readyState === 1 ? "connected" : "disconnected"; + }, 4000); +} const urlPw = new URLSearchParams(location.search).get("pw") || ""; // --- Line numbers --- @@ -796,15 +987,17 @@ function connect(){ } else { isAuthed = true; hideOverlay(); + // One path for the first snapshot, the post-auth snapshot and every resync. + // Edits the server has never seen survive: keep the textarea as it is and let the + // next diff carry them onto the fresh snapshot rather than pushing a stale copy. + const local = diffOp(shadow, ta.value); + shadow = sent = msg.text; ver = msg.ver; - if (dirty && ta.value !== msg.text) { - // Reconnected with unsent edits: push them rather than lose them. - ws.send(JSON.stringify({type: "edit", ver, text: ta.value, clientId})); - } else { - ta.value = msg.text; - } + inflight = resyncing = false; + if (!local) ta.value = msg.text; refresh(); applyLang(msg.lang || ""); + flush(); // ?pw= on an unprotected pad sets the password instead of unlocking. // Guard on !isProtected: the server re-sends init after a successful // auth, and that init lands here too — without the guard every unlock @@ -823,13 +1016,50 @@ function connect(){ const errEl = $("#auth-error"); errEl.textContent = msg.message; errEl.style.display = "block"; + } else { + // An edit was refused - it would have pushed the pad over the size limit. Roll + // back to the server's text and release the send slot: leaving the two diverged + // is what used to make the limit look like random text loss. + inflight = false; + ta.value = sent = shadow; + refresh(); + notify(msg.message); } - } else if (msg.type === "update" && isAuthed && msg.ver > ver && msg.clientId !== clientId) { - const {selectionStart:s, selectionEnd:e} = ta; - const oldText = ta.value; - ta.value = msg.text; ver = msg.ver; refresh(); - ta.selectionStart = adjustCursor(oldText, msg.text, s); - ta.selectionEnd = adjustCursor(oldText, msg.text, e); + } else if (msg.type === "update" && isAuthed) { + if (msg.ver <= ver) return; // stale: a snapshot already covered it + if (msg.ver > ver + 1) { resync(); return; } // we missed one + if (msg.clientId === clientId) { + // Our own edit, echoed back: the ack. It carries the op as the server actually + // applied it, which is not necessarily the op we sent - it may have been rebased + // onto edits we had not seen when we sent it. + shadow = applyOp(shadow, msg.op); + ver = msg.ver; + inflight = false; + if (shadow !== sent) resync(); // our model of the server was wrong + else flush(); // push whatever was typed since + } else { + // Rebase the remote op past our own uncommitted edits - the one in flight and + // anything typed since - so it lands where the server put it, relative to text + // the server has not seen yet. Nothing else in the document is touched, which is + // the whole reason the caret stays where it is. + const mine = diffOp(shadow, sent), buffered = diffOp(sent, ta.value); + shadow = applyOp(shadow, msg.op); + ver = msg.ver; + // The remote op was ordered before anything of ours that is still local, so it + // comes first in both rebases. + let r = mine ? transform(msg.op, mine, true) : msg.op; + sent = applyOp(sent, r); + if (buffered) r = transform(r, buffered, true); + // setRangeText is the native splice: the spec shifts the selection by the length + // delta for us, and the browser's undo stack survives - both of which assigning + // to .value destroys. It fires no input event, so there is no echo back to the + // server, but that also means the gutter and overlay need refreshing by hand. + ta.setRangeText(r.ins, r.pos, r.pos + r.del, "preserve"); + refresh(); + } + // Divergence tripwire. shadow is exactly what we believe the server holds, so this + // is valid whether or not we have local edits outstanding. + if (shadow.length !== msg.len) resync(); } else if (msg.type === "peers_changed") { setPeers(msg.count); } else if (msg.type === "ping") { @@ -889,25 +1119,6 @@ async function copyToClipboard() { setTimeout(() => setIcon(btn, "copy"), 1500); } -// Adjust cursor position after a remote text update. -// Finds the single changed region (common prefix + suffix), -// then shifts the cursor accordingly: -// - change is after cursor → no movement -// - change is before cursor → shift by length delta -// - cursor was inside the changed region → place at end of new content -function adjustCursor(oldText, newText, pos) { - let start = 0; - const minLen = Math.min(oldText.length, newText.length); - while (start < minLen && oldText[start] === newText[start]) start++; - if (pos <= start) return pos; // change is entirely after cursor - let oldEnd = oldText.length, newEnd = newText.length; - while (oldEnd > start && newEnd > start && oldText[oldEnd - 1] === newText[newEnd - 1]) { - oldEnd--; newEnd--; - } - if (pos >= oldEnd) return pos + (newEnd - oldEnd); // change is before cursor - return newEnd; // cursor was inside changed region -} - connect(); // Handle Tab key to insert 4 spaces instead of navigation @@ -923,16 +1134,12 @@ ta.addEventListener("keydown", (e) => { } }); -// Send edits (debounced) +// Send edits (coalesced). Short window: the payload is a splice now rather than the +// whole document, and the less local text is sitting unsent, the smaller the window in +// which two people's edits can overlap at all. ta.addEventListener("input", () => { - dirty = true; clearTimeout(debounce); - debounce = setTimeout(() => { - if (ws?.readyState === 1 && isAuthed) { - ws.send(JSON.stringify({type:"edit", ver, text: ta.value, clientId})); - dirty = false; - } - }, 120); + debounce = setTimeout(flush, 30); }); """ @@ -1293,6 +1500,20 @@ def get_raw_pad_content(doc_id: str, request: Request, pw: str = ""): update_room_access_time(doc_id) return PlainTextResponse(room["text"]) +def init_payload(room: dict, authed: bool) -> dict: + """The full-snapshot message. Used for the first connect, after a successful unlock, + and for every resync, so a client resets its sync state through one code path.""" + return { + "type": "init", + "text": room["text"] if authed else "", + "ver": room["ver"], + "protected": room["pw_hash"] is not None, + "peers": len(room["peers"]), + # Withheld until authed: the language is content metadata and would tell + # someone sitting at the unlock overlay what kind of file this is. + "lang": room.get("lang") if authed else None, + } + async def _broadcast(doc_id: str, message: dict, exclude: WebSocket | None = None, authed_only: bool = False): room = rooms.get(doc_id) if not room: return @@ -1368,16 +1589,7 @@ async def ws(doc_id: str, ws: WebSocket): room["authed_peers"].add(ws) # Send init; withhold text if protected and not yet authed - await ws.send_text(json.dumps({ - "type": "init", - "text": room["text"] if authed else "", - "ver": room["ver"], - "protected": room["pw_hash"] is not None, - "peers": len(room["peers"]), - # Withheld until authed: the language is content metadata and would tell - # someone sitting at the unlock overlay what kind of file this is. - "lang": room.get("lang") if authed else None, - })) + await ws.send_text(json.dumps(init_payload(room, authed))) try: while True: msg = await ws.receive_text() @@ -1388,10 +1600,7 @@ async def ws(doc_id: str, ws: WebSocket): authed = True room["authed_peers"].add(ws) await ws.send_text(json.dumps({"type": "auth_ok"})) - await ws.send_text(json.dumps({ - "type": "init", "text": room["text"], "ver": room["ver"], "protected": False, - "lang": room.get("lang"), - })) + await ws.send_text(json.dumps(init_payload(room, True))) else: if not check_auth_rate_limit(client_ip): await ws.send_text(json.dumps({"type": "error", "message": "Too many failed attempts. Try again later."})) @@ -1404,10 +1613,7 @@ async def ws(doc_id: str, ws: WebSocket): authed = True room["authed_peers"].add(ws) await ws.send_text(json.dumps({"type": "auth_ok"})) - await ws.send_text(json.dumps({ - "type": "init", "text": room["text"], "ver": room["ver"], "protected": True, - "lang": room.get("lang"), - })) + await ws.send_text(json.dumps(init_payload(room, True))) else: record_auth_failure(client_ip) await ws.send_text(json.dumps({"type": "error", "message": "Wrong password"})) @@ -1418,27 +1624,80 @@ async def ws(doc_id: str, ws: WebSocket): continue if data.get("type") == "edit": - new_text = str(data.get("text", "")) + op = parse_op(data.get("op")) + base = data.get("base") + if op is None or isinstance(base, bool) or not isinstance(base, int) or base < 0: + await ws.send_text(json.dumps({"type": "error", "message": "Malformed edit"})) + continue - # Check text size limit - if len(new_text.encode('utf-8')) > MAX_TEXT_SIZE: + # Derived state, initialized on the first edit rather than in the three + # places a room gets built. Both are then O(1) to read and to maintain, + # which is what keeps a keystroke off the length of the document. + log = room.setdefault("ops", deque(maxlen=MAX_OP_LOG)) + astral = room.setdefault("astral", count_astral(room["text"])) + nbytes = room.setdefault("bytes", len(room["text"].encode("utf-8"))) + + # Rebase onto everything that landed since the client's base version. If + # the log no longer reaches that far back the client cannot be rebased and + # has to start from a snapshot - the same path a room restored from cache + # takes, since its op history did not survive. + if base < room["ver"] - len(log): + await ws.send_text(json.dumps(init_payload(room, True))) + continue + for v, prev in log: + if v > base: + # The logged op was ordered first, so the incoming one follows it. + op = transform(op, prev, False) + + if op[0] + op[1] > len(room["text"]) + astral: + await ws.send_text(json.dumps(init_payload(room, True))) + continue + + try: + new_text, removed = splice(room["text"], op, astral > 0) + delta = len(op[2].encode("utf-8")) - len(removed.encode("utf-8")) + except UnicodeEncodeError: + # A clamped op split a surrogate pair, so the result is not storable. + # Rare, and cheaper to resync than to repair. + await ws.send_text(json.dumps(init_payload(room, True))) + continue + + if nbytes + delta > MAX_TEXT_SIZE: + # The client rolls its own text back to ours, so the two cannot + # diverge silently the way they used to when this was ignored. await ws.send_text(json.dumps({"type": "error", "message": f"Text too large. Max size: {MAX_TEXT_SIZE} bytes"})) continue room["text"] = new_text + room["bytes"] = nbytes + delta + room["astral"] = astral - count_astral(removed) + count_astral(op[2]) room["ver"] += 1 room["last_access"] = time.time() + log.append((room["ver"], op)) + while len(log) > 1 and sum(len(o[2]) for _, o in log) > MAX_OP_LOG_BYTES: + log.popleft() # Save to cache save_room_data_to_cache(doc_id, room) + # Goes to everyone including the author: the author recognises its own + # clientId and treats the echo as the ack, which carries the op as it was + # actually applied - rebased, if it arrived stale. `len` is the divergence + # tripwire: a client whose own length disagrees asks for a snapshot. await _broadcast(doc_id, { "type": "update", - "text": room["text"], + "op": {"pos": op[0], "del": op[1], "ins": op[2]}, "ver": room["ver"], + "len": len(room["text"]) + room["astral"], "clientId": data.get("clientId"), }, authed_only=True) + elif data.get("type") == "resync": + # A client whose shadow no longer matches us asks for a fresh snapshot. + # Unthrottled on purpose: unlike a ping this fans out to nobody, so the + # cost lands on the asker's own connection, same as hitting /raw. + await ws.send_text(json.dumps(init_payload(room, True))) + elif data.get("type") == "set_password": pw = str(data.get("password", "")) if pw: