cf7/aukpad
cf7/aukpad
1
0
Fork 0
Code Issues 6 Pull requests Projects Releases 5 Packages 1 Wiki Activity Actions

ux:FIX problems while working at same time - fixed by pushing changed instead of whole pad #21

Browse source
This commit is contained in:
Caffeine Fueled 2026-09-07 17:52:06 +02:00
parent 973e4bcedc
commit 12b8607f15
Signed by: cf7
GPG key ID: CA295D643074C68C
1 changed files with 324 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

389
app.py
View file

@ -3,7 +3,7 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request, HTTPExcept
from fastapi.responses import HTMLResponse, RedirectResponse, PlainTextResponse, FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
import json, re, secrets, string, time, os, threading, asyncio, hashlib
from collections import defaultdict
from collections import defaultdict, deque
from typing import Optional
app = FastAPI()
@ -45,6 +45,111 @@ DOC_ID_RE = re.compile(r"^[a-zA-Z0-9_-]{1,64}$")
def is_valid_doc_id(doc_id: str) -> bool:
return bool(DOC_ID_RE.match(doc_id))
# --- Operational transform core ------------------------------------------------
# An op is a single splice: at `pos`, remove `dele` code units, insert `ins`. Every
# textarea interaction - typing, deleting, pasting, Tab, autocorrect - reduces to one
# splice, so this is the whole edit vocabulary. Ops travel over the wire instead of
# document snapshots because two snapshots taken concurrently cannot be combined: one
# has to win, which is what used to drop a peer's text and yank carets between lines.
#
# Offsets are UTF-16 code units, because that is what a textarea reports and what JS
# string indexing uses. Python indexes by code point; the two diverge only on pads
# holding astral characters (emoji), which is what splice()'s second path is for.
#
# diffOp/applyOp/transform in the client mirror this. The two copies MUST agree:
# test_ot.py fuzzes convergence, and every update carries the server's length so a
# disagreement resyncs instead of silently drifting.
MAX_OP_LOG = 256 # ops kept per room, for rebasing edits from stale clients
MAX_OP_LOG_BYTES = 262144 # ...and a size ceiling, so one big paste cannot pin memory
def count_astral(s: str) -> int:
"""Chars needing two UTF-16 code units. isascii() is a flag on the string object,
so the common case is O(1) and never walks the document."""
if s.isascii():
return 0
return sum(1 for ch in s if ord(ch) > 0xFFFF)
def u16len(s: str) -> int:
return len(s) + count_astral(s)
def splice(text: str, op, u16: bool):
"""Apply an op; returns (new_text, removed_text).
`u16` says the text holds astral characters, so code-unit offsets no longer line up
with Python's code-point indexing and the splice has to happen in UTF-16 space. That
path costs an encode/decode of the document, which is why it is gated rather than
unconditional."""
pos, dele, ins = op
if not u16:
return text[:pos] + ins + text[pos + dele:], text[pos:pos + dele]
b = text.encode("utf-16-le")
lo, hi = pos * 2, (pos + dele) * 2
return ((b[:lo] + ins.encode("utf-16-le") + b[hi:]).decode("utf-16-le"),
b[lo:hi].decode("utf-16-le"))
def transform(a, b, a_first: bool):
"""Rewrite op `a` so it applies to a text that already has op `b` applied.
`a_first` says whether `a` precedes `b` in the server's total order - which is not the
same question as which one is already applied. The server rebases an incoming op onto
ops it has already ordered (a_first=False); a client rebases an op the server has
already ordered onto its own uncommitted edits (a_first=True). Both sides must agree
on the resulting order or their documents drift apart, so the flag is required rather
than defaulted."""
apos, adel, ains = a
bpos, bdel, bins = b
aend, bend = apos + adel, bpos + bdel
# Two pure inserts at one position do not overlap - only the order of the two
# insertions is ambiguous - so settle it on the server's ordering. Without this each
# side puts the other's text first and two people typing at the same spot diverge
# immediately.
if adel == 0 and bdel == 0 and apos == bpos:
return a if a_first else (apos + u16len(bins), 0, ains)
if bend <= apos: # b entirely before a
return (apos + u16len(bins) - bdel, adel, ains)
if aend <= bpos: # b entirely after a
return a
# The two edits touch the same characters. A single splice cannot express the honest
# result - a's surviving fragments are no longer contiguous - so clamp, identically on
# both sides so they still converge. Only reachable when two people edit the same
# characters inside one round trip, where any answer is arbitrary.
if apos < bpos:
return (apos, bpos - apos, ains)
return (bpos + u16len(bins), max(0, aend - bend), ains)
def parse_op(data):
"""Validate a wire op into (pos, dele, ins), or None. Structural checks only: the
range is checked after rebasing, since these offsets describe the client's older
version of the text, and the size limit is checked against the resulting document so
that an oversize paste gets told it was too large rather than malformed."""
if not isinstance(data, dict):
return None
pos, dele, ins = data.get("pos"), data.get("del"), data.get("ins")
# bool subclasses int, so True would otherwise pass as position 1
if isinstance(pos, bool) or isinstance(dele, bool):
return None
if not isinstance(pos, int) or not isinstance(dele, int) or not isinstance(ins, str):
return None
if pos < 0 or dele < 0:
return None
if "\x00" in ins: # matches the POST / check
return None
try:
ins.encode("utf-8") # rejects lone surrogates, which cannot be stored
except UnicodeEncodeError:
return None
return (pos, dele, ins)
# --- end operational transform core --------------------------------------------
# test_ot.py slices the block between these two markers and execs it, so that the
# fuzzer can run on a bare python with no fastapi installed. Keep them in place.
def get_client_ip(conn) -> str:
# Only honor proxy headers when explicitly opted in — otherwise attackers
# can spoof them to bypass per-IP limits.
@ -510,9 +615,95 @@ document.title = docId + " - aukpad";
let ws, ver = 0, clientId = Math.random().toString(36).slice(2), debounce;
let isProtected = false, isAuthed = false;
// dirty: local edits the server has not acknowledged. Guards the reconnect path
// from overwriting text typed while the socket was down.
let dirty = false, reconnectTimer = null, reconnectDelay = 500;
let reconnectTimer = null, reconnectDelay = 500;
// --- Sync state ---
// Invariant: shadow -> sent -> ta.value, each step being one splice.
// shadow the server text this client knows about, at version `ver`
// sent shadow with the op currently in flight applied, i.e. what the server will
// hold once it acks. Equal to shadow when nothing is in flight.
// Ops are never stored, only derived by diffing these three strings. That is what keeps
// the state machine small enough to reason about: there is no operation queue.
let shadow = "", sent = "", inflight = false, resyncing = false;
// --- Operational transform core (mirror of the Python side; the two MUST agree) ---
// An op is one splice {pos, del, ins} in UTF-16 code units, which is what textarea
// offsets and JS string indices already use, so nothing is converted on this side.
const isHigh = c => c >= 0xD800 && c <= 0xDBFF;
const isLow = c => c >= 0xDC00 && c <= 0xDFFF;
// The single changed region between two texts: common prefix, common suffix, and
// whatever is left in the middle. Every textarea interaction produces exactly one.
function diffOp(a, b) {
if (a === b) return null;
let s = 0;
const min = Math.min(a.length, b.length);
while (s < min && a[s] === b[s]) s++;
let ae = a.length, be = b.length;
while (ae > s && be > s && a[ae-1] === b[be-1]) { ae--; be--; }
// Never split a surrogate pair: a lone surrogate has no UTF-8 encoding, so the server
// would reject the op instead of syncing it. Nudge the boundaries outward instead.
if (s > 0 && isLow(a.charCodeAt(s)) && isHigh(a.charCodeAt(s-1))) s--;
if (ae < a.length && isLow(a.charCodeAt(ae)) && isHigh(a.charCodeAt(ae-1))) { ae++; be++; }
return {pos: s, del: ae - s, ins: b.slice(s, be)};
}
function applyOp(text, op) {
return text.slice(0, op.pos) + op.ins + text.slice(op.pos + op.del);
}
// Rewrite op `a` so it applies to a text that already has op `b` applied. `aFirst` says
// whether `a` precedes `b` in the server's total order, which is a different question
// from which of the two is already applied: here it is always the op the server has
// ordered that comes first, and the server applies the identical rule.
function transform(a, b, aFirst) {
const aEnd = a.pos + a.del, bEnd = b.pos + b.del;
// Two pure inserts at one position do not overlap - only the order of the two
// insertions is ambiguous - so settle it on the server's ordering. Without this each
// side puts the other's text first and two people typing at the same spot diverge
// immediately.
if (!a.del && !b.del && a.pos === b.pos)
return aFirst ? a : {pos: a.pos + b.ins.length, del: 0, ins: a.ins};
if (bEnd <= a.pos) return {pos: a.pos + b.ins.length - b.del, del: a.del, ins: a.ins};
if (aEnd <= b.pos) return a;
// Same-characters overlap, which one splice cannot express faithfully. Clamp exactly
// as the server does; the length tripwire below catches anything this gets wrong.
if (a.pos < b.pos) return {pos: a.pos, del: b.pos - a.pos, ins: a.ins};
return {pos: b.pos + b.ins.length, del: Math.max(0, aEnd - bEnd), ins: a.ins};
}
// Our picture of the server is wrong, so stop guessing and ask for a snapshot. Cheap to
// trigger and self-healing, which is what lets transform() clamp overlaps crudely rather
// than carry the machinery needed to always be exactly right. Blocks sending meanwhile:
// an op diffed against a bad shadow would be nonsense.
function resync() {
if (resyncing || ws?.readyState !== 1) return;
resyncing = true;
ws.send(JSON.stringify({type: "resync"}));
}
// Send edits: one op in flight at a time, so the server always has a base version to
// rebase against. Anything typed while it is in flight is picked up by the next diff.
function flush() {
if (inflight || resyncing || ws?.readyState !== 1 || !isAuthed) return;
const op = diffOp(shadow, ta.value);
if (!op) return;
ws.send(JSON.stringify({type: "edit", base: ver, op, clientId}));
sent = ta.value;
inflight = true;
}
// Transient message in the status slot. There is no error surface in the editor, which
// is how a refused edit used to pass unnoticed.
let notifyTimer = null;
function notify(text) {
const el = $("#status");
el.textContent = text;
clearTimeout(notifyTimer);
notifyTimer = setTimeout(() => {
el.textContent = ws?.readyState === 1 ? "connected" : "disconnected";
}, 4000);
}
const urlPw = new URLSearchParams(location.search).get("pw") || "";
// --- Line numbers ---
@ -796,15 +987,17 @@ function connect(){
} else {
isAuthed = true;
hideOverlay();
// One path for the first snapshot, the post-auth snapshot and every resync.
// Edits the server has never seen survive: keep the textarea as it is and let the
// next diff carry them onto the fresh snapshot rather than pushing a stale copy.
const local = diffOp(shadow, ta.value);
shadow = sent = msg.text;
ver = msg.ver;
if (dirty && ta.value !== msg.text) {
// Reconnected with unsent edits: push them rather than lose them.
ws.send(JSON.stringify({type: "edit", ver, text: ta.value, clientId}));
} else {
ta.value = msg.text;
}
inflight = resyncing = false;
if (!local) ta.value = msg.text;
refresh();
applyLang(msg.lang || "");
flush();
// ?pw= on an unprotected pad sets the password instead of unlocking.
// Guard on !isProtected: the server re-sends init after a successful
// auth, and that init lands here too — without the guard every unlock
@ -823,13 +1016,50 @@ function connect(){
const errEl = $("#auth-error");
errEl.textContent = msg.message;
errEl.style.display = "block";
} else {
// An edit was refused - it would have pushed the pad over the size limit. Roll
// back to the server's text and release the send slot: leaving the two diverged
// is what used to make the limit look like random text loss.
inflight = false;
ta.value = sent = shadow;
refresh();
notify(msg.message);
}
} else if (msg.type === "update" && isAuthed && msg.ver > ver && msg.clientId !== clientId) {
const {selectionStart:s, selectionEnd:e} = ta;
const oldText = ta.value;
ta.value = msg.text; ver = msg.ver; refresh();
ta.selectionStart = adjustCursor(oldText, msg.text, s);
ta.selectionEnd = adjustCursor(oldText, msg.text, e);
} else if (msg.type === "update" && isAuthed) {
if (msg.ver <= ver) return; // stale: a snapshot already covered it
if (msg.ver > ver + 1) { resync(); return; } // we missed one
if (msg.clientId === clientId) {
// Our own edit, echoed back: the ack. It carries the op as the server actually
// applied it, which is not necessarily the op we sent - it may have been rebased
// onto edits we had not seen when we sent it.
shadow = applyOp(shadow, msg.op);
ver = msg.ver;
inflight = false;
if (shadow !== sent) resync(); // our model of the server was wrong
else flush(); // push whatever was typed since
} else {
// Rebase the remote op past our own uncommitted edits - the one in flight and
// anything typed since - so it lands where the server put it, relative to text
// the server has not seen yet. Nothing else in the document is touched, which is
// the whole reason the caret stays where it is.
const mine = diffOp(shadow, sent), buffered = diffOp(sent, ta.value);
shadow = applyOp(shadow, msg.op);
ver = msg.ver;
// The remote op was ordered before anything of ours that is still local, so it
// comes first in both rebases.
let r = mine ? transform(msg.op, mine, true) : msg.op;
sent = applyOp(sent, r);
if (buffered) r = transform(r, buffered, true);
// setRangeText is the native splice: the spec shifts the selection by the length
// delta for us, and the browser's undo stack survives - both of which assigning
// to .value destroys. It fires no input event, so there is no echo back to the
// server, but that also means the gutter and overlay need refreshing by hand.
ta.setRangeText(r.ins, r.pos, r.pos + r.del, "preserve");
refresh();
}
// Divergence tripwire. shadow is exactly what we believe the server holds, so this
// is valid whether or not we have local edits outstanding.
if (shadow.length !== msg.len) resync();
} else if (msg.type === "peers_changed") {
setPeers(msg.count);
} else if (msg.type === "ping") {
@ -889,25 +1119,6 @@ async function copyToClipboard() {
setTimeout(() => setIcon(btn, "copy"), 1500);
}
// Adjust cursor position after a remote text update.
// Finds the single changed region (common prefix + suffix),
// then shifts the cursor accordingly:
// - change is after cursor → no movement
// - change is before cursor → shift by length delta
// - cursor was inside the changed region → place at end of new content
function adjustCursor(oldText, newText, pos) {
let start = 0;
const minLen = Math.min(oldText.length, newText.length);
while (start < minLen && oldText[start] === newText[start]) start++;
if (pos <= start) return pos; // change is entirely after cursor
let oldEnd = oldText.length, newEnd = newText.length;
while (oldEnd > start && newEnd > start && oldText[oldEnd - 1] === newText[newEnd - 1]) {
oldEnd--; newEnd--;
}
if (pos >= oldEnd) return pos + (newEnd - oldEnd); // change is before cursor
return newEnd; // cursor was inside changed region
}
connect();
// Handle Tab key to insert 4 spaces instead of navigation
@ -923,16 +1134,12 @@ ta.addEventListener("keydown", (e) => {
}
});
// Send edits (debounced)
// Send edits (coalesced). Short window: the payload is a splice now rather than the
// whole document, and the less local text is sitting unsent, the smaller the window in
// which two people's edits can overlap at all.
ta.addEventListener("input", () => {
dirty = true;
clearTimeout(debounce);
debounce = setTimeout(() => {
if (ws?.readyState === 1 && isAuthed) {
ws.send(JSON.stringify({type:"edit", ver, text: ta.value, clientId}));
dirty = false;
}
}, 120);
debounce = setTimeout(flush, 30);
});
</script>
"""
@ -1293,6 +1500,20 @@ def get_raw_pad_content(doc_id: str, request: Request, pw: str = ""):
update_room_access_time(doc_id)
return PlainTextResponse(room["text"])
def init_payload(room: dict, authed: bool) -> dict:
"""The full-snapshot message. Used for the first connect, after a successful unlock,
and for every resync, so a client resets its sync state through one code path."""
return {
"type": "init",
"text": room["text"] if authed else "",
"ver": room["ver"],
"protected": room["pw_hash"] is not None,
"peers": len(room["peers"]),
# Withheld until authed: the language is content metadata and would tell
# someone sitting at the unlock overlay what kind of file this is.
"lang": room.get("lang") if authed else None,
}
async def _broadcast(doc_id: str, message: dict, exclude: WebSocket | None = None, authed_only: bool = False):
room = rooms.get(doc_id)
if not room: return
@ -1368,16 +1589,7 @@ async def ws(doc_id: str, ws: WebSocket):
room["authed_peers"].add(ws)
# Send init; withhold text if protected and not yet authed
await ws.send_text(json.dumps({
"type": "init",
"text": room["text"] if authed else "",
"ver": room["ver"],
"protected": room["pw_hash"] is not None,
"peers": len(room["peers"]),
# Withheld until authed: the language is content metadata and would tell
# someone sitting at the unlock overlay what kind of file this is.
"lang": room.get("lang") if authed else None,
}))
await ws.send_text(json.dumps(init_payload(room, authed)))
try:
while True:
msg = await ws.receive_text()
@ -1388,10 +1600,7 @@ async def ws(doc_id: str, ws: WebSocket):
authed = True
room["authed_peers"].add(ws)
await ws.send_text(json.dumps({"type": "auth_ok"}))
await ws.send_text(json.dumps({
"type": "init", "text": room["text"], "ver": room["ver"], "protected": False,
"lang": room.get("lang"),
}))
await ws.send_text(json.dumps(init_payload(room, True)))
else:
if not check_auth_rate_limit(client_ip):
await ws.send_text(json.dumps({"type": "error", "message": "Too many failed attempts. Try again later."}))
@ -1404,10 +1613,7 @@ async def ws(doc_id: str, ws: WebSocket):
authed = True
room["authed_peers"].add(ws)
await ws.send_text(json.dumps({"type": "auth_ok"}))
await ws.send_text(json.dumps({
"type": "init", "text": room["text"], "ver": room["ver"], "protected": True,
"lang": room.get("lang"),
}))
await ws.send_text(json.dumps(init_payload(room, True)))
else:
record_auth_failure(client_ip)
await ws.send_text(json.dumps({"type": "error", "message": "Wrong password"}))
@ -1418,27 +1624,80 @@ async def ws(doc_id: str, ws: WebSocket):
continue
if data.get("type") == "edit":
new_text = str(data.get("text", ""))
op = parse_op(data.get("op"))
base = data.get("base")
if op is None or isinstance(base, bool) or not isinstance(base, int) or base < 0:
await ws.send_text(json.dumps({"type": "error", "message": "Malformed edit"}))
continue
# Check text size limit
if len(new_text.encode('utf-8')) > MAX_TEXT_SIZE:
# Derived state, initialized on the first edit rather than in the three
# places a room gets built. Both are then O(1) to read and to maintain,
# which is what keeps a keystroke off the length of the document.
log = room.setdefault("ops", deque(maxlen=MAX_OP_LOG))
astral = room.setdefault("astral", count_astral(room["text"]))
nbytes = room.setdefault("bytes", len(room["text"].encode("utf-8")))
# Rebase onto everything that landed since the client's base version. If
# the log no longer reaches that far back the client cannot be rebased and
# has to start from a snapshot - the same path a room restored from cache
# takes, since its op history did not survive.
if base < room["ver"] - len(log):
await ws.send_text(json.dumps(init_payload(room, True)))
continue
for v, prev in log:
if v > base:
# The logged op was ordered first, so the incoming one follows it.
op = transform(op, prev, False)
if op[0] + op[1] > len(room["text"]) + astral:
await ws.send_text(json.dumps(init_payload(room, True)))
continue
try:
new_text, removed = splice(room["text"], op, astral > 0)
delta = len(op[2].encode("utf-8")) - len(removed.encode("utf-8"))
except UnicodeEncodeError:
# A clamped op split a surrogate pair, so the result is not storable.
# Rare, and cheaper to resync than to repair.
await ws.send_text(json.dumps(init_payload(room, True)))
continue
if nbytes + delta > MAX_TEXT_SIZE:
# The client rolls its own text back to ours, so the two cannot
# diverge silently the way they used to when this was ignored.
await ws.send_text(json.dumps({"type": "error", "message": f"Text too large. Max size: {MAX_TEXT_SIZE} bytes"}))
continue
room["text"] = new_text
room["bytes"] = nbytes + delta
room["astral"] = astral - count_astral(removed) + count_astral(op[2])
room["ver"] += 1
room["last_access"] = time.time()
log.append((room["ver"], op))
while len(log) > 1 and sum(len(o[2]) for _, o in log) > MAX_OP_LOG_BYTES:
log.popleft()
# Save to cache
save_room_data_to_cache(doc_id, room)
# Goes to everyone including the author: the author recognises its own
# clientId and treats the echo as the ack, which carries the op as it was
# actually applied - rebased, if it arrived stale. `len` is the divergence
# tripwire: a client whose own length disagrees asks for a snapshot.
await _broadcast(doc_id, {
"type": "update",
"text": room["text"],
"op": {"pos": op[0], "del": op[1], "ins": op[2]},
"ver": room["ver"],
"len": len(room["text"]) + room["astral"],
"clientId": data.get("clientId"),
}, authed_only=True)
elif data.get("type") == "resync":
# A client whose shadow no longer matches us asks for a fresh snapshot.
# Unthrottled on purpose: unlike a ping this fans out to nobody, so the
# cost lands on the asker's own connection, same as hitting /raw.
await ws.send_text(json.dumps(init_payload(room, True)))
elif data.get("type") == "set_password":
pw = str(data.get("password", ""))
if pw: