diff --git a/app.py b/app.py
index 9bf09f5..5d3f599 100644
--- a/app.py
+++ b/app.py
@@ -3,7 +3,7 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request, HTTPExcept
from fastapi.responses import HTMLResponse, RedirectResponse, PlainTextResponse, FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
import json, re, secrets, string, time, os, threading, asyncio, hashlib
-from collections import defaultdict
+from collections import defaultdict, deque
from typing import Optional
app = FastAPI()
@@ -15,6 +15,11 @@ application = app # alias if you prefer "application"
app.mount("/static", StaticFiles(directory=os.path.join(os.path.dirname(os.path.abspath(__file__)), "vendor")),
name="vendor")
+# Minimum gap between accepted pings, per connection. A ping is one inbound message
+# fanned out to every peer, so this bounds that amplification. Low enough that rapid
+# human clicking gets through.
+PING_MIN_INTERVAL = 0.15
+
# Languages offered in the editor dropdown. This is a security boundary as much as a
# feature list: the value is interpolated into a module URL client-side, so it must
# never be free-form. Keep in sync with vendor/fetch.sh.
@@ -40,6 +45,111 @@ DOC_ID_RE = re.compile(r"^[a-zA-Z0-9_-]{1,64}$")
def is_valid_doc_id(doc_id: str) -> bool:
return bool(DOC_ID_RE.match(doc_id))
+# --- Operational transform core ------------------------------------------------
+# An op is a single splice: at `pos`, remove `dele` code units, insert `ins`. Every
+# textarea interaction - typing, deleting, pasting, Tab, autocorrect - reduces to one
+# splice, so this is the whole edit vocabulary. Ops travel over the wire instead of
+# document snapshots because two snapshots taken concurrently cannot be combined: one
+# has to win, which is what used to drop a peer's text and yank carets between lines.
+#
+# Offsets are UTF-16 code units, because that is what a textarea reports and what JS
+# string indexing uses. Python indexes by code point; the two diverge only on pads
+# holding astral characters (emoji), which is what splice()'s second path is for.
+#
+# diffOp/applyOp/transform in the client mirror this. The two copies MUST agree:
+# test_ot.py fuzzes convergence, and every update carries the server's length so a
+# disagreement resyncs instead of silently drifting.
+
+MAX_OP_LOG = 256 # ops kept per room, for rebasing edits from stale clients
+MAX_OP_LOG_BYTES = 262144 # ...and a size ceiling, so one big paste cannot pin memory
+
+
+def count_astral(s: str) -> int:
+ """Chars needing two UTF-16 code units. isascii() is a flag on the string object,
+ so the common case is O(1) and never walks the document."""
+ if s.isascii():
+ return 0
+ return sum(1 for ch in s if ord(ch) > 0xFFFF)
+
+
+def u16len(s: str) -> int:
+ return len(s) + count_astral(s)
+
+
+def splice(text: str, op, u16: bool):
+ """Apply an op; returns (new_text, removed_text).
+
+ `u16` says the text holds astral characters, so code-unit offsets no longer line up
+ with Python's code-point indexing and the splice has to happen in UTF-16 space. That
+ path costs an encode/decode of the document, which is why it is gated rather than
+ unconditional."""
+ pos, dele, ins = op
+ if not u16:
+ return text[:pos] + ins + text[pos + dele:], text[pos:pos + dele]
+ b = text.encode("utf-16-le")
+ lo, hi = pos * 2, (pos + dele) * 2
+ return ((b[:lo] + ins.encode("utf-16-le") + b[hi:]).decode("utf-16-le"),
+ b[lo:hi].decode("utf-16-le"))
+
+
+def transform(a, b, a_first: bool):
+ """Rewrite op `a` so it applies to a text that already has op `b` applied.
+
+ `a_first` says whether `a` precedes `b` in the server's total order - which is not the
+ same question as which one is already applied. The server rebases an incoming op onto
+ ops it has already ordered (a_first=False); a client rebases an op the server has
+ already ordered onto its own uncommitted edits (a_first=True). Both sides must agree
+ on the resulting order or their documents drift apart, so the flag is required rather
+ than defaulted."""
+ apos, adel, ains = a
+ bpos, bdel, bins = b
+ aend, bend = apos + adel, bpos + bdel
+ # Two pure inserts at one position do not overlap - only the order of the two
+ # insertions is ambiguous - so settle it on the server's ordering. Without this each
+ # side puts the other's text first and two people typing at the same spot diverge
+ # immediately.
+ if adel == 0 and bdel == 0 and apos == bpos:
+ return a if a_first else (apos + u16len(bins), 0, ains)
+ if bend <= apos: # b entirely before a
+ return (apos + u16len(bins) - bdel, adel, ains)
+ if aend <= bpos: # b entirely after a
+ return a
+ # The two edits touch the same characters. A single splice cannot express the honest
+ # result - a's surviving fragments are no longer contiguous - so clamp, identically on
+ # both sides so they still converge. Only reachable when two people edit the same
+ # characters inside one round trip, where any answer is arbitrary.
+ if apos < bpos:
+ return (apos, bpos - apos, ains)
+ return (bpos + u16len(bins), max(0, aend - bend), ains)
+
+
+def parse_op(data):
+ """Validate a wire op into (pos, dele, ins), or None. Structural checks only: the
+ range is checked after rebasing, since these offsets describe the client's older
+ version of the text, and the size limit is checked against the resulting document so
+ that an oversize paste gets told it was too large rather than malformed."""
+ if not isinstance(data, dict):
+ return None
+ pos, dele, ins = data.get("pos"), data.get("del"), data.get("ins")
+ # bool subclasses int, so True would otherwise pass as position 1
+ if isinstance(pos, bool) or isinstance(dele, bool):
+ return None
+ if not isinstance(pos, int) or not isinstance(dele, int) or not isinstance(ins, str):
+ return None
+ if pos < 0 or dele < 0:
+ return None
+ if "\x00" in ins: # matches the POST / check
+ return None
+ try:
+ ins.encode("utf-8") # rejects lone surrogates, which cannot be stored
+ except UnicodeEncodeError:
+ return None
+ return (pos, dele, ins)
+
+# --- end operational transform core --------------------------------------------
+# test_ot.py slices the block between these two markers and execs it, so that the
+# fuzzer can run on a bare python with no fastapi installed. Keep them in place.
+
def get_client_ip(conn) -> str:
# Only honor proxy headers when explicitly opted in — otherwise attackers
# can spoof them to bypass per-IP limits.
@@ -188,10 +298,15 @@ def record_auth_failure(client_ip: str):
HTML = """
+
aukpad
-
- disconnected
+
+ disconnected
+
-
+
+
+
@@ -333,11 +537,20 @@ HTML = """
1
+
+
+
+
+ Content is deleted after __RETENTION_HOURS__ hours of inactivity and can be accessed
+ by the server and anyone with the link and optional password.
+
+
+
This pad is password protected
@@ -381,17 +594,116 @@ function toggleTheme() {
});
})();
+// Bottom notice: click anywhere on it to dismiss, remembered across pads and reloads.
+function hideNotice() {
+ $("#notice").style.display = "none";
+ try { localStorage.setItem("notice", "off"); } catch (e) {}
+}
+(function() {
+ try { if (localStorage.getItem("notice") === "off") $("#notice").style.display = "none"; }
+ catch (e) {} // private mode / storage disabled: just leave the notice up
+})();
+
// Derive docId from path; redirect root to random
let docId = decodeURIComponent(location.pathname.replace(/(^\\/|\\/$)/g, ""));
if (!docId) { location.replace("/" + rand() + "/"); }
$("#padname").textContent = "/"+docId+"/";
+// Set client-side rather than server-side: the pad page is one shared HTML constant,
+// so interpolating per request would mean rebuilding it on every hit.
+document.title = docId + " - aukpad";
let ws, ver = 0, clientId = Math.random().toString(36).slice(2), debounce;
let isProtected = false, isAuthed = false;
-// dirty: local edits the server has not acknowledged. Guards the reconnect path
-// from overwriting text typed while the socket was down.
-let dirty = false, reconnectTimer = null, reconnectDelay = 500;
+let reconnectTimer = null, reconnectDelay = 500;
+
+// --- Sync state ---
+// Invariant: shadow -> sent -> ta.value, each step being one splice.
+// shadow the server text this client knows about, at version `ver`
+// sent shadow with the op currently in flight applied, i.e. what the server will
+// hold once it acks. Equal to shadow when nothing is in flight.
+// Ops are never stored, only derived by diffing these three strings. That is what keeps
+// the state machine small enough to reason about: there is no operation queue.
+let shadow = "", sent = "", inflight = false, resyncing = false;
+
+// --- Operational transform core (mirror of the Python side; the two MUST agree) ---
+// An op is one splice {pos, del, ins} in UTF-16 code units, which is what textarea
+// offsets and JS string indices already use, so nothing is converted on this side.
+const isHigh = c => c >= 0xD800 && c <= 0xDBFF;
+const isLow = c => c >= 0xDC00 && c <= 0xDFFF;
+
+// The single changed region between two texts: common prefix, common suffix, and
+// whatever is left in the middle. Every textarea interaction produces exactly one.
+function diffOp(a, b) {
+ if (a === b) return null;
+ let s = 0;
+ const min = Math.min(a.length, b.length);
+ while (s < min && a[s] === b[s]) s++;
+ let ae = a.length, be = b.length;
+ while (ae > s && be > s && a[ae-1] === b[be-1]) { ae--; be--; }
+ // Never split a surrogate pair: a lone surrogate has no UTF-8 encoding, so the server
+ // would reject the op instead of syncing it. Nudge the boundaries outward instead.
+ if (s > 0 && isLow(a.charCodeAt(s)) && isHigh(a.charCodeAt(s-1))) s--;
+ if (ae < a.length && isLow(a.charCodeAt(ae)) && isHigh(a.charCodeAt(ae-1))) { ae++; be++; }
+ return {pos: s, del: ae - s, ins: b.slice(s, be)};
+}
+
+function applyOp(text, op) {
+ return text.slice(0, op.pos) + op.ins + text.slice(op.pos + op.del);
+}
+
+// Rewrite op `a` so it applies to a text that already has op `b` applied. `aFirst` says
+// whether `a` precedes `b` in the server's total order, which is a different question
+// from which of the two is already applied: here it is always the op the server has
+// ordered that comes first, and the server applies the identical rule.
+function transform(a, b, aFirst) {
+ const aEnd = a.pos + a.del, bEnd = b.pos + b.del;
+ // Two pure inserts at one position do not overlap - only the order of the two
+ // insertions is ambiguous - so settle it on the server's ordering. Without this each
+ // side puts the other's text first and two people typing at the same spot diverge
+ // immediately.
+ if (!a.del && !b.del && a.pos === b.pos)
+ return aFirst ? a : {pos: a.pos + b.ins.length, del: 0, ins: a.ins};
+ if (bEnd <= a.pos) return {pos: a.pos + b.ins.length - b.del, del: a.del, ins: a.ins};
+ if (aEnd <= b.pos) return a;
+ // Same-characters overlap, which one splice cannot express faithfully. Clamp exactly
+ // as the server does; the length tripwire below catches anything this gets wrong.
+ if (a.pos < b.pos) return {pos: a.pos, del: b.pos - a.pos, ins: a.ins};
+ return {pos: b.pos + b.ins.length, del: Math.max(0, aEnd - bEnd), ins: a.ins};
+}
+
+// Our picture of the server is wrong, so stop guessing and ask for a snapshot. Cheap to
+// trigger and self-healing, which is what lets transform() clamp overlaps crudely rather
+// than carry the machinery needed to always be exactly right. Blocks sending meanwhile:
+// an op diffed against a bad shadow would be nonsense.
+function resync() {
+ if (resyncing || ws?.readyState !== 1) return;
+ resyncing = true;
+ ws.send(JSON.stringify({type: "resync"}));
+}
+
+// Send edits: one op in flight at a time, so the server always has a base version to
+// rebase against. Anything typed while it is in flight is picked up by the next diff.
+function flush() {
+ if (inflight || resyncing || ws?.readyState !== 1 || !isAuthed) return;
+ const op = diffOp(shadow, ta.value);
+ if (!op) return;
+ ws.send(JSON.stringify({type: "edit", base: ver, op, clientId}));
+ sent = ta.value;
+ inflight = true;
+}
+
+// Transient message in the status slot. There is no error surface in the editor, which
+// is how a refused edit used to pass unnoticed.
+let notifyTimer = null;
+function notify(text) {
+ const el = $("#status");
+ el.textContent = text;
+ clearTimeout(notifyTimer);
+ notifyTimer = setTimeout(() => {
+ el.textContent = ws?.readyState === 1 ? "connected" : "disconnected";
+ }, 4000);
+}
const urlPw = new URLSearchParams(location.search).get("pw") || "";
// --- Line numbers ---
@@ -405,12 +717,85 @@ function updateGutter() {
gutter.textContent = s;
}
ta.addEventListener("input", refresh);
+// The textarea's horizontal scrollbar eats into its client height, so it can scroll
+// further than the mirrors, which have overflow:hidden and no scrollbar. At the bottom
+// they clamp and lag behind by the scrollbar's height. Give them matching bottom padding
+// so their scrollable extents line up. Recomputed on input and resize, the only times a
+// horizontal scrollbar can appear or vanish.
+function syncExtent() {
+ const sb = ta.offsetHeight - ta.clientHeight; // 0 when there is no h-scrollbar
+ const pad = parseFloat(getComputedStyle(ta).paddingBottom) + sb;
+ gutter.style.paddingBottom = pad + "px";
+ hl.style.paddingBottom = pad + "px";
+ // Same idea horizontally, for anything anchored to the editor's right edge.
+ document.documentElement.style.setProperty("--sbw", (ta.offsetWidth - ta.clientWidth) + "px");
+}
+window.addEventListener("resize", () => { syncExtent(); syncScroll(); });
+
function syncScroll() {
gutter.scrollTop = ta.scrollTop;
// The overlay needs horizontal sync too: white-space:pre means long lines
// scroll sideways, and the gutter never does.
hl.scrollTop = ta.scrollTop; hl.scrollLeft = ta.scrollLeft;
+ // Pings sit at document coordinates; move the wrapper rather than every bar.
+ pingsIn.style.transform = "translateY(" + (-ta.scrollTop) + "px)";
+ // Only drop the notice if the document shrank past the line it points at.
+ if (jumpLine && jumpLine > lineCount()) hideJump();
}
+
+// --- Line ping: click a line number to flash that line for every peer ---
+// Read from computed style, never hardcoded: the touch media query switches the
+// editor font from 14px to 16px, so a literal line height would be wrong on mobile.
+const pingsIn = $("#pings-in");
+const lineH = () => parseFloat(getComputedStyle(ta).lineHeight);
+const padTop = () => parseFloat(getComputedStyle(ta).paddingTop);
+const lineCount = () => ta.value.split("\\n").length;
+
+gutter.addEventListener("click", (e) => {
+ if (!isAuthed || ws?.readyState !== 1) return;
+ const y = e.clientY - gutter.getBoundingClientRect().top + ta.scrollTop - padTop();
+ const line = Math.floor(y / lineH()) + 1;
+ if (line < 1 || line > lineCount()) return; // e.g. clicked below the last line
+ ws.send(JSON.stringify({type: "ping", line}));
+});
+
+function showPing(line) {
+ // Re-pinging a visible line restarts it rather than stacking a second bar.
+ const prev = pingsIn.querySelector('[data-line="' + line + '"]');
+ if (prev) prev.remove();
+ const el = document.createElement("div");
+ el.className = "ping";
+ el.dataset.line = line;
+ el.style.top = (padTop() + (line - 1) * lineH()) + "px";
+ el.style.height = lineH() + "px";
+ el.addEventListener("animationend", () => el.remove());
+ pingsIn.appendChild(el);
+}
+
+// --- Ping notice ---
+// Raised for every ping, for every peer, whether or not the line is on screen: on a long
+// pad the bar alone is easy to miss. Click it to jump to the line and re-flash it.
+const jump = $("#pingjump");
+let jumpLine = 0, jumpTimer = null;
+
+function showJump(line) {
+ jumpLine = line;
+ $("#pingjump-txt").textContent = "Ping on line " + line;
+ jump.classList.toggle("up", padTop() + (line - 1) * lineH() < ta.scrollTop);
+ jump.classList.add("show");
+ clearTimeout(jumpTimer);
+ jumpTimer = setTimeout(hideJump, 10000);
+}
+
+function hideJump() { jump.classList.remove("show"); jumpLine = 0; }
+
+jump.addEventListener("click", () => {
+ const line = jumpLine;
+ hideJump();
+ ta.scrollTop = Math.max(0, padTop() + (line - 1) * lineH() - (ta.clientHeight - lineH()) / 2);
+ syncScroll();
+ showPing(line); // re-flash: the original bar may have almost faded by now
+});
ta.addEventListener("scroll", syncScroll);
// Also sync on keydown for immediate response
ta.addEventListener("keydown", () => { setTimeout(syncScroll, 0); });
@@ -487,6 +872,7 @@ function scheduleHl() {
// Re-render gutter and highlighting together.
function refresh() {
updateGutter();
+ syncExtent();
if (ta.value.length > HL_MAX) {
wrap.classList.remove("hl");
langSel.disabled = true;
@@ -568,6 +954,13 @@ function submitAuth() {
}
}
+function setPeers(n) {
+ const el = $("#peers");
+ $("#peer-count").textContent = n; // never el.textContent: it would wipe the