feat: ready to submit, a run to watch, the README, the review's fixes

Sprint 08 (roadmap/08. ready to submit.md), the last milestone.

- scripts/watch.sh runs the collector with a live view of its log, stops
  cleanly on Ctrl-C, then checks the output (reconcile) and shows one
  symbol's book as text (show, a new binary).
- The README for the solution, with the final runs: Polymarket through an
  outage it recovered from on its own, Kalshi's demo with two slow
  connections recycled.
- From a review of the whole codebase: Kalshi takes RSA keys too
  (RSA-PSS-SHA256); a slow Kalshi connection takes the hourly recycle's
  next turn, which it nearly never got before; a market tracked again no
  longer fails reconcile; Book::add checks for overflow; usage and --help
  name the key variables; the first scan says how far it got.
- Tests for every gap the reviews found, an offline run of the whole
  orchestrator among them; ShellCheck in CI.
This commit is contained in:
pascal committed 2026-11-12 22:32:39 +01:00
1 parent 515e85941d
commit cf6c16c9cf
35 files changed
+2791 -207

No files matched your search

+5
View File
@@ -13,3 +13,8 @@ cargo test
uvx ruff@0.15.0 format --check scripts/latency
uvx ruff@0.15.0 check scripts/latency
python3 -m unittest discover -s scripts/latency
# The shell scripts: ShellCheck pinned, run through uvx as ruff is.
uvx --from shellcheck-py==0.10.0.1 shellcheck scripts/*.sh
# The view of watch.sh, on a sample log.
sh scripts/test_watch.sh
+157
View File
@@ -0,0 +1,157 @@
#!/bin/sh
# Pipes a sample log of each exchange through scripts/watch.awk, live and
# with -v final=1, and compares what it prints with what the view must show;
# exits non-zero on a difference. The lines are worded as the collector
# writes them: the book threads' minute lines (log_minute in
# src/polymarket/feed.rs and src/kalshi/feed.rs), the orchestrator's
# (src/core/orchestrator.rs), the connections' (src/core/socket.rs) and a
# Snapshot's cause (src/cause.rs).
set -eu
cd "$(dirname "$0")/.."
failed=0
# check NAME LOG EXPECTED [AWK OPTION...]
check() {
name=$1 log=$2 expected=$3
shift 3
actual=$(printf '%s\n' "$log" | awk -v book_threads=2 "$@" -f scripts/watch.awk)
if [ "$actual" != "$expected" ]; then
printf 'watch.awk, %s: printed\n%s\nexpected\n%s\n' "$name" "$actual" "$expected"
failed=1
fi
}
polymarket=$(
cat <<'EOF'
12:00:00.100 polymarket: orchestrator: first scan, listing the exchange's markets
12:00:30.200 polymarket: orchestrator: first scan done, 40 live markets, subscribing
12:00:30.300 polymarket: connection 0: connected, 40 symbols
12:00:30.310 polymarket: connection 1: connected, 40 symbols
12:01:00.001 polymarket: book thread 0: 10 symbols; so far 100 changes written (90 verified), 50 duplicates, 2 rescued, 0 held dropped, 1 chains broken, 3 resyncs, 0 full books behind ours, 1 subscription gaps, 2 losses, 0 switches, 0 quiet subscriptions resubscribed, 0 markets reported quiet again, 4 reconnect books judged (1 losses); duplicates' lateness 1.5 ms at the median, 9.0 ms at p99; this minute 150 changes received, at most 4 events waited
12:01:00.002 polymarket: book thread 1: 30 symbols; so far 300 changes written (290 verified), 150 duplicates, 4 rescued, 0 held dropped, 0 chains broken, 1 resyncs, 0 full books behind ours, 0 subscription gaps, 1 losses, 0 switches, 0 quiet subscriptions resubscribed, 0 markets reported quiet again, 0 reconnect books judged (0 losses); duplicates' lateness 2.0 ms at the median, 8.0 ms at p99; this minute 450 changes received, at most 9 events waited
12:01:00.050 polymarket: Snapshot of 123 at 1794390060050000: loss
12:01:30.400 polymarket: orchestrator: 40 markets on 2 connections; 40 added, 0 removed, 0 moved (0 quiet), 0 splits, 0 recycled (0 slow), 0 silent this minute; busiest connection at 1200 bytes/s
12:02:00.001 polymarket: book thread 0: 10 symbols; so far 160 changes written (150 verified), 80 duplicates, 3 rescued, 0 held dropped, 1 chains broken, 3 resyncs, 0 full books behind ours, 1 subscription gaps, 2 losses, 0 switches, 1 quiet subscriptions resubscribed, 0 markets reported quiet again, 4 reconnect books judged (1 losses); duplicates' lateness 1.5 ms at the median, 9.0 ms at p99; this minute 90 changes received, at most 2 events waited
12:02:00.002 polymarket: book thread 1: 30 symbols; so far 500 changes written (480 verified), 250 duplicates, 6 rescued, 0 held dropped, 0 chains broken, 2 resyncs, 0 full books behind ours, 0 subscription gaps, 1 losses, 0 switches, 0 quiet subscriptions resubscribed, 0 markets reported quiet again, 0 reconnect books judged (0 losses); duplicates' lateness 2.0 ms at the median, 8.0 ms at p99; this minute 300 changes received, at most 5 events waited
12:02:10.000 polymarket: connection 1: recycled
12:02:11.000 polymarket: connection 1: connected, 40 symbols
12:02:30.400 polymarket: orchestrator: 40 markets on 2 connections; 0 added, 0 removed, 0 moved (0 quiet), 0 splits, 1 recycled (0 slow), 0 silent this minute; busiest connection at 1300 bytes/s
EOF
)
check "Polymarket, live" "$polymarket" "$(
cat <<'EOF'
12:00:00 polymarket: orchestrator: first scan, listing the exchange's markets
12:00:30 polymarket: orchestrator: first scan done, 40 live markets, subscribing
12:00:30 connections opening (counted in the minute line)
12:01 books (2 threads): 40 symbols, 400 changes written, 200 duplicates, 6 rescued, 4 resyncs, 1 subscription gaps, 3 losses
12:01:30 40 markets on 2 connections; 40 added, 0 removed, 0 moved (0 quiet), 0 splits, 0 recycled (0 slow), 0 silent this minute; busiest connection at 1200 bytes/s; 2 connected (0 refused, retried), 1 Snapshots with a cause
12:02 books (2 threads): 40 symbols, 660 changes written, 330 duplicates, 9 rescued, 5 resyncs, 1 subscription gaps, 3 losses
12:02:10 polymarket: connection 1: recycled
12:02:30 40 markets on 2 connections; 0 added, 0 removed, 0 moved (0 quiet), 0 splits, 1 recycled (0 slow), 0 silent this minute; busiest connection at 1300 bytes/s; 1 connected (0 refused, retried), 0 Snapshots with a cause
EOF
)"
check "Polymarket, final" "$polymarket" \
"12:02 books (2 threads): 40 symbols, 660 changes written, 330 duplicates, 9 rescued, 5 resyncs, 1 subscription gaps, 3 losses" \
-v final=1
# A minute with one of its two book threads' lines: printed with the first
# line 10 s after it, before the orchestrator's.
missing=$(
cat <<'EOF'
12:01:00.001 polymarket: book thread 0: 10 symbols; so far 100 changes written (90 verified), 50 duplicates, 2 rescued, 0 held dropped, 1 chains broken, 3 resyncs, 0 full books behind ours, 1 subscription gaps, 2 losses, 0 switches, 0 quiet subscriptions resubscribed, 0 markets reported quiet again, 4 reconnect books judged (1 losses); duplicates' lateness 1.5 ms at the median, 9.0 ms at p99; this minute 150 changes received, at most 4 events waited
12:01:00.500 polymarket: connection 1: recycled
12:01:30.400 polymarket: orchestrator: 40 markets on 2 connections; 0 added, 0 removed, 0 moved (0 quiet), 0 splits, 1 recycled (0 slow), 0 silent this minute; busiest connection at 1200 bytes/s
EOF
)
check "a book thread missing" "$missing" "$(
cat <<'EOF'
12:01:00 polymarket: connection 1: recycled
12:01 books (1 threads): 10 symbols, 100 changes written, 50 duplicates, 2 rescued, 3 resyncs, 1 subscription gaps, 2 losses
12:01:30 40 markets on 2 connections; 0 added, 0 removed, 0 moved (0 quiet), 0 splits, 1 recycled (0 slow), 0 silent this minute; busiest connection at 1200 bytes/s; 0 connected (0 refused, retried), 0 Snapshots with a cause
EOF
)"
# A thread's line 1.6 s after the first, past a line between them: still
# the same minute's, summed whole.
late=$(
cat <<'EOF'
12:01:00.075 polymarket: book thread 0: 10 symbols; so far 100 changes written (90 verified), 50 duplicates, 2 rescued, 0 held dropped, 1 chains broken, 3 resyncs, 0 full books behind ours, 1 subscription gaps, 2 losses, 0 switches, 0 quiet subscriptions resubscribed, 0 markets reported quiet again, 4 reconnect books judged (1 losses); duplicates' lateness 1.5 ms at the median, 9.0 ms at p99; this minute 150 changes received, at most 4 events waited
12:01:01.600 polymarket: connection 1: recycled
12:01:01.672 polymarket: book thread 1: 30 symbols; so far 300 changes written (290 verified), 150 duplicates, 4 rescued, 0 held dropped, 0 chains broken, 1 resyncs, 0 full books behind ours, 0 subscription gaps, 1 losses, 0 switches, 0 quiet subscriptions resubscribed, 0 markets reported quiet again, 0 reconnect books judged (0 losses); duplicates' lateness 2.0 ms at the median, 8.0 ms at p99; this minute 450 changes received, at most 9 events waited
EOF
)
check "a thread 1.6 s late" "$late" "$(
cat <<'EOF'
12:01:01 polymarket: connection 1: recycled
12:01 books (2 threads): 40 symbols, 400 changes written, 200 duplicates, 6 rescued, 4 resyncs, 1 subscription gaps, 3 losses
EOF
)"
check "a thread 1.6 s late, final" "$late" \
"12:01 books (2 threads): 40 symbols, 400 changes written, 200 duplicates, 6 rescued, 4 resyncs, 1 subscription gaps, 3 losses" \
-v final=1
# Across midnight: a minute begun at 23:59 still ends 10 s on.
midnight=$(
cat <<'EOF'
23:59:00.001 polymarket: book thread 0: 10 symbols; so far 100 changes written (90 verified), 50 duplicates, 2 rescued, 0 held dropped, 1 chains broken, 3 resyncs, 0 full books behind ours, 1 subscription gaps, 2 losses, 0 switches, 0 quiet subscriptions resubscribed, 0 markets reported quiet again, 4 reconnect books judged (1 losses); duplicates' lateness 1.5 ms at the median, 9.0 ms at p99; this minute 150 changes received, at most 4 events waited
00:00:30.000 polymarket: connection 1: recycled
EOF
)
check "across midnight" "$midnight" "$(
cat <<'EOF'
23:59 books (1 threads): 10 symbols, 100 changes written, 50 duplicates, 2 rescued, 3 resyncs, 1 subscription gaps, 2 losses
00:00:30 polymarket: connection 1: recycled
EOF
)"
# A line between a minute's book threads: the minute is still summed whole.
interleaved=$(
cat <<'EOF'
12:01:00.001 polymarket: book thread 0: 10 symbols; so far 100 changes written (90 verified), 50 duplicates, 2 rescued, 0 held dropped, 1 chains broken, 3 resyncs, 0 full books behind ours, 1 subscription gaps, 2 losses, 0 switches, 0 quiet subscriptions resubscribed, 0 markets reported quiet again, 4 reconnect books judged (1 losses); duplicates' lateness 1.5 ms at the median, 9.0 ms at p99; this minute 150 changes received, at most 4 events waited
12:01:00.002 polymarket: connection 1: recycled
12:01:00.003 polymarket: book thread 1: 30 symbols; so far 300 changes written (290 verified), 150 duplicates, 4 rescued, 0 held dropped, 0 chains broken, 1 resyncs, 0 full books behind ours, 0 subscription gaps, 1 losses, 0 switches, 0 quiet subscriptions resubscribed, 0 markets reported quiet again, 0 reconnect books judged (0 losses); duplicates' lateness 2.0 ms at the median, 8.0 ms at p99; this minute 450 changes received, at most 9 events waited
EOF
)
check "a line between book threads" "$interleaved" "$(
cat <<'EOF'
12:01:00 polymarket: connection 1: recycled
12:01 books (2 threads): 40 symbols, 400 changes written, 200 duplicates, 6 rescued, 4 resyncs, 1 subscription gaps, 3 losses
EOF
)"
kalshi=$(
cat <<'EOF'
12:00:00.100 kalshi: orchestrator: first scan, listing the exchange's markets
12:00:10.100 kalshi: orchestrator: first scan, 1000 live markets listed so far
12:00:20.200 kalshi: orchestrator: first scan done, 1500 live markets, subscribing
12:00:20.300 kalshi: connection 0: connected, 1500 symbols
12:00:20.310 kalshi: connection 1: connect failed: handshake refused: 429 Too Many Requests
12:00:21.320 kalshi: connection 1: connected, 1500 symbols
12:01:00.001 kalshi: book thread 0: 700 symbols; so far 100 changes written, 90 duplicates, 1 before their connection's book, 0 ahead of the book, 3 fresh books (1 losses), 0 below zero, 2 resyncs, 0 quiet subscriptions resubscribed, 0 markets reported quiet again; this minute 190 changes received, at most 3 events waited
12:01:00.002 kalshi: book thread 1: 800 symbols; so far 200 changes written, 190 duplicates, 0 before their connection's book, 1 ahead of the book, 2 fresh books (0 losses), 1 below zero, 1 resyncs, 0 quiet subscriptions resubscribed, 0 markets reported quiet again; this minute 390 changes received, at most 6 events waited
12:01:00.030 kalshi: Snapshot of KXA at 1794390060030000: loss (its clock 12, ours 11)
12:01:20.400 kalshi: orchestrator: 1500 markets on 2 connections; 1500 added, 0 removed, 0 moved (0 quiet), 0 splits, 0 recycled (0 slow), 0 silent this minute; busiest connection at 9000 bytes/s
12:02:00.001 kalshi: book thread 0: 700 symbols; so far 150 changes written, 140 duplicates, 1 before their connection's book, 0 ahead of the book, 3 fresh books (1 losses), 0 below zero, 2 resyncs, 0 quiet subscriptions resubscribed, 0 markets reported quiet again; this minute 100 changes received, at most 2 events waited
12:02:00.002 kalshi: book thread 1: 800 symbols; so far 260 changes written, 250 duplicates, 0 before their connection's book, 1 ahead of the book, 2 fresh books (0 losses), 1 below zero, 1 resyncs, 0 quiet subscriptions resubscribed, 0 markets reported quiet again; this minute 120 changes received, at most 4 events waited
12:02:20.400 kalshi: orchestrator: 1500 markets on 2 connections; 0 added, 0 removed, 0 moved (0 quiet), 0 splits, 0 recycled (0 slow), 0 silent this minute; busiest connection at 8000 bytes/s
EOF
)
check "Kalshi, live" "$kalshi" "$(
cat <<'EOF'
12:00:00 kalshi: orchestrator: first scan, listing the exchange's markets
12:00:10 kalshi: orchestrator: first scan, 1000 live markets listed so far
12:00:20 kalshi: orchestrator: first scan done, 1500 live markets, subscribing
12:00:20 connections opening (counted in the minute line)
12:01 books (2 threads): 1500 symbols, 300 changes written, 280 duplicates, 3 resyncs, 1 losses
12:01:20 1500 markets on 2 connections; 1500 added, 0 removed, 0 moved (0 quiet), 0 splits, 0 recycled (0 slow), 0 silent this minute; busiest connection at 9000 bytes/s; 2 connected (1 refused, retried), 1 Snapshots with a cause
12:02 books (2 threads): 1500 symbols, 410 changes written, 390 duplicates, 3 resyncs, 1 losses
12:02:20 1500 markets on 2 connections; 0 added, 0 removed, 0 moved (0 quiet), 0 splits, 0 recycled (0 slow), 0 silent this minute; busiest connection at 8000 bytes/s; 0 connected (0 refused, retried), 0 Snapshots with a cause
EOF
)"
check "Kalshi, final" "$kalshi" \
"12:02 books (2 threads): 1500 symbols, 410 changes written, 390 duplicates, 3 resyncs, 1 losses" \
-v final=1
exit "$failed"
+107
View File
@@ -0,0 +1,107 @@
# The view of scripts/watch.sh: reads the collector's log and prints, each
# minute, the orchestrator's line and the book threads' lines summed into one,
# and between them only the notable events. The Snapshot cause lines and the
# connections coming up are counted, not shown.
#
# With -v final=1 it prints nothing live, only the last minute's sums: the
# run's totals, since the book threads count "so far". A minute's sum is
# printed once all its book threads' lines are in (-v book_threads=N, 16 by
# default, src/core/mod.rs's BOOK_THREADS), or with the first other line 10 s
# or more after the minute's first, or when the next minute begins. A book
# thread that received nothing logs no line; the others' come within a few
# seconds of the minute (1.6 s apart at most in a 10-minute Polymarket run,
# 2026-11-12), and other lines often fall between them.
# The book threads' counters shown, in this order; a name missing from an
# exchange's line (Kalshi has no "rescued") is left out.
BEGIN {
if (book_threads == "") book_threads = 16
shown = "symbols|changes written|duplicates|rescued|resyncs|subscription gaps|losses"
count = split(shown, names, "|")
}
function flush( line, i) {
if (threads == 0) return
line = minute " books (" threads " threads):"
for (i = 1; i <= count; i++)
if (names[i] in sum) line = line " " sum[names[i]] " " names[i] ","
sub(/,$/, "", line)
last = line
event(line)
threads = 0
split("", sum)
}
# Printed at once: the view is read while the run goes on, and mawk
# buffers its output otherwise.
function event(text) {
if (final) return
print text
fflush()
}
{
time = substr($1, 1, 8)
text = substr($0, index($0, " ") + 1)
# To the millisecond: $1 is HH:MM:SS.mmm.
seconds = substr($1, 1, 2) * 3600 + substr($1, 4, 2) * 60 + substr($1, 7)
}
/: book thread [0-9]+: / {
this = substr(time, 1, 5)
if (threads > 0 && this != minute) flush()
if (threads == 0) first_at = seconds
minute = this
threads++
body = text
sub(/; this minute .*/, "", body)
split("", seen)
# Each "N name": a count and its name, up to the next comma, semicolon
# or parenthesis. A name is summed once a line (its first occurrence).
while (match(body, /[0-9]+ [a-z][a-z' ]*[a-z]/)) {
pair = substr(body, RSTART, RLENGTH)
body = substr(body, RSTART + RLENGTH)
n = substr(pair, 1, index(pair, " ") - 1)
name = substr(pair, index(pair, " ") + 1)
sub(/^so far /, "", name)
if (name in seen) continue
seen[name] = 1
sum[name] += n
}
if (threads == book_threads) flush()
next
}
# A minute with a book thread's line missing: printed once it is clearly over
# (a day's seconds added back across midnight).
threads > 0 && (seconds - first_at + 86400) % 86400 >= 10 { flush() }
/: orchestrator: [0-9]+ markets on/ {
line = text
sub(/^[a-z-]+: orchestrator: /, "", line)
event(time " " line "; " up + 0 " connected (" refused + 0 " refused, retried), " snapshots + 0 " Snapshots with a cause")
up = 0
refused = 0
snapshots = 0
next
}
/: connection [0-9]+: connected, / {
if (!opened) event(time " connections opening (counted in the minute line)")
opened = 1
up++
next
}
# Kalshi refuses some of ~100 handshakes opened at once (429); each retries.
/: connect failed: handshake refused: 429/ { refused++; next }
/: Snapshot of / { snapshots++; next }
/ changes go unchecked$| tick size changed, resync$| unreadable / { next }
# Everything else is an event worth seeing: a connection lost, recycled,
# split, moved off when quiet, a refused handshake, a failed fetch, the end.
{ event(time " " text) }
END {
flush()
if (final && last != "") print last
}
+101
View File
@@ -0,0 +1,101 @@
#!/bin/sh
# watch.sh [EXCHANGE] [MINUTES]: runs the collector and shows what it does.
# EXCHANGE is polymarket (the default, no key needed), kalshi or kalshi-demo
# (KALSHI_KEY_ID_FILE and KALSHI_PRIVATE_KEY_FILE exported); MINUTES
# defaults to 3. Ctrl-C stops the run early, cleanly.
#
# The output and the full log go to watch/; the terminal shows, each minute,
# the orchestrator's line and the book threads' summed (scripts/watch.awk),
# and the notable events between. Then the output is checked against the
# log (reconcile) and one symbol is shown as text (show). It exits non-zero
# when the collector failed or the output doesn't reconcile.
set -eu
cd "$(dirname "$0")/.."
exchange=${1:-polymarket}
minutes=${2:-3}
case $exchange in
polymarket | kalshi | kalshi-demo) ;;
*)
echo "usage: scripts/watch.sh [polymarket|kalshi|kalshi-demo] [MINUTES]" >&2
exit 2
;;
esac
echo "building (cargo build --release)"
cargo build --release --quiet
mkdir -p watch
out=watch/$exchange.sbe log=watch/$exchange.log memory=watch/$exchange.memory
rm -f "$out" "$log" "$memory"
: >"$log"
echo "load before the run: $(cut -d' ' -f1-3 /proc/loadavg) (1, 5, 15 min)"
echo "collecting $exchange for $minutes minutes into $out (log: $log); Ctrl-C stops it"
echo "the markets are listed first (a minute or two), then subscribed"
timeout -s INT "${minutes}m" ./target/release/engineering-challenge "$exchange" "$out" 2>>"$log" &
run=$!
# The view ends by itself once the run is over and the log read to its end.
# mawk reads a pipe a buffer at a time unless interactive: the view would
# show nothing until the end.
live=
if awk -W version 2>/dev/null | grep -q mawk; then live='-W interactive'; fi
# shellcheck disable=SC2086 # $live is empty or two words, both wanted.
tail -n +1 -F --pid="$run" "$log" 2>/dev/null | awk $live -f scripts/watch.awk &
view=$!
# A job started with & ignores Ctrl-C (sh without job control), so the
# script passes it on: timeout hands SIGINT to the collector, which stops
# cleanly, and the script carries on to the checks. The view, also a job,
# keeps showing the end.
trap 'echo; echo "stopping (Ctrl-C)"; kill -INT "$run" 2>/dev/null || true' INT
# The collector's peak memory (VmHWM, the kernel's own peak) and most
# threads, sampled every 10 s while it runs, from /proc: /usr/bin/time isn't
# installed everywhere. The last sample may fall in the shutdown, when the
# connections' threads are gone: the most seen is kept.
(
sleep 1
pid=$(pgrep -P "$run" || true)
most=0
while [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; do
proc=$(cat "/proc/$pid/status" 2>/dev/null || true)
peak=$(printf '%s\n' "$proc" | awk '/^VmHWM:/ {print $2}')
threads=$(printf '%s\n' "$proc" | awk '/^Threads:/ {print $2}')
if [ -n "$threads" ] && [ "$threads" -gt "$most" ]; then most=$threads; fi
if [ -n "$peak" ]; then echo "$peak kB, at most $most threads" >"$memory"; fi
sleep 10
done
) &
# wait returns early when Ctrl-C interrupts it: wait again until the
# collector has flushed and exited.
status=0
while kill -0 "$run" 2>/dev/null; do
wait "$run" && status=0 || status=$?
done
trap - INT
wait "$view" || true
# 124: stopped by timeout, at the end or on Ctrl-C (passed on by it).
failed=0
case $status in
0 | 124 | 130) ;;
*)
echo "the collector failed (exit $status); its log: $log"
failed=1
;;
esac
if [ ! -e "$out" ]; then
echo "no output was written"
exit 1
fi
echo
echo "== totals at the last minute"
awk -v final=1 -f scripts/watch.awk "$log"
if [ -s "$memory" ]; then echo "peak memory $(cat "$memory")"; fi
echo
echo "== the output against the log (reconcile)"
./target/release/reconcile "$out" "$log" || failed=1
echo
echo "== one symbol, as text (show)"
# An output stopped before any book is empty: said, not a failure.
./target/release/show "$out" || true
exit "$failed"