# The view of scripts/watch.sh: reads the collector's log and prints, each # minute, the orchestrator's line and the book threads' lines summed into one, # and between them only the notable events. The Snapshot cause lines and the # connections coming up are counted, not shown. # # With -v final=1 it prints nothing live, only the last minute's sums: the # run's totals, since the book threads count "so far". A minute's sum is # printed once all its book threads' lines are in (-v book_threads=N, 16 by # default, src/core/mod.rs's BOOK_THREADS), or with the first other line 10 s # or more after the minute's first, or when the next minute begins. A book # thread that received nothing logs no line; the others' come within a few # seconds of the minute (1.6 s apart at most in a 10-minute Polymarket run, # 2026-11-12), and other lines often fall between them. # The book threads' counters shown, in this order; a name missing from an # exchange's line (Kalshi has no "rescued") is left out. BEGIN { if (book_threads == "") book_threads = 16 shown = "symbols|changes written|duplicates|rescued|resyncs|subscription gaps|losses" count = split(shown, names, "|") } function flush( line, i) { if (threads == 0) return line = minute " books (" threads " threads):" for (i = 1; i <= count; i++) if (names[i] in sum) line = line " " sum[names[i]] " " names[i] "," sub(/,$/, "", line) last = line event(line) threads = 0 split("", sum) } # Printed at once: the view is read while the run goes on, and mawk # buffers its output otherwise. function event(text) { if (final) return print text fflush() } { time = substr($1, 1, 8) text = substr($0, index($0, " ") + 1) # To the millisecond: $1 is HH:MM:SS.mmm. seconds = substr($1, 1, 2) * 3600 + substr($1, 4, 2) * 60 + substr($1, 7) } /: book thread [0-9]+: / { this = substr(time, 1, 5) if (threads > 0 && this != minute) flush() if (threads == 0) first_at = seconds minute = this threads++ body = text sub(/; this minute .*/, "", body) split("", seen) # Each "N name": a count and its name, up to the next comma, semicolon # or parenthesis. A name is summed once a line (its first occurrence). while (match(body, /[0-9]+ [a-z][a-z' ]*[a-z]/)) { pair = substr(body, RSTART, RLENGTH) body = substr(body, RSTART + RLENGTH) n = substr(pair, 1, index(pair, " ") - 1) name = substr(pair, index(pair, " ") + 1) sub(/^so far /, "", name) if (name in seen) continue seen[name] = 1 sum[name] += n } if (threads == book_threads) flush() next } # A minute with a book thread's line missing: printed once it is clearly over # (a day's seconds added back across midnight). threads > 0 && (seconds - first_at + 86400) % 86400 >= 10 { flush() } /: orchestrator: [0-9]+ markets on/ { line = text sub(/^[a-z-]+: orchestrator: /, "", line) event(time " " line "; " up + 0 " connected (" refused + 0 " refused, retried), " snapshots + 0 " Snapshots with a cause") up = 0 refused = 0 snapshots = 0 next } /: connection [0-9]+: connected, / { if (!opened) event(time " connections opening (counted in the minute line)") opened = 1 up++ next } # Kalshi refuses some of ~100 handshakes opened at once (429); each retries. /: connect failed: handshake refused: 429/ { refused++; next } /: Snapshot of / { snapshots++; next } / changes go unchecked$| tick size changed, resync$| unreadable / { next } # Everything else is an event worth seeing: a connection lost, recycled, # split, moved off when quiet, a refused handshake, a failed fetch, the end. { event(time " " text) } END { flush() if (final && last != "") print last }