From 480ee3ad2d2aba1c73f5de394b4af900a9b830e3 Mon Sep 17 00:00:00 2001 From: Pascal Perrenoud Date: Wed, 11 Nov 2026 17:42:27 +0000 Subject: [PATCH] feat(polymarket): subscription monitoring and connection recycling The exchange can fail without telling us, and first one wins hides a subscription gone quiet on one of a symbol's two connections. - A connection that receives nothing for 30 s, not even the keepalive's answer, reconnects. - Each book thread counts a symbol's changes per connection every minute: a subscription with none against three or more on the other is resubscribed, and moved to another connection if it stays quiet. - The orchestrator recycles one connection at a time, each about hourly, and never while another connection is down or just back. - A reconnect's opening book of a symbol unchanged for 10 s is judged like a resync's answer: one newer than ours that differs shows a symbol quiet on both connections, written as a Snapshot and logged as a loss. --- CLAUDE.md | 14 +- GLOSSARY.md | 21 + docs/notes.md | 21 + docs/project.md | 18 +- docs/roadmap.md | 9 +- roadmap/05. subscription monitoring.md | 301 ++++++++++++ src/cause.rs | 8 + src/polymarket/feed.rs | 632 +++++++++++++++++++++++-- src/polymarket/orchestrator.rs | 235 +++++++-- src/polymarket/socket.rs | 189 +++++++- 10 files changed, 1346 insertions(+), 102 deletions(-) create mode 100644 roadmap/05. subscription monitoring.md diff --git a/CLAUDE.md b/CLAUDE.md index 253c845..f018862 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,13 +18,14 @@ outside this repo. loss detection, message shapes). Read before touching exchange parsing. - `GLOSSARY.md`: the terms (symbol, connection, load, first one wins, duplicate, held, rescued, follow, resync, loss, subscription gap, split, - Snapshot, Delta). + quiet, resubscribe, silent, recycle, Snapshot, Delta). - `src/polymarket/orchestrator.rs`: what is tracked, which connections carry - each market, and their balancing; `src/polymarket/gamma.rs`: the fetchers + each market, their balancing and their recycling; `src/polymarket/gamma.rs`: the fetchers (discovery); `src/polymarket/socket.rs`: a connection (read, stamp, decode, - route); `src/polymarket/wire.rs`: the exchange's messages; - `src/polymarket/feed.rs`: a book thread and its loss rules (the subtle - part); `src/book.rs`: the book and fixed point; `src/reconcile.rs` + + route; reconnects when dropped, silent or recycled); + `src/polymarket/wire.rs`: the exchange's messages; + `src/polymarket/feed.rs`: a book thread, its loss rules (the subtle + part) and its watch for quiet subscriptions; `src/book.rs`: the book and fixed point; `src/reconcile.rs` + `src/bin/reconcile.rs`: the output check, reading the log's cause lines (`src/cause.rs`). `tests/fixtures/`: recordings of the live feed, on one and two connections, that the tests replay. @@ -73,6 +74,9 @@ and let the user settle it. Claude owns the data model. "capacity scan"` matches the waiting shell's own command line and never ends, and `pkill -f PATTERN` can kill the shell running it. Take the PID (`pgrep -f '^./target/release/...'`, anchored) and loop on `kill -0 $pid`. +- **No `/usr/bin/time` on the server**: for a run's peak memory and thread + count, read `VmHWM` and `Threads` from `/proc//status` while it runs + (sprint 05's run script sampled them every 30 s). - **The server is shared and gets overloaded** (load up to 92 on 12 cores, 2026-11-11): a starved collector falls behind and gets closed with 1013. Check `cat /proc/loadavg` before a live run whose numbers are compared, and diff --git a/GLOSSARY.md b/GLOSSARY.md index 2dfd1b2..bd40845 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -71,6 +71,27 @@ verified and until a fresh book says where it stands. **Split**: Moving half the markets off a connection the exchange keeps closing. +**Quiet**: +Said of a symbol's subscription on a connection that is up, while the +exchange has stopped sending that symbol's changes there. Nothing tells us: +it has to be looked for. +_Avoid_: Stale, dead (a connection can be fine while one of its +subscriptions is quiet) + +**Resubscribe**: +To unsubscribe a symbol on one connection and subscribe it again there, which +makes the exchange start its subscription afresh. A resync is a resubscribe +whose book we asked for. + +**Silent**: +Said of a connection that is open but delivers nothing at all, not even the +answer to its keepalive. + +**Recycle**: +To close a working connection on purpose and reconnect it with the same +symbols, to shed what has gone wrong on the exchange's side of it. +_Avoid_: Restart, rotate + **Receive time**: When a message came off the socket, by the local clock. _Avoid_: Arrival time, local timestamp diff --git a/docs/notes.md b/docs/notes.md index a597445..16cb78a 100644 --- a/docs/notes.md +++ b/docs/notes.md @@ -173,6 +173,27 @@ From a 60 s run of the M1 spike on one market, both tokens subscribed, and the (0 of 660,084 changes on each of the two): one older than the last applied is a late copy. +## Quiet subscriptions and reconnects (observed 2026-11-11) + +From a 25-minute live run on every live market, two connections each, one +connection recycled every 60 s (sprint 05). + +- **Quiet subscriptions are rare**: in 21 minutes, one market once had 3 or + more changes on one connection and none on the other; none had 10. Both + tokens of a market go one-sided together. +- **A reconnect's opening book can lag by minutes**: for symbols unchanged + for 10 s or more, 514 of 19,379 differed from ours, 512 of them equal to + one of our earlier states (1 to 4 levels behind, some over 10 minutes), + none with a newer timestamp. Books of symbols never changed all matched + (42,096). Under 10 s, some ran ahead of our stream. +- **With the monitoring on** (30 minutes, 107 recycles): 26 subscriptions + resubscribed for being quiet, none quiet again the next minute; of 335,488 + reconnect books of unchanged symbols, 7,240 fresh books in all were behind + ours; no connection silent for 30 s. +- **A reconnect takes ~1.2 s** from close to up (1 s of it our wait); the + opening books of ~4,500 tokens come 0.3 s later on most connections, 20 s + on one; 2 of 4,490 never came on another. + ## Gamma, the market listing (observed 2026-11-04) - **Size:** 238,308 markets `active` and not `closed`; 238,201 of them with diff --git a/docs/project.md b/docs/project.md index 0791569..6d544e5 100644 --- a/docs/project.md +++ b/docs/project.md @@ -135,14 +135,15 @@ and drop from the global workflow for a job-challenge repo. ## Pipeline The primary thing we debate sprint by sprint (this project's equivalent of the -schema debate). As settled in sprint 04: +schema debate). As settled in sprints 04 and 05: ``` rescan fetcher ──findings──▶ ┌──────────────┐ ◀──findings── recent fetcher - │ orchestrator │ ◀── 1013 closes ── connections - └──────┬───▲───┘ + │ orchestrator │ ◀── up, down, 1013 ─── connections + └──────┬───▲───┘ ──── recycle ────────▶ connections Add / Move / Remove (a market and │ │ load reports, each minute - its connections), to its thread │ │ (bytes/s per market) + its connections), to its thread │ │ (bytes/s per market); markets + │ │ quiet twice on a connection ┌──────▼───┴───┐ connection 0..N ──events───▶ │ book threads │ ──Snapshot | Delta──▶ writer (read, stamp, parse, route) │ 0..15 │ @@ -151,15 +152,16 @@ schema debate). As settled in sprint 04: ``` - The **orchestrator** owns what is tracked (each market's two connections, - the timers) and decides: placing, moving, splitting. It never blocks on - HTTP. + the timers) and decides: placing, moving, splitting, and recycling one + connection at a time, about hourly. It never blocks on HTTP. - The two **fetchers** only page Gamma on request and stream what they read. - A **connection** reads, stamps the receive time, parses, and routes each event to the book thread of its market. Every symbol is on two connections, - chosen independently. + chosen independently. One silent for 30 s reconnects. - A **book thread** owns its markets' books and applies first one wins: a change is a duplicate, the next one, or held until it fits (fold, check, - resync, Snapshots as before). + resync, Snapshots as before). It resubscribes a subscription gone quiet on + one connection, and judges a reconnect's books of unchanged symbols. - One **writer** owns the output. Settled: OS threads, shared-nothing, no mutexes diff --git a/docs/roadmap.md b/docs/roadmap.md index 0a74fea..65f32c7 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -55,12 +55,16 @@ abstraction is *extracted* once a second exchange exists, not designed up front. Result: losses a quarter lower in each pair of alternating runs; losses and gaps together lower in one pair, even in the other (sprint 04's Outcome). -5. **Extension 2 — subscription monitoring** — *not started* +5. **Extension 2 — subscription monitoring** — *done (2026-11-11)* Per-symbol activity monitoring (data points per second/minute); unsubscribe + resubscribe symbols that go too quiet (with two connections, first one wins hides a subscription gone quiet on one of them: it has to be looked for); periodic full-socket recycle (disconnect → reconnect → resubscribe) to shed cumulative exchange-side bugs. + Sprint: [../roadmap/05. subscription monitoring.md](../roadmap/05.%20subscription%20monitoring.md). *Ends:* monitoring + recycling running. + Result: a 30-minute run recycled 107 connections with losses no higher + than sprint 04's; quiet subscriptions are rare (26 resubscribed, none + quiet again) and no connection fell silent (sprint 05's Outcome). 6. **Extension 3 — generic core + Kalshi adapter** — *not started* Extract the exchange-agnostic core (the `NormalizedEvent` seam) from the @@ -115,6 +119,9 @@ abstraction is *extracted* once a second exchange exists, not designed up front. book, when the first one's chain breaks before any change verified. - An exact rescued count: a change remembered past the window of 128 until its copies are counted (sprint 04 reports an upper bound). +- Telling a change still in flight from a symbol quiet on both connections, + when a reconnect's book is newer than ours (`maybe`: sprint 05 logs both + as a loss; the Snapshot is right either way). - Fewer Snapshots for unchanged books (the brief asks for every symbol every minute: ~130 MB a minute at full coverage). - Re-reading a market's `orderMinSize` / `negRisk` if they can change. diff --git a/roadmap/05. subscription monitoring.md b/roadmap/05. subscription monitoring.md new file mode 100644 index 0000000..e5644bf --- /dev/null +++ b/roadmap/05. subscription monitoring.md @@ -0,0 +1,301 @@ +# 05. Subscription monitoring + +**Status: done (2026-11-11)** + +The exchange can fail without telling us: a connection stays open and sends +nothing, a symbol's subscription goes quiet on a connection that still works, +and a connection accumulates the exchange's own bugs. First one wins hides a +quiet subscription as long as the symbol's other connection delivers, so it +has to be looked for. This sprint finds a silent connection and reconnects +it, finds a subscription quiet on one connection and resubscribes it (then +moves it), and recycles every connection about once an hour; the recycle's +opening books also show the symbols quiet on both connections. This is +Extension 2 of the brief. Design: [../docs/project.md](../docs/project.md) +(pipeline), [ADR 0003](../docs/adr/0003-first-one-wins.md), +[../docs/notes.md](../docs/notes.md), [../GLOSSARY.md](../GLOSSARY.md) +(quiet, silent, recycle). + +## Measured before planning (2026-11-11) + +A throwaway build, 25 minutes live (~198k markets, ~180 connections, the +server loaded: 14 to 45 on 12 cores), with a forced recycle every 60 s: + +- **Quiet subscriptions are rare.** Of ~392,000 symbols judged each minute, + ~75,000 changed. In 21 minutes, one market once had 3 or more changes on + one connection and none on the other (not the next minute); none had 10. + One change against none came 0 to 16 times a minute: a copy landing in + the next minute. Both tokens of a market go one-sided together. +- **A reconnect's opening book can lag by minutes.** For symbols unchanged + for 10 s or more, 514 of 19,379 reconnect books differed from ours; 512 of + them equal one of our earlier states (1 to 4 levels, some over 10 minutes + old), never with a newer timestamp. 2 (one market) were neither. All + 42,096 books of symbols never changed since their first book matched. + Under 10 s, some books ran ahead of our stream (2 between 1 and 10 s). +- **A recycle is cheap**: 1.2 s from close to up; all opening books 0.3 s + later on most connections, 20 s on one, and 2 of 4,490 never on another. + No 1013 close in the run; chains broken and losses per minute the same + during recycles as before. + +## A silent connection + +A connection that has received nothing, data or `PONG`, for 30 s (three +keepalives) closes itself and reconnects like a dropped one, telling the book +threads it went down. Today nothing notices: the read loop +(`socket.rs:222`) times out every 200 ms and carries on, and `PONG` decodes +to no event (`wire.rs:37`). + +## A quiet subscription + +Each book thread counts, per symbol and per connection, the changes received +each wall-clock minute: every one, whether it is then applied, a duplicate, +held or stale. At the minute, a verified symbol on two connections, both up +the whole minute and subscribed on both since before it began, whose one +connection delivered at least 3 changes and the other none, is +**quiet** on the other: + +- the first time, that connection gets a `Resubscribe` (unsubscribe, + subscribe: the command a resync already used, renamed from `Resync` since + here no book is asked for). The symbol stays verified on the other + connection, so the book that comes back is ignored like any unasked one: + no Snapshot, no subscription gap; +- quiet again the next minute on the same connection, the book thread reports + the market to the orchestrator, which moves that subscription with the + existing `Move`, outside the cap of 50 moves a minute, and logs it; a + market moved in the last 10 minutes stays, as for rebalancing. + +A symbol with no change on either connection is not judged: the market may +just be still. That case is the recycle's. + +## Recycling + +The orchestrator recycles, once the startup deal is complete: every 3600/N +seconds (N connections), the connection whose last connect is oldest closes +and reconnects through the path a dropped connection takes (1 s, then its +symbols subscribed in operations of 500). The next recycle waits until that +connection has reported up again, none starts while any connection is down, +nor for one interval after any connection came up (up means its subscribes +are sent; its opening books take 0.3 to 20 s), so a recycle never takes down +a market's two connections together. A connection that dropped on its own +counts as recycled from its reconnect. + +For this the orchestrator keeps each connection's command sender (today +`Threads::open` only hands it to the book threads), a `Recycle` command +closes the socket, and the channel that carries 1013 closes today +(`Sender`) carries a connection's notices: up, down, closed with +1013, closed for silence. + +## Reconnect books + +Sprint 04 decided that only asked-for books count, because a moving symbol's +books arrive ahead of or behind our stream. A reconnect's opening book for a +symbol unchanged for at least 10 s is an exception: nothing moved, so it is +judged like a resync's answer. Equal to ours or to one of our earlier states +(the server's opening books lag, by minutes: above), ours stands and it +counts as behind; so does any book whose timestamp is no newer than our last +change, since our chain holds up to that change and we only remember 128 +earlier states. A book that is unreadable or fails its own hash is ignored. +Otherwise both subscriptions were quiet: it replaces ours, +is written as a Snapshot with a cause line (a loss, its detail saying a +reconnect found it), and the symbol's other connection is resubscribed. +This holds for any reconnect, recycled or dropped. + +## Data model + +- **Book thread**, per symbol: the changes received this minute on each of + its connections, and since when it is subscribed on each; whether it was + quiet last minute, and on which connection. Per connection, when it last + came up. +- **Book thread → orchestrator**: the load report, or a market quiet twice on + a connection. +- **Connection → orchestrator**: up, down, a 1013 close, a close for silence. +- **Orchestrator**, per connection: when it last connected; the recycle under + way. +- **Cause lines**: a reconnect book that differs is a `loss`, with a detail + naming the reconnect, so `reconcile` pairs it and the log tells them apart. + +## The minute lines + +Each book thread's adds: the changes received this minute (all connections +together: one figure per connection would put ~180 in each line), quiet +subscriptions resubscribed and reported, reconnect books judged and found +differing (those behind count with the resyncs' "full books behind ours"). The orchestrator's adds: connections recycled, quiet moves, and +silent connections closed. Totals only: a per-symbol table would be ~400k +lines a minute. + +## Decisions + +- **Changes received per symbol and connection, per minute** are the + activity measure. *Rejected:* bytes (the load is per market, averaged over + its connections); changes applied (the second connection applies almost + none, by design). +- **Quiet means one connection delivered and the other nothing**; a dip + against the symbol's own past rate is not used. Prediction markets are + bursty, so a dip is a weak signal, while the other connection, or a fresh + book, is a real test. *Rejected:* the brief's literal per-symbol rate + threshold. +- **Resubscribe once, then move.** *Rejected:* resubscribing every minute + forever (never leaves a connection broken for that symbol). +- **The book thread reports a market quiet twice; the orchestrator moves + it.** Only the orchestrator knows which connections have room. + *Rejected:* the book thread choosing a target. +- **The orchestrator recycles one connection at a time, oldest first, about + hourly, starting after the startup deal.** *Rejected:* each connection + recycling itself at an age of an hour plus jitter (~600 markets an hour + would have both connections down at once); waiting an hour before the + first recycle (a 30-minute run would show none). +- **A recycle is a plain close and reconnect.** M4's second connection + covers the seconds it takes. *Rejected:* opening the replacement before + closing (a third connection per symbol and a handover). +- **No recycle while any connection is down.** Covers every market with no + per-market lookup. *Rejected:* checking each market's other connection. +- **A reconnect's opening book is compared for a long-unchanged symbol, on + every reconnect** (amends sprint 04's "only asked-for books count", whose + reason, books ahead of or behind a moving stream, doesn't apply to a + symbol that hasn't moved). *Rejected:* comparing only after a recycle (a + dropped connection's reconnect is the same check). +- **Quiet at 3 changes against none.** Crossed once in 21 live minutes; a + needless resubscribe costs no Snapshot or gap. *Rejected:* 1 (up to 16 + needless resubscribes a minute, from copies landing in the next minute); + 10 (a slowly trading market's quiet subscription goes unseen). +- **A reconnect book is judged by the resync's rule, from 10 s unchanged** + (supersedes "it must equal ours", because 2.7% of them equal an earlier + state of ours: the server's books lag by minutes). *Rejected:* exact + equality (~500 false losses per 15 recycles); under 10 s (books run ahead + of a moving stream). +- **The orchestrator hears down and silent too** (supersedes "a 1013 close, + or up", because "no recycle while any connection is down" needs down, and + the minute line counts the silent closes). +- **A reconnect book no newer than our last change is behind** (added at the + code review: a book lagging further back than the 128 changes we remember + was taken for a loss and replaced a correct book). *Rejected:* remembering + more states. +- **No recycle for one interval after any connection came up** (added at the + code review: an overdue recycle fired the moment a connection reported up, + before its opening books). +- **A differing reconnect book is a loss**, its cause line saying how it was + found. *Rejected:* a new cause (the changes are ones no connection + delivered: that is a loss). +- **A silent connection is closed after 30 s with nothing received**, one + constant. + +## Tests + +No network, except the local websocket server of sprint 03's reconnect test. + +- **Quiet**: the two-connection recording, one symbol's changes removed from + one connection for a minute: that connection is resubscribed and no + Snapshot is written; removed again the next minute: the market is reported. + A subscription younger than the minute is not judged; a symbol with no + change on either connection is not quiet. +- **Reconnect books**: after an up marker, an opening book equal to ours for + a long-unchanged symbol changes nothing; one that differs is written as a + Snapshot with its cause line and resubscribes the other connection; a book + for a recently changed symbol is ignored as before. +- **Recycle** (on the `Tracker`): the oldest connection first, the next only + after the previous is up, none while a connection is down, paced at 3600/N. +- **A quiet market reported** is moved, outside the cap. +- **Silent**: on the local websocket server, a connection that hears nothing + for the timeout (shortened in the test) reconnects with its symbols. +- **Reconcile**: a reconnect book's cause line pairs with its Snapshot. + +## Not in this sprint + +- A dip against a symbol's own past rate (see Decisions). +- Opening a replacement connection before closing the recycled one. +- Per-second rates: a minute decides a resubscribe. +- Acting on delay: M7. +- Injecting a fault into a live run: the tests carry the rules. + +## Done when + +- `scripts/ci.sh` is green. +- A 30-minute live run with recycling on (about 90 connections recycled at + ~178 connections): + - every `reconcile` mismatch is explained by the log; + - the minute lines show recycles, quiet subscriptions and reconnect books, + and their counts are reported (none quiet in the run: the tests carry it, + and the Outcome says so); + - losses and subscription gaps per million changes are no higher than + sprint 04's two-subscription rounds (32 to 50 losses, 150 to 160 gaps); + - peak memory and thread count are reported. +- Free disk and the server's load are checked before the run, the output + deleted after. +- `GLOSSARY.md` (quiet, silent, recycle), `docs/project.md`'s pipeline, + `docs/notes.md` (the measures above) and the roadmap match. + +## Outcome + +The collector now looks for the exchange failing silently. A connection that +hears nothing for 30 s reconnects. Each book thread counts every symbol's +changes per connection and, at the minute, resubscribes a subscription with +none against three or more on the other; quiet again, its market is reported +and the orchestrator moves it. The orchestrator recycles one connection at a +time, the one up the longest, each about hourly, and a reconnect's opening +book of a symbol unchanged for 10 s is judged like a resync's answer. + +**Two live runs**, each reconciled with every mismatch explained by its log +line, no orphan Delta: + +| Run | Recycles | Losses / M changes | Gaps / M | Quiet resubscribed | Reconnect books judged | Peak memory, threads | +|---|---|---|---|---|---|---| +| 30 min, before the review's fixes | 107 | 26.5 | 56 | 26 (none quiet again) | 335,488 (5 losses) | 2.46 GB, 204 | +| 10 min, after them | 19 | 28.4 | 190 | 0 | 69,340 (0 losses) | 1.94 GB, 202 | + +Sprint 04's two-subscription rounds (10 minutes each) had 32 to 50 losses +and 150 to 160 gaps per million. Losses are no higher with recycling. The +gaps are the startup subscribes' (3,111 of them by minute 10, 3,275 by +minute 30), so their rate says how long the run was, not what recycling +costs: about 160 more in 20 minutes of recycling. No connection fell silent +and no market was reported quiet twice in either run: the tests carry both, +and the silent close's count. Memory grows with the run's length (the last +128 changes of each symbol fill up); at 10 minutes it is 0.16 GB above +sprint 04's, not looked into further. + +**The 5 reconnect losses of the first run** are doubtful: 4 came at +17:00:04 with book times 0.3 to 2 s old. The review then found that a book +lagging further back than the changes we remember was taken for a loss, and +the rule "no newer than our last change is behind" was added; the second run +judged 69,340 books and found none. + +**Fixed at the close**, from the code review, each with a test that fails +without it: a reconnect book older than our last change, unreadable, or +failing its own hash now leaves a verified symbol alone (the first replaced +a correct book, the last made the symbol unverifiable for good); no recycle +for one interval after a connection came up (an overdue one fired before +the returning connection's books); a market moved in the last 10 minutes is +not moved again for being quiet (it ended the move under way). The tests +were strengthened after the reviewer removed rules and they stayed green: +twelve rules were then removed one by one, each caught by a named test. + +**Diverged from the plan**, each in Decisions or its section: the +orchestrator hears down and silent as well as up; `Resync` the command +became `Resubscribe`; the minute line gives one total of changes received; +the three rules above. + +**Left out**, and where it went: telling a change still in flight from a +symbol quiet on both connections when a reconnect book is newer than ours +(Possible features); the dip against a symbol's own past rate, the +replacement connection opened first, per-second rates (not planned, see +Not in this sprint); acting on delay (M7). Not covered by a test: the +orchestrator's loop (sending the Recycle, counting the notices), a quiet +judging skipped for a symbol being resynced, and `reconcile` pairing a +reconnect's loss end to end (its cause line's parsing is tested, and the +first live run paired five). + +### Retrospective + +- **Went well:** the measurement before planning overturned a rule on paper + (reconnect books lag by minutes) instead of in a live run. The reviewer's + mutation check, removing a rule and running the tests, showed which tests + proved nothing; redone here after the fixes, it took minutes. +- **Went wrong:** the first tests were written to pass, not to fail without + their rule: six rules were uncovered. The reconnect rule leaned on a + memory of 128 changes without asking what happens past it. The first live + run was launched with a tool the server doesn't have (`/usr/bin/time`), + and I announced its end time wrongly. +- **Tested enough?** The rules are, by mutation. The orchestrator's loop is + still untested, as in sprint 04, and neither silent connections nor quiet + moves happened live. +- **Next time:** for each rule, write the test, then remove the rule and see + it fail, before the review does. Ask of every "we remember N" what happens + at N + 1. diff --git a/src/cause.rs b/src/cause.rs index 5aea2f5..880f17c 100644 --- a/src/cause.rs +++ b/src/cause.rs @@ -71,6 +71,14 @@ mod tests { let logged = format!("12:00:00.001 polymarket: {text}"); assert_eq!(parse(&logged), Some(("123".to_string(), at, cause))); } + // A reconnect's loss says how it was found, after its cause. + let text = line( + "123", + at, + Cause::Loss, + ", found by a reconnect (its time 12:00:00.000)", + ); + assert_eq!(parse(&text), Some(("123".to_string(), at, Cause::Loss))); assert_eq!( parse("12:00:00.001 polymarket: connection 3: connected"), None diff --git a/src/polymarket/feed.rs b/src/polymarket/feed.rs index f7411e5..fa47f28 100644 --- a/src/polymarket/feed.rs +++ b/src/polymarket/feed.rs @@ -2,8 +2,11 @@ //! per symbol, first one wins: a change is taken from whichever connection //! delivers it first, Polymarket's book hash saying where it belongs //! (ADR 0003). It resyncs when a change fits on no connection in time, and -//! emits Deltas and Snapshots. The rules are in -//! `roadmap/04. redundant connections.md`, the protocol facts and the hash +//! emits Deltas and Snapshots. It also looks for what first one wins hides: +//! a subscription gone quiet on one connection, each minute, and symbols +//! quiet on both, in a reconnect's opening books. The rules are in +//! `roadmap/04. redundant connections.md` and +//! `roadmap/05. subscription monitoring.md`, the protocol facts and the hash //! formula in `docs/notes.md`, the terms in `GLOSSARY.md`. use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; @@ -47,15 +50,34 @@ const MAX_HELD: usize = 512; const TAILS_TRIED: usize = 4; /// When a change's copies are counted: no second copy came later than 0.7 s. const COPIES_BY: TimeDelta = TimeDelta::seconds(2); +/// A connection that delivered none of a symbol's changes in a minute while +/// another delivered at least this many is quiet for it (sprint 05: crossed +/// once in 21 live minutes; at 1, copies landing in the next minute would +/// count). +const QUIET_AGAINST: u32 = 3; +/// A reconnect's opening book is judged only for a symbol unchanged this +/// long: a moving symbol's books run ahead of or behind our stream. +const STILL_FOR: TimeDelta = TimeDelta::seconds(10); /// What the book thread does with an event or a command: write output, ask a -/// connection (by number) for something, or report its markets' loads to the -/// orchestrator. +/// connection (by number) for something, or report to the orchestrator. #[derive(Debug)] pub enum Effect { Write(Output), Send(usize, Command), - Report(Loads), + Report(Report), +} + +/// What a book thread tells the orchestrator. +#[derive(Debug, PartialEq, Eq)] +pub enum Report { + Loads(Loads), + /// The market stayed quiet on the connection after being resubscribed + /// there: its subscription should move. + Quiet { + market: String, + connection: usize, + }, } /// Each market's load (`conditionId` → bytes per second on one connection), @@ -130,6 +152,33 @@ impl RecentChange { } } +/// A symbol's subscription on one of its connections, as watched for going +/// quiet. +#[derive(Debug, Clone, Copy)] +struct Watch { + connection: usize, + /// Its changes received from this connection this minute: every one, + /// whatever first one wins then made of it. + received: u32, + /// When it was last subscribed there: a minute it didn't cover whole is + /// not judged. + since: DateTime, + /// When the connection last delivered a complete book of it: its first + /// since the connection came up is the reconnect's. + book_at: DateTime, +} + +impl Watch { + fn new(connection: usize, since: DateTime) -> Self { + Self { + connection, + received: 0, + since, + book_at: DateTime::UNIX_EPOCH, + } + } +} + /// A change as applied, with the exchange's hash of the book after it. struct RecentChange { side: Side, @@ -172,6 +221,10 @@ struct Symbol { had_book: bool, /// The connections it is subscribed on (three while a move overlaps). connections: Vec, + /// One per connection in `connections`. + watches: Vec, + /// The connection it was quiet on last minute, and resubscribed. + quiet_on: Option, /// The receive time of its last Delta: a held change is stamped no earlier, /// so replaying the output by time keeps its order. last_written: DateTime, @@ -184,7 +237,7 @@ struct Symbol { } impl Symbol { - fn new(market: &Market, connections: &[usize]) -> Self { + fn new(market: &Market, connections: &[usize], now: DateTime) -> Self { Self { state: State::AwaitingBook, book: Book::default(), @@ -196,6 +249,11 @@ impl Symbol { verified_since_book: false, had_book: false, connections: connections.to_vec(), + watches: connections + .iter() + .map(|&connection| Watch::new(connection, now)) + .collect(), + quiet_on: None, last_written: DateTime::UNIX_EPOCH, market: market.condition_id.clone(), tick_size: String::new(), @@ -216,7 +274,39 @@ impl Symbol { // A switch already asked stays one until its book comes. let switch = switch || self.asked.is_some_and(|(_, switch)| switch); self.asked = Some((connection, switch)); - effects.push(Effect::Send(connection, Command::Resync(token.to_string()))); + effects.push(Effect::Send( + connection, + Command::Resubscribe(token.to_string()), + )); + } + + fn watch_mut(&mut self, connection: usize) -> Option<&mut Watch> { + self.watches + .iter_mut() + .find(|watch| watch.connection == connection) + } + + /// The connection quiet for it over the minute begun at `started`: one + /// that delivered none of its changes while another delivered at least + /// [`QUIET_AGAINST`], both up and subscribed for the whole minute. + fn quiet(&self, started: DateTime, up_since: &[Option>]) -> Option { + let judged: Vec<&Watch> = self + .watches + .iter() + .filter(|watch| { + watch.since <= started + && up_since + .get(watch.connection) + .copied() + .flatten() + .is_some_and(|up| up <= started) + }) + .collect(); + let busiest = judged.iter().map(|watch| watch.received).max()?; + (busiest >= QUIET_AGAINST) + .then(|| judged.iter().find(|watch| watch.received == 0)) + .flatten() + .map(|watch| watch.connection) } /// One of its connections that is up, other than `except`. @@ -404,6 +494,15 @@ struct Counts { subscription_gaps: u64, losses: u64, switches: u64, + /// Subscriptions found quiet, each resubscribed. + quiet: u64, + /// Markets quiet again after their resubscribe: reported for a move. + quiet_reported: u64, + /// A reconnect's opening books judged, those of still symbols. + reconnect_books: u64, + /// Of them, books that differed from every state of ours: both + /// subscriptions were quiet. + reconnect_losses: u64, } pub struct Feed { @@ -414,6 +513,13 @@ pub struct Feed { markets: HashMap, /// By connection number: whether it is up. up: Vec, + /// By connection number: since when it is up, if it is. + up_since: Vec>>, + /// When the minute being counted began: its quiet subscriptions are + /// judged at its end. None before the first. + minute_started: Option>, + /// Changes received this minute, from all connections. + received_this_minute: u64, /// The symbols with held changes: their ceilings are checked. waiting: BTreeSet, /// The receive time of the last event applied: the time a Snapshot's book @@ -436,6 +542,9 @@ impl Feed { symbols: BTreeMap::new(), markets: HashMap::new(), up: Vec::new(), + up_since: Vec::new(), + minute_started: None, + received_this_minute: 0, waiting: BTreeSet::new(), last_received_at: None, next_minute_at: None, @@ -456,7 +565,7 @@ impl Feed { ) -> Vec { for token in &market.tokens { self.symbols - .insert(token.clone(), Symbol::new(market, connections)); + .insert(token.clone(), Symbol::new(market, connections, now)); } let tracked = Tracked { tokens: market.tokens.clone(), @@ -476,7 +585,13 @@ impl Feed { /// Moves a market's subscription from one connection to another: it /// subscribes on `to` now, and leaves `from` once `to` delivers a book of /// the market. The book stays as it is: no Snapshot, no subscription gap. - pub fn move_market(&mut self, market: &Market, from: usize, to: usize) -> Vec { + pub fn move_market( + &mut self, + market: &Market, + from: usize, + to: usize, + now: DateTime, + ) -> Vec { let Some(tracked) = self.markets.get_mut(&market.condition_id) else { return Vec::new(); }; @@ -493,6 +608,7 @@ impl Feed { && !symbol.connections.contains(&to) { symbol.connections.push(to); + symbol.watches.push(Watch::new(to, now)); } } effects.push(Effect::Send(to, Command::Subscribe(market.tokens.clone()))); @@ -513,6 +629,9 @@ impl Feed { for token in &tokens { if let Some(symbol) = self.symbols.get_mut(token) { symbol.connections.retain(|&on| on != connection); + symbol + .watches + .retain(|watch| watch.connection != connection); symbol.held.remove(&connection); Self::lose( token, @@ -558,16 +677,25 @@ impl Feed { match received { Received::Event(arrival) => self.on_event(arrival), Received::Up(connection) => { - self.set_up(connection, true); + // Its first events come after this marker; any other + // connection's before it were received earlier. + let at = self.last_received_at.unwrap_or(DateTime::UNIX_EPOCH); + self.set_up(connection, true, at); Vec::new() } Received::Down(connection) => self.on_down(connection), } } - fn set_up(&mut self, connection: usize, up: bool) { + fn set_up(&mut self, connection: usize, up: bool, at: DateTime) { if self.up.len() <= connection { self.up.resize(connection + 1, false); + self.up_since.resize(connection + 1, None); + } + if up && !self.up[connection] { + self.up_since[connection] = Some(at); + } else if !up { + self.up_since[connection] = None; } self.up[connection] = up; } @@ -576,7 +704,7 @@ impl Feed { let (received_at, connection) = (arrival.received_at, arrival.connection); // A connection that delivers is up, its marker possibly still behind. if !is_up(&self.up, connection) { - self.set_up(connection, true); + self.set_up(connection, true, received_at); } // Minute Snapshots due before this event hold the books as they were. let mut effects = self.tick(received_at); @@ -631,7 +759,7 @@ impl Feed { } fn on_down(&mut self, connection: usize) -> Vec { - self.set_up(connection, false); + self.set_up(connection, false, DateTime::UNIX_EPOCH); let mut effects = Vec::new(); for (token, symbol) in &mut self.symbols { if !symbol.connections.contains(&connection) { @@ -692,14 +820,16 @@ impl Feed { } /// Once `now` reaches the next wall-clock minute: reports the loads, - /// emits the minute Snapshots, and asks again for the fresh books still - /// not come. + /// emits the minute Snapshots, asks again for the fresh books still not + /// come, and resubscribes (or reports) the subscriptions quiet over the + /// minute that ends. pub fn tick(&mut self, now: DateTime) -> Vec { let next = *self.next_minute_at.get_or_insert_with(|| next_minute(now)); if now < next { return Vec::new(); } self.next_minute_at = Some(next_minute(now)); + let started = self.minute_started.replace(now); let mut loads = Loads::new(); for (market, tracked) in &mut self.markets { let connections = tracked @@ -711,13 +841,48 @@ impl Feed { loads.insert(market.clone(), load); } } - let mut effects = vec![Effect::Report(loads)]; + let mut effects = vec![Effect::Report(Report::Loads(loads))]; let Some(received_at) = self.last_received_at else { return effects; }; self.log_minute(); + let mut reported = BTreeSet::new(); for (token, symbol) in &mut self.symbols { symbol.count_old_copies(now, &mut self.counts); + let quiet = started + .filter(|_| symbol.state == State::Verified && symbol.asked.is_none()) + .and_then(|started| symbol.quiet(started, &self.up_since)); + for watch in &mut symbol.watches { + watch.received = 0; + } + match quiet { + // Resubscribed already, and still quiet: it should move. + Some(connection) if symbol.quiet_on == Some(connection) => { + symbol.quiet_on = None; + if reported.insert((symbol.market.clone(), connection)) { + self.counts.quiet_reported += 1; + effects.push(Effect::Report(Report::Quiet { + market: symbol.market.clone(), + connection, + })); + } + } + // The other connection carries it meanwhile: the book that + // comes back is ignored, no Snapshot, no subscription gap. + Some(connection) => { + log!("polymarket: {token}: quiet on connection {connection}, resubscribed"); + self.counts.quiet += 1; + symbol.quiet_on = Some(connection); + if let Some(watch) = symbol.watch_mut(connection) { + watch.since = now; + } + effects.push(Effect::Send( + connection, + Command::Resubscribe(token.clone()), + )); + } + None => symbol.quiet_on = None, + } if symbol.state != State::AwaitingBook { effects.push(Effect::Write(symbol.book.snapshot( token, @@ -759,8 +924,10 @@ impl Feed { log!( "polymarket: book thread {}: {} symbols; so far {} changes written ({} verified), {} duplicates, \ {} rescued, {} held dropped, {} chains broken, {} resyncs, {} full books behind ours, \ - {} subscription gaps, {} losses, {} switches; second copies {:.1} ms late at the median, \ - {:.1} ms at p99; at most {} events waited this minute", + {} subscription gaps, {} losses, {} switches, {} quiet subscriptions resubscribed, \ + {} markets reported quiet again, {} reconnect books judged ({} losses); second copies \ + {:.1} ms late at the median, {:.1} ms at p99; this minute {} changes received, at most \ + {} events waited", self.number, self.symbols.len(), counts.written, @@ -774,12 +941,18 @@ impl Feed { counts.subscription_gaps, counts.losses, counts.switches, + counts.quiet, + counts.quiet_reported, + counts.reconnect_books, + counts.reconnect_losses, lag(0.5), lag(0.99), + self.received_this_minute, self.peak_waiting ); self.lags.clear(); self.peak_waiting = 0; + self.received_this_minute = 0; } /// Held changes past their ceiling: dropped if the book moved on without @@ -862,11 +1035,14 @@ impl Feed { counts.resyncs += 1; } - /// A full book counts only when asked for: a symbol's first (from any of - /// its connections), or the answer to a resync. It replaces ours, brought - /// forward first if it is behind our stream, and is written as a Snapshot - /// when it is the symbol's first, when it differs from ours, or when the - /// connection followed changed. Then it is checked against its own hash. + /// A full book counts when asked for (a symbol's first, from any of its + /// connections, or the answer to a resync), or when it is a reconnect's + /// opening book of a symbol unchanged for [`STILL_FOR`]. It replaces + /// ours, brought forward first if it is behind our stream, and is written + /// as a Snapshot when it is the symbol's first, when it differs from + /// ours, or when the connection followed changed. Then it is checked + /// against its own hash. A reconnect's book that is ours, or behind + /// ours, changes nothing at all. fn on_book( &mut self, received_at: DateTime, @@ -895,7 +1071,24 @@ impl Feed { let complete = message.last_trade_price.is_some(); let asked = symbol.asked.filter(|&(asked, _)| asked == connection); let first = symbol.state == State::AwaitingBook && symbol.connections.contains(&connection); - if !complete || (asked.is_none() && !first) { + // The connection's first complete book since it came up, for a + // symbol subscribed there before: the reconnect's opening book. + let up_since = self.up_since.get(connection).copied().flatten(); + let reconnect = complete + && symbol.watch_mut(connection).is_some_and(|watch| { + // A book at the very instant it came up came before it. + let opening = up_since.is_some_and(|up| watch.since < up && watch.book_at <= up); + watch.book_at = received_at; + opening + }); + // Judged only for a still, verified symbol, nothing held or asked. + let reconnect = reconnect + && asked.is_none() + && symbol.asked.is_none() + && symbol.state == State::Verified + && symbol.held.is_empty() + && received_at - symbol.advanced_at >= STILL_FOR; + if !complete || (asked.is_none() && !first && !reconnect) { return; } let (Some(book), Some(exchange_timestamp), Some(book_hash)) = ( @@ -903,6 +1096,10 @@ impl Feed { parse_millis(&message.timestamp), parse_hash(&message.hash), ) else { + // Nobody asked for a reconnect's book: unreadable, it is ignored. + if reconnect { + return; + } // A protocol error: ask again. A book that stays unreadable is asked // for once per answer, and its symbol stays unchecked meanwhile. log!("polymarket: {token}: unreadable book, asking for a fresh one"); @@ -910,6 +1107,28 @@ impl Feed { self.counts.resyncs += 1; return; }; + if reconnect { + self.counts.reconnect_books += 1; + // The exchange's opening books lag by minutes (docs/notes.md): one + // that is ours, or a state we passed, leaves ours, and the hash + // inputs, as they are. + let brought = symbol + .recent_changes + .iter() + .rposition(|change| change.hash == book_hash) + .map(|position| symbol.brought_forward(&book, position + 1)); + if book == symbol.book || brought.as_ref() == Some(&symbol.book) { + return; + } + // Our chain holds up to our last change, so a book no newer than + // it holds nothing we missed, however far back it lags: further + // than the changes we remember, too. + if exchange_timestamp <= symbol.exchange_timestamp || symbol.was_ours(&book) { + self.counts.books_behind += 1; + return; + } + } + let inputs = reconnect.then(|| (symbol.tick_size.clone(), symbol.last_trade_price.clone())); if let Some(tick_size) = message.tick_size { symbol.tick_size = tick_size; } @@ -918,6 +1137,25 @@ impl Feed { } // Checked as received, before being brought forward below. let verified = hash(&token, symbol, &book, &message.timestamp) == book_hash; + if let Some(inputs) = inputs { + // A book failing its own hash proves nothing against a verified + // one: ours stands, with its hash inputs. + if !verified { + (symbol.tick_size, symbol.last_trade_price) = inputs; + return; + } + // Newer than ours and no state of ours: both subscriptions were + // quiet. The other one is renewed too; its book is not waited for. + self.counts.reconnect_losses += 1; + for other in symbol.connections.clone() { + if other != connection { + if let Some(watch) = symbol.watch_mut(other) { + watch.since = received_at; + } + effects.push(Effect::Send(other, Command::Resubscribe(token.clone()))); + } + } + } let switch = asked.is_some_and(|(_, switch)| switch); let compared = !first || symbol.had_book; @@ -947,7 +1185,7 @@ impl Feed { let differs = compared && book != symbol.book; let cause = if switch { Some(Cause::Switch) - } else if differs && !symbol.verified_since_book && !symbol.had_book { + } else if differs && !reconnect && !symbol.verified_since_book && !symbol.had_book { // A subscribe hides a few ms of changes (docs/notes.md): until a // change is verified, a differing book is that gap. Some(Cause::SubscriptionGap) @@ -978,8 +1216,13 @@ impl Feed { effects.push(Effect::Write(snapshot)); symbol.last_written = symbol.last_written.max(received_at); if let Some(cause) = cause { + let found = if reconnect { + ", found by a reconnect" + } else { + "" + }; let detail = format!( - " (its time {}, ours {})", + "{found} (its time {}, ours {})", book_time.format("%H:%M:%S%.3f"), exchange_timestamp.format("%H:%M:%S%.3f") ); @@ -1015,6 +1258,10 @@ impl Feed { let Some(symbol) = self.symbols.get_mut(&token) else { continue; }; + if let Some(watch) = symbol.watch_mut(connection) { + watch.received += 1; + } + self.received_this_minute += 1; let side = match change.side.as_str() { "BUY" => Some(Side::Buy), "SELL" => Some(Side::Sell), @@ -1146,7 +1393,7 @@ pub fn run( events: Receiver, commands: Receiver, outputs: Sender, - reports: Sender, + reports: Sender, ) { serve(feed, Vec::new(), events, commands, outputs, reports) } @@ -1159,7 +1406,7 @@ fn serve( events: Receiver, commands: Receiver, outputs: Sender, - reports: Sender, + reports: Sender, ) { let no_commands = never(); let mut orchestrator_gone = false; @@ -1190,7 +1437,9 @@ fn serve( Vec::new() } Ok(BookCommand::Add(market, on)) => feed.add_market(&market, &on, Utc::now()), - Ok(BookCommand::Move { market, from, to }) => feed.move_market(&market, from, to), + Ok(BookCommand::Move { market, from, to }) => { + feed.move_market(&market, from, to, Utc::now()) + } Ok(BookCommand::Remove(market)) => feed.remove_market(&market), // It is stopping: the connections' end still decides ours. Err(_) => { @@ -1210,7 +1459,7 @@ fn serve( // The connection may be gone already, while stopping, and the // orchestrator too. Effect::Send(connection, command) => _ = connections[connection].send(command), - Effect::Report(loads) => _ = reports.send(loads), + Effect::Report(report) => _ = reports.send(report), } } } @@ -1417,7 +1666,7 @@ mod tests { effects .iter() .filter_map(|effect| match effect { - Effect::Send(connection, Command::Resync(token)) => { + Effect::Send(connection, Command::Resubscribe(token)) => { Some((*connection, token.as_str())) } _ => None, @@ -1549,9 +1798,13 @@ mod tests { /// A complete book frame for `book`, as the exchange would send it in /// answer to a resync, 5 ms after the symbol's last change. fn fresh_book(token: &str, symbol: &Symbol, book: &Book) -> String { - let timestamp = (symbol.exchange_timestamp + TimeDelta::milliseconds(5)) - .timestamp_millis() - .to_string(); + let at = symbol.exchange_timestamp + TimeDelta::milliseconds(5); + book_frame(token, symbol, book, at) + } + + /// A complete book frame for `book`, last changed at `at` (exchange time). + fn book_frame(token: &str, symbol: &Symbol, book: &Book, at: DateTime) -> String { + let timestamp = at.timestamp_millis().to_string(); let levels = |levels: &BTreeMap| { levels .iter() @@ -2379,6 +2632,302 @@ mod tests { assert_eq!(feed.counts.losses, losses + 1); } + /// A frame setting `token`'s bid at `price` to `quantity`, `millis` its + /// exchange time, from the book as it is now. + fn change(feed: &Feed, token: &str, millis: i64, price: i64, quantity: i64) -> String { + let symbol = &feed.symbols[token]; + let changes = [(Side::Buy, price, quantity)]; + changes_frame(token, symbol, &symbol.book, &changes, millis).0 + } + + fn resyncs_of<'a>(effects: &'a [Effect], token: &str) -> Vec<(usize, &'a str)> { + resyncs(effects) + .into_iter() + .filter(|&(_, theirs)| theirs == token) + .collect() + } + + /// From the next wall-clock minute after `at`, which the feed has begun: + /// the following minute is counted whole. + fn begin_minute(feed: &mut Feed, at: DateTime) -> DateTime { + let minute = next_minute(at); + feed.idle(minute); + minute + } + + /// `count` changes of `token`, from 10 s after `start`, on the + /// connections `on` only. + fn changes( + feed: &mut Feed, + token: &str, + on: &[usize], + start: DateTime, + price: i64, + count: i64, + ) { + let millis = feed.symbols[token].exchange_timestamp.timestamp_millis() + 1_000; + for step in 0..count { + let at = start + TimeDelta::seconds(10 + step); + let frame = change(feed, token, millis + step, price, (step + 1) * 1_000_000); + for &connection in on { + feed.on_frame_from(connection, at, &frame); + } + } + } + + fn quiet_reports(effects: &[Effect]) -> Vec<(&str, usize)> { + effects + .iter() + .filter_map(|effect| match effect { + Effect::Report(Report::Quiet { market, connection }) => { + Some((market.as_str(), *connection)) + } + _ => None, + }) + .collect() + } + + #[test] + fn a_subscription_quiet_on_one_connection_is_resubscribed_then_reported() { + let (mut feed, token, _, free, _, at) = two_connection_symbol(); + let market = feed.symbols[&token].market.clone(); + let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n); + begin_minute(&mut feed, at); + let (losses, gaps) = (feed.counts.losses, feed.counts.subscription_gaps); + + // Two changes against none: too few to say. + changes(&mut feed, &token, &[0], minute(0), free, 2); + let effects = feed.idle(minute(1)); + assert!(resyncs_of(&effects, &token).is_empty()); + assert_eq!(feed.counts.quiet, 0); + + // Three: quiet on connection 1, resubscribed there. + changes(&mut feed, &token, &[0], minute(1), free, 3); + let effects = feed.idle(minute(2)); + assert_eq!(resyncs_of(&effects, &token), [(1, token.as_str())]); + assert!(quiet_reports(&effects).is_empty()); + assert_eq!(feed.counts.quiet, 1); + assert_eq!(feed.symbols[&token].state, State::Verified); + // The book the resubscribe brings back is not asked for: ignored. + let book = feed.symbols[&token].book.clone(); + let back = fresh_book(&token, &feed.symbols[&token], &book); + let effects = feed.on_frame_from(1, minute(2) + TimeDelta::seconds(1), &back); + assert!(snapshot_symbols(&effects).is_empty()); + + // Still quiet the minute after: reported for a move, not resubscribed. + changes(&mut feed, &token, &[0], minute(2), free, 3); + let effects = feed.idle(minute(3)); + assert!(resyncs_of(&effects, &token).is_empty()); + assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]); + assert_eq!( + (feed.counts.losses, feed.counts.subscription_gaps), + (losses, gaps) + ); + } + + /// Both tokens of a market go quiet together: one report for the market. + #[test] + fn a_market_quiet_on_both_its_tokens_is_reported_once() { + let (mut feed, token, _, free, _, at) = two_connection_symbol(); + let market = feed.symbols[&token].market.clone(); + let tokens = feed.markets[&market].tokens.clone(); + assert!( + tokens + .iter() + .all(|token| feed.symbols[token].state == State::Verified) + ); + let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n); + begin_minute(&mut feed, at); + for n in 0..2 { + for token in &tokens { + changes(&mut feed, token, &[0], minute(n), free, 3); + } + let effects = feed.idle(minute(n + 1)); + if n == 0 { + assert_eq!(feed.counts.quiet, 2); + } else { + assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]); + } + } + } + + #[test] + fn a_subscription_delivering_or_not_there_the_whole_minute_is_not_quiet() { + let (mut feed, token, _, free, _, at) = two_connection_symbol(); + let markets = fixture_markets(); + let market = markets + .iter() + .find(|market| market.tokens.contains(&token)) + .unwrap(); + // Connection 5 is up from before the minutes below. + feed.on_received(Received::Up(5)); + let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n); + begin_minute(&mut feed, at); + + // Both deliver: neither is quiet; the symbols that didn't change + // aren't either. + changes(&mut feed, &token, &[0, 1], minute(0), free, 3); + let effects = feed.idle(minute(1)); + assert!(resyncs(&effects).iter().all(|&(_, theirs)| theirs != token)); + + // Connection 1 dropped and came back within the minute: not judged + // for the changes it missed. + changes(&mut feed, &token, &[0], minute(1), free, 3); + feed.on_received(Received::Down(1)); + feed.on_received(Received::Up(1)); + let effects = feed.idle(minute(2)); + assert!(resyncs_of(&effects, &token).is_empty()); + + // Connection 5, subscribed in the middle of the minute and delivering + // nothing yet: not judged either. + feed.move_market(market, 1, 5, minute(2) + TimeDelta::seconds(5)); + changes(&mut feed, &token, &[0, 1], minute(2), free, 3); + let effects = feed.idle(minute(3)); + assert!(resyncs_of(&effects, &token).is_empty()); + assert_eq!(feed.counts.quiet, 0); + } + + /// A verified symbol on connections 0 and 1, changed once and then still + /// for 30 s: the feed, its token, its book before the change and now, a + /// free bid price, and the time 30 s after. + fn still_symbol() -> (Feed, String, Book, Book, i64, DateTime) { + let (mut feed, token, before, free, millis, at) = two_connection_symbol(); + let x = change(&feed, &token, millis, free, 1_000_000); + feed.on_frame_from(0, at, &x); + feed.on_frame_from(1, at, &x); + let ours = feed.symbols[&token].book.clone(); + let later = at + TimeDelta::seconds(30); + // The minute's Snapshots, out of the way. + feed.idle(later); + (feed, token, before, ours, free, later) + } + + /// Connection 1 drops and reconnects; its opening book of `token` holds + /// `book`. + fn reconnect_with(feed: &mut Feed, token: &str, book: &Book, at: DateTime) -> Vec { + feed.on_received(Received::Down(1)); + feed.on_received(Received::Up(1)); + let opening = fresh_book(token, &feed.symbols[token], book); + feed.on_frame_from(1, at, &opening) + } + + #[test] + fn a_reconnect_book_of_a_still_symbol_equal_to_ours_or_behind_changes_nothing() { + let (mut feed, token, before, ours, _, later) = still_symbol(); + let losses = feed.counts.losses; + let effects = reconnect_with(&mut feed, &token, &ours, later); + assert!(snapshot_symbols(&effects).is_empty()); + assert_eq!(feed.counts.reconnect_books, 1); + // The server's opening books lag: a state we passed is no loss. + let behind = feed.counts.books_behind; + let effects = reconnect_with(&mut feed, &token, &before, later + TimeDelta::seconds(1)); + assert!(snapshot_symbols(&effects).is_empty()); + assert_eq!(feed.counts.reconnect_books, 2); + assert_eq!(feed.counts.books_behind, behind + 1); + assert_eq!(feed.symbols[&token].book, ours); + // It lags further back than the changes we remember: no newer than + // our last change, it still holds nothing we missed. + let symbol = feed.symbols.get_mut(&token).unwrap(); + symbol.recent_changes.clear(); + let old = symbol.exchange_timestamp - TimeDelta::minutes(10); + feed.on_received(Received::Down(1)); + feed.on_received(Received::Up(1)); + let lagging = book_frame(&token, &feed.symbols[&token], &before, old); + let effects = feed.on_frame_from(1, later + TimeDelta::seconds(2), &lagging); + assert!(snapshot_symbols(&effects).is_empty()); + assert_eq!(feed.counts.books_behind, behind + 2); + assert_eq!(feed.symbols[&token].book, ours); + assert_eq!(feed.counts.losses, losses); + assert_eq!(feed.symbols[&token].state, State::Verified); + } + + /// A reconnect's book that fails its own hash proves nothing: the symbol + /// stays verified, with its book and its hash inputs. + #[test] + fn a_reconnect_book_failing_its_own_hash_changes_nothing() { + let (mut feed, token, _, ours, free, later) = still_symbol(); + let mut other = ours.clone(); + other.set(Side::Buy, free, 7_000_000); + let frame = fresh_book(&token, &feed.symbols[&token], &other); + let mut frame: serde_json::Value = serde_json::from_str(&frame).unwrap(); + frame["hash"] = "0".repeat(40).into(); + frame["last_trade_price"] = "0.123".into(); + let last_trade_price = feed.symbols[&token].last_trade_price.clone(); + feed.on_received(Received::Down(1)); + feed.on_received(Received::Up(1)); + let effects = feed.on_frame_from(1, later, &frame.to_string()); + assert!(effects.is_empty()); + let symbol = &feed.symbols[&token]; + assert_eq!(symbol.state, State::Verified); + assert_eq!(symbol.book, ours); + assert_eq!(symbol.last_trade_price, last_trade_price); + } + + /// Only a reconnect's book of a symbol with nothing asked is judged: not + /// the opening book of a subscription made since the connection came up + /// (a move's), nor any while a fresh book is asked. + #[test] + fn a_book_that_is_no_reconnects_or_comes_while_one_is_asked_is_not_judged() { + let (mut feed, token, _, ours, free, later) = still_symbol(); + let markets = fixture_markets(); + let market = markets + .iter() + .find(|market| market.tokens.contains(&token)) + .unwrap(); + let mut other = ours.clone(); + other.set(Side::Buy, free, 7_000_000); + feed.on_received(Received::Up(5)); + feed.move_market(market, 0, 5, later); + let opening = fresh_book(&token, &feed.symbols[&token], &other); + let effects = feed.on_frame_from(5, later + TimeDelta::seconds(1), &opening); + assert!(snapshot_symbols(&effects).is_empty()); + assert_eq!(feed.counts.reconnect_books, 0); + + feed.symbols.get_mut(&token).unwrap().asked = Some((5, false)); + let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(2)); + assert!(snapshot_symbols(&effects).is_empty()); + assert_eq!(feed.counts.reconnect_books, 0); + assert_eq!(feed.symbols[&token].book, ours); + } + + #[test] + fn a_reconnect_book_of_a_still_symbol_that_differs_is_a_loss_and_resubscribes_the_other() { + let (mut feed, token, _, ours, free, later) = still_symbol(); + let (losses, gaps) = (feed.counts.losses, feed.counts.subscription_gaps); + // Even before any change of it was verified: a loss, not the gap of + // a subscribe, which is long past. + feed.symbols.get_mut(&token).unwrap().verified_since_book = false; + let mut other = ours.clone(); + other.set(Side::Buy, free, 7_000_000); + let effects = reconnect_with(&mut feed, &token, &other, later); + assert_eq!(snapshot_symbols(&effects), [token.as_str()]); + assert_eq!(resyncs_of(&effects, &token), [(0, token.as_str())]); + assert_eq!(feed.counts.losses, losses + 1); + assert_eq!(feed.counts.subscription_gaps, gaps); + assert_eq!(feed.counts.reconnect_losses, 1); + assert_eq!(feed.symbols[&token].book, other); + assert_eq!(feed.symbols[&token].state, State::Verified); + } + + #[test] + fn a_reconnect_book_of_a_symbol_changed_lately_is_ignored() { + let (mut feed, token, _, ours, free, later) = still_symbol(); + let millis = feed.symbols[&token].exchange_timestamp.timestamp_millis() + 1_000; + let x = change(&feed, &token, millis, free, 2_000_000); + feed.on_frame_from(0, later, &x); + let mut other = ours.clone(); + other.set(Side::Buy, free, 7_000_000); + let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(1)); + assert!(snapshot_symbols(&effects).is_empty()); + assert_eq!(feed.counts.reconnect_books, 0); + // And the next book of it from that connection is no reconnect's. + feed.idle(later + TimeDelta::seconds(30)); + let opening = fresh_book(&token, &feed.symbols[&token], &other); + let effects = feed.on_frame_from(1, later + TimeDelta::seconds(31), &opening); + assert!(snapshot_symbols(&effects).is_empty()); + assert_eq!(feed.counts.reconnect_books, 0); + } + #[test] fn the_followed_connection_down_switches_with_a_snapshot() { let (mut feed, token, book, _, _, at) = two_connection_symbol(); @@ -2402,7 +2951,7 @@ mod tests { .iter() .find(|market| market.tokens.contains(&token)) .unwrap(); - let effects = feed.move_market(market, 0, 5); + let effects = feed.move_market(market, 0, 5, at); assert!(matches!( effects.as_slice(), [Effect::Send(5, Command::Subscribe(tokens))] if *tokens == market.tokens @@ -2442,7 +2991,7 @@ mod tests { let symbol = feed.symbols.get_mut(&token).unwrap(); symbol.state = State::Following(1); symbol.asked = Some((0, false)); - feed.move_market(market, 0, 5); + feed.move_market(market, 0, 5, at); let opening = fresh_book(&token, &feed.symbols[&token], &book); let effects = feed.on_frame_from(5, at, &opening); assert!( @@ -2515,7 +3064,7 @@ mod tests { feed.on_frame_from(0, minute + TimeDelta::seconds(10), &change); feed.on_frame_from(1, minute + TimeDelta::seconds(10), &change); let effects = feed.tick(minute + TimeDelta::seconds(60)); - let Some(Effect::Report(loads)) = effects.first() else { + let Some(Effect::Report(Report::Loads(loads))) = effects.first() else { panic!("no report"); }; assert_eq!(loads["0xm"], change.len() as u64 / 60); @@ -2583,9 +3132,10 @@ mod tests { let mentions = |effect: &Effect| match effect { Effect::Write(Output::Snapshot(snapshot)) => removed.contains(&snapshot.symbol), Effect::Write(Output::Delta(delta)) => removed.contains(&delta.symbol), - Effect::Send(_, Command::Resync(token)) => removed.contains(token), + Effect::Send(_, Command::Resubscribe(token)) => removed.contains(token), Effect::Send(..) => false, - Effect::Report(loads) => loads.contains_key(&markets[0].condition_id), + Effect::Report(Report::Loads(loads)) => loads.contains_key(&markets[0].condition_id), + Effect::Report(Report::Quiet { market, .. }) => *market == markets[0].condition_id, }; assert!(!effects.iter().any(mentions)); // The minute Snapshots of the 38 others, twice. @@ -2627,7 +3177,7 @@ mod tests { let mut reports = Vec::new(); let mut on_frame = |feed: &mut Feed, seconds, text: &str| { for effect in feed.on_frame(at(seconds), text) { - if let Effect::Report(loads) = effect { + if let Effect::Report(Report::Loads(loads)) = effect { reports.push(loads); } } @@ -2671,7 +3221,7 @@ mod tests { let reports: Vec = replay(&mut feed, &frames) .into_iter() .filter_map(|effect| match effect { - Effect::Report(loads) => Some(loads), + Effect::Report(Report::Loads(loads)) => Some(loads), _ => None, }) .collect(); diff --git a/src/polymarket/orchestrator.rs b/src/polymarket/orchestrator.rs index a403de2..2cbb8fc 100644 --- a/src/polymarket/orchestrator.rs +++ b/src/polymarket/orchestrator.rs @@ -1,7 +1,7 @@ //! The orchestrator: the one owner of what is tracked. It keeps the tracked //! markets equal to Gamma's live ones, decides which connections carry each -//! market and owns every timer; the two fetchers only fetch (ADR 0002, ADR -//! 0003, roadmap/04). +//! market and owns every timer, recycling connections among them; the two +//! fetchers only fetch (ADR 0002, ADR 0003, roadmap/04, roadmap/05). use std::cmp::Reverse; use std::collections::{HashMap, HashSet, VecDeque}; @@ -14,8 +14,9 @@ use chrono::{DateTime, TimeDelta, Utc}; use crossbeam::channel::{Sender, select, unbounded}; use engineering_challenge::book::Output; -use super::feed::{self, BookCommand, Feed, Loads}; +use super::feed::{self, BookCommand, Feed, Loads, Report}; use super::gamma::{self, Finding, Request}; +use super::socket::{Command, Notice}; use super::wire::Received; use super::{BOOK_THREADS, Market, book_thread, socket}; @@ -50,6 +51,9 @@ const RESCAN_EVERY: Duration = Duration::from_secs(60 * 60); /// `createdAt` (docs/notes.md); known markets are skipped, so the margin costs /// only the pages it reads again. const RECENT_MARGIN: TimeDelta = TimeDelta::minutes(15); +/// Each connection is recycled about this often, one at a time, to shed what +/// accumulates on the exchange's side of it. +const RECYCLE_EVERY: Duration = Duration::from_secs(60 * 60); /// So SIGINT is seen while no timer is due. const WAKE_EVERY: Duration = Duration::from_secs(1); @@ -70,10 +74,10 @@ pub fn run(outputs: Sender, stop: &Arc) -> usize { }) .collect(); drop(findings_tx); - let (reports_tx, reports) = unbounded::(); - let (closes_tx, closes) = unbounded::(); - let mut threads = Threads::start(&outputs, &reports_tx, &closes_tx, stop); - drop(closes_tx); + let (reports_tx, reports) = unbounded::(); + let (notices_tx, notices) = unbounded::(); + let mut threads = Threads::start(&outputs, &reports_tx, ¬ices_tx, stop); + drop(notices_tx); // The book threads hold the only senders the writer waits for. drop(outputs); drop(reports_tx); @@ -91,6 +95,7 @@ pub fn run(outputs: Sender, stop: &Arc) -> usize { let mut queue: VecDeque = VecDeque::new(); let mut next_deal = Instant::now(); let (mut added, mut removed, mut moved, mut splits) = (0, 0, 0, 0); + let (mut recycled, mut quiet_moves, mut silent) = (0, 0, 0); // The minute checks cover the first scan from its start. let mut recent_since = Utc::now() - RECENT_MARGIN; let mut recent_asked_at: Option> = None; @@ -134,12 +139,22 @@ pub fn run(outputs: Sender, stop: &Arc) -> usize { let busiest = tracker.connections.iter().map(|one| one.load).max(); log!( "polymarket: orchestrator: {} markets on {} connections; {added} added, {removed} \ - removed, {moved} moved, {splits} splits this minute; busiest connection at {} bytes/s", + removed, {moved} moved ({quiet_moves} quiet), {splits} splits, {recycled} recycled, \ + {silent} silent this minute; busiest connection at {} bytes/s", tracker.markets.len(), tracker.connections.len(), busiest.unwrap_or(0) ); (added, removed, moved, splits) = (0, 0, 0, 0); + (recycled, quiet_moves, silent) = (0, 0, 0); + } + // Recycling starts once the startup deal is sent. + if first_scan.is_none() && queue.is_empty() { + tracker.start_recycling(now); + } + if let Some(connection) = tracker.recycle(now) { + threads.recycle(connection); + recycled += 1; } if !queue.is_empty() && now >= next_deal { for command in queue.drain(..DEAL_AT_ONCE.min(queue.len())) { @@ -217,13 +232,26 @@ pub fn run(outputs: Sender, stop: &Arc) -> usize { // Both fetchers are gone: the check above stops the run. Err(_) => {} }, - recv(reports) -> loads => { - if let Ok(loads) = loads { - tracker.report(&loads); + recv(reports) -> report => match report { + Ok(Report::Loads(loads)) => tracker.report(&loads), + Ok(Report::Quiet { market, connection }) => { + if let Some(command) = tracker.quiet(&market, connection, Instant::now()) { + log!( + "polymarket: orchestrator: {market} still quiet on connection \ + {connection} after a resubscribe, moved" + ); + quiet_moves += 1; + moved += 1; + commands.push(command); + } } + Err(_) => {} }, - recv(closes) -> closed => { - if let Ok(connection) = closed { + recv(notices) -> notice => match notice { + Ok(Notice::Up(connection)) => tracker.up(connection, Instant::now()), + Ok(Notice::Down(connection)) => tracker.down(connection), + Ok(Notice::Silent(_)) => silent += 1, + Ok(Notice::TooSlow(connection)) => { let moves = tracker.closed(connection, Instant::now()); if !moves.is_empty() { log!( @@ -236,6 +264,7 @@ pub fn run(outputs: Sender, stop: &Arc) -> usize { } commands.extend(moves); } + Err(_) => {} }, default(timeout) => {} } @@ -261,7 +290,9 @@ struct Threads { /// Every book thread's events channel, by number: each connection gets a /// clone of them all. events: Vec>, - closes: Sender, + notices: Sender, + /// Each connection's commands, by number, for its recycles. + connection_commands: Vec>, book_threads: Vec>, connections: Vec>, stop: Arc, @@ -270,14 +301,15 @@ struct Threads { impl Threads { fn start( outputs: &Sender, - reports: &Sender, - closes: &Sender, + reports: &Sender, + notices: &Sender, stop: &Arc, ) -> Self { let mut threads = Self { book_commands: Vec::new(), events: Vec::new(), - closes: closes.clone(), + notices: notices.clone(), + connection_commands: Vec::new(), book_threads: Vec::new(), connections: Vec::new(), stop: Arc::clone(stop), @@ -302,23 +334,29 @@ impl Threads { let number = self.connections.len(); let (commands, commands_rx) = unbounded(); let (events, stop) = (self.events.clone(), Arc::clone(&self.stop)); - let closes = self.closes.clone(); + let notices = self.notices.clone(); self.connections.push(thread::spawn(move || { socket::run( socket::MARKET_CHANNEL_URL, number, &events, &commands_rx, - &closes, + ¬ices, + socket::SILENT_AFTER, &stop, ) })); + self.connection_commands.push(commands.clone()); for book_thread in &self.book_commands { _ = book_thread.send(BookCommand::Connected(number, commands.clone())); } } } + fn recycle(&self, connection: usize) { + _ = self.connection_commands[connection].send(Command::Recycle); + } + /// To the market's book thread. One that died keeps its markets unserved, /// and its connections stop the run. fn send(&self, command: BookCommand) { @@ -360,16 +398,19 @@ struct Tracked { moved_at: Option, } -/// A connection's tokens and load, against its two limits, and its 1013 -/// closes. +/// A connection's tokens and load, against its two limits, its 1013 closes, +/// and whether and since when it is up. #[derive(Debug, Default, Clone, PartialEq, Eq)] -struct ConnectionLoad { +struct ConnectionState { tokens: usize, /// The sum of its markets' loads, bytes per second. load: u64, /// When it was last closed with 1013. closed_at: Option, split_at: Option, + up: bool, + /// When it last came up: the one longest ago is recycled next. + connected_at: Option, } /// Which connections carry each market and what it weighs, and which markets @@ -379,10 +420,14 @@ struct Tracker { /// By `conditionId`. markets: HashMap, /// By connection number. - connections: Vec, + connections: Vec, /// While a rescan runs: the markets known at its start, not seen since. unseen: Option>, random: Random, + /// When the next recycle is due; None until recycling starts. + next_recycle: Option, + /// The connection recycled and not yet up again. + recycling: Option, } impl Tracker { @@ -393,6 +438,8 @@ impl Tracker { connections: Vec::new(), unseen: None, random: Random(seed | 1), + next_recycle: None, + recycling: None, } } @@ -410,7 +457,7 @@ impl Tracker { let count = (self.subscriptions * tokens) .div_ceil(STARTUP_FILL) .max(self.subscriptions); - self.connections = vec![ConnectionLoad::default(); count]; + self.connections = vec![ConnectionState::default(); count]; let mut placed = vec![Vec::new(); markets.len()]; for _ in 0..self.subscriptions { let mut order: Vec = (0..markets.len()).collect(); @@ -474,7 +521,7 @@ impl Tracker { }) .collect(); if with_room.is_empty() { - self.connections.push(ConnectionLoad::default()); + self.connections.push(ConnectionState::default()); return self.connections.len() - 1; } with_room[self.random.below(with_room.len())] @@ -604,6 +651,73 @@ impl Tracker { .collect() } + /// A market the book thread found quiet on `connection` again after a + /// resubscribe: its subscription there moves, beyond rebalancing's + /// limits. None if it no longer is there. + fn quiet(&mut self, id: &str, connection: usize, now: Instant) -> Option { + let tracked = self.markets.get(id)?; + // A market moved lately stays, as for rebalancing: its last move may + // still be under way, and a second one would end it early. + let stays = tracked + .moved_at + .is_some_and(|at| now.duration_since(at) < MOVED_STAYS); + (tracked.connections.contains(&connection) && !stays) + .then(|| self.move_off(id, connection, now)) + } + + /// A connection came up: its subscribes are sent, its opening books + /// still to come (0.3 to 20 s, docs/notes.md), so no recycle starts for + /// one interval. + fn up(&mut self, connection: usize, now: Instant) { + if let Some(up) = self.connections.get_mut(connection) { + up.up = true; + up.connected_at = Some(now); + } + if self.recycling == Some(connection) { + self.recycling = None; + } + let settled = now + self.recycle_interval(); + if let Some(next) = &mut self.next_recycle { + *next = (*next).max(settled); + } + } + + fn down(&mut self, connection: usize) { + if let Some(down) = self.connections.get_mut(connection) { + down.up = false; + } + } + + fn start_recycling(&mut self, now: Instant) { + if self.next_recycle.is_none() { + self.next_recycle = Some(now + self.recycle_interval()); + } + } + + /// Each connection about once per [`RECYCLE_EVERY`], one at a time. + fn recycle_interval(&self) -> Duration { + RECYCLE_EVERY / self.connections.len().max(1) as u32 + } + + /// The connection to recycle now, if one is due: the one up the longest, + /// once the last recycled is up again, and only while every connection is + /// up and has been for an interval, so no market loses both of its + /// connections at once. + fn recycle(&mut self, now: Instant) -> Option { + let due = self.next_recycle.is_some_and(|next| now >= next); + if !due + || self.recycling.is_some() + || self.connections.iter().any(|connection| !connection.up) + { + return None; + } + let oldest = (0..self.connections.len()) + .min_by_key(|&number| self.connections[number].connected_at)?; + self.recycling = Some(oldest); + self.next_recycle = Some(now + self.recycle_interval()); + Some(oldest) + } + fn scan_started(&mut self) { self.unseen = Some(self.markets.keys().cloned().collect()); } @@ -795,7 +909,7 @@ mod tests { for id in 0..10 { tracker.add(market(id)); } - tracker.connections.push(ConnectionLoad::default()); + tracker.connections.push(ConnectionState::default()); let budget = CONNECTION_BYTES_PER_SECOND; // Both connections at 1.5 budgets, market 0 weighing half a budget. let mut loads = vec![(0, budget / 2)]; @@ -831,7 +945,7 @@ mod tests { tracker.add(market(id)); } // Room elsewhere, on several connections. - tracker.connections.resize(6, ConnectionLoad::default()); + tracker.connections.resize(6, ConnectionState::default()); report( &mut tracker, &[(0, CONNECTION_BYTES_PER_SECOND), (1, 1_000)], @@ -850,8 +964,8 @@ mod tests { fn the_busiest_connection_over_budget_is_rebalanced_first() { let mut tracker = Tracker::new(1, 7); tracker.deal(vec![market(0), market(1)]); - tracker.connections.push(ConnectionLoad::default()); - tracker.connections.push(ConnectionLoad::default()); + tracker.connections.push(ConnectionState::default()); + tracker.connections.push(ConnectionState::default()); // One connection so far: put markets 2 and 3 on another. for id in [2, 3] { tracker.connections[0].tokens = CONNECTION_TOKENS; @@ -931,6 +1045,69 @@ mod tests { assert_eq!(moves(&commands), [("0x2", 1, 2)]); } + #[test] + fn recycling_takes_the_oldest_connection_one_at_a_time_while_every_one_is_up() { + let mut tracker = tracker(); + tracker.connections = vec![ConnectionState::default(); 3]; + let start = Instant::now(); + let at = |seconds: u64| start + Duration::from_secs(seconds); + // Up in the order 2, 0, 1. + for (seconds, connection) in [(0, 2), (1, 0), (2, 1)] { + tracker.up(connection, at(seconds)); + } + assert_eq!(tracker.recycle(at(5_000)), None, "not started"); + tracker.start_recycling(at(10)); + // One every hour / 3 connections. + assert_eq!(tracker.recycle(at(1_209)), None); + assert_eq!(tracker.recycle(at(1_210)), Some(2)); + // The next waits for the recycled one, whose Down may not even have + // come yet, to be up again... + assert_eq!(tracker.recycle(at(2_410)), None); + tracker.down(2); + assert_eq!(tracker.recycle(at(2_410)), None); + tracker.up(2, at(2_411)); + // ...and for its opening books: an interval more. + assert_eq!(tracker.recycle(at(3_610)), None); + assert_eq!(tracker.recycle(at(3_611)), Some(0)); + tracker.down(0); + tracker.up(0, at(3_612)); + // A connection down for its own reasons holds every recycle, and its + // return delays the next like any other. + tracker.down(2); + assert_eq!(tracker.recycle(at(6_000)), None); + tracker.up(2, at(6_001)); + assert_eq!(tracker.recycle(at(7_200)), None); + assert_eq!(tracker.recycle(at(7_201)), Some(1)); + } + + #[test] + fn a_market_quiet_again_moves_off_that_connection_unless_it_moved_lately() { + let mut tracker = tracker(); + tracker.deal((0..50).map(market).collect()); + let now = Instant::now(); + let on = connections(&tracker, 0); + let first = [tracker.quiet("0x0", on[0], now).expect("a move")]; + let [(id, from, to)] = moves(&first)[..] else { + panic!("one move"); + }; + assert_eq!((id, from), ("0x0", on[0])); + assert!(!on.contains(&to)); + // Moved a minute ago, its move perhaps still under way: it stays, + // as for rebalancing. + let on = connections(&tracker, 0); + let soon = now + Duration::from_secs(60); + assert!(tracker.quiet("0x0", on[1], soon).is_none()); + assert!(tracker.quiet("0x0", on[1], now + MOVED_STAYS).is_some()); + let now = now + MOVED_STAYS + MOVED_STAYS; + // Reported for a connection it isn't on (any more): nothing. + let on = connections(&tracker, 0); + let elsewhere = (0..tracker.connections.len()) + .find(|connection| !on.contains(connection)) + .unwrap(); + assert!(tracker.quiet("0x0", elsewhere, now).is_none()); + assert!(tracker.quiet("0xunknown", 0, now).is_none()); + } + #[test] fn a_connection_closed_twice_within_ten_minutes_is_split() { let mut tracker = Tracker::new(1, 7); diff --git a/src/polymarket/socket.rs b/src/polymarket/socket.rs index 438182e..b4e455d 100644 --- a/src/polymarket/socket.rs +++ b/src/polymarket/socket.rs @@ -1,6 +1,7 @@ //! A connection: reads the market channel, stamps each frame's receive time, //! decodes it and sends each event to the book thread of its market; and //! sends the exchange what the book threads ask for (subscriptions, resyncs). +//! It reconnects when it drops, falls silent, or is recycled. use std::collections::BTreeSet; use std::io::ErrorKind; @@ -29,6 +30,9 @@ const READ_TIMEOUT: Duration = Duration::from_millis(200); const PING_EVERY: Duration = Duration::from_secs(10); const RECONNECT_AFTER: Duration = Duration::from_secs(1); const CLOSE_TIMEOUT: Duration = Duration::from_secs(2); +/// Nothing received this long, not even the answer to three keepalives: the +/// connection is silent, and reconnects. +pub const SILENT_AFTER: Duration = Duration::from_secs(30); /// Tokens per subscribe or unsubscribe message: a first message of more than /// 500 gets no opening books (docs/notes.md). const CHUNK: usize = 500; @@ -39,27 +43,46 @@ pub enum Command { Subscribe(Vec), Unsubscribe(Vec), /// Unsubscribe and subscribe again: the server then sends a fresh book. - Resync(String), + /// A resync asks for that book; a quiet subscription only for the new + /// subscription. + Resubscribe(String), + /// Close and reconnect, subscribing every token again. + Recycle, +} + +/// What a connection tells the orchestrator. +#[derive(Debug, PartialEq, Eq)] +pub enum Notice { + /// Connected, its tokens subscribed. + Up(usize), + Down(usize), + /// Closed by the server with 1013: it couldn't send to us fast enough. + TooSlow(usize), + /// Closed by us: nothing came for [`SILENT_AFTER`]. + Silent(usize), } type Socket = WebSocket>; /// Runs until `stop` is set or a book thread is gone, reconnecting (and -/// subscribing its tokens again) whenever the connection drops, and telling -/// every book thread when it comes up and goes down. `book_threads` holds -/// every book thread's channel, by number; `closes` hears of each 1013 close. +/// subscribing its tokens again) whenever the connection drops, falls silent +/// for `silent_after` or is recycled, and telling every book thread and the +/// orchestrator (`notices`) when it comes up and goes down. `book_threads` +/// holds every book thread's channel, by number. pub fn run( url: &str, number: usize, book_threads: &[Sender], commands: &Receiver, - closes: &Sender, + notices: &Sender, + silent_after: Duration, stop: &AtomicBool, ) { - let tell = |received: fn(usize) -> Received| { + let tell = |received: fn(usize) -> Received, notice: fn(usize) -> Notice| { for book_thread in book_threads { _ = book_thread.send(received(number)); } + _ = notices.send(notice(number)); }; let mut tokens = BTreeSet::new(); while !stop.load(Ordering::Relaxed) { @@ -74,21 +97,23 @@ pub fn run( "polymarket: connection {number}: connected, {} symbols", tokens.len() ); - tell(Received::Up); + tell(Received::Up, Notice::Up); let ended = read( &mut socket, number, &mut tokens, book_threads, commands, - closes, + notices, + silent_after, stop, ); if !matches!(ended, Ok(Ended::Stopped)) { - tell(Received::Down); + tell(Received::Down, Notice::Down); } match ended { Ok(Ended::Stopped) => return close(&mut socket, number, book_threads), + Ok(Ended::Recycled) => log!("polymarket: connection {number}: recycled"), Ok(Ended::BookThreadGone) => { // Only a panic ends a book thread before us: its // markets would go unserved, so the run stops. @@ -109,6 +134,7 @@ pub fn run( enum Ended { Stopped, BookThreadGone, + Recycled, } /// Updates the token set, and returns the messages that carry `command` out @@ -127,12 +153,14 @@ fn apply(tokens: &mut BTreeSet, command: &Command) -> Vec { // Subscribing a token already subscribed sends nothing back but // `NO NEW ASSETS`: unsubscribing first makes the server send a fresh // book. - Command::Resync(token) if tokens.contains(token) => [ + Command::Resubscribe(token) if tokens.contains(token) => [ operations("unsubscribe", [token]), operations("subscribe", [token]), ] .concat(), - Command::Resync(_) => Vec::new(), + Command::Resubscribe(_) => Vec::new(), + // The read loop acts on it; disconnected, the reconnect does. + Command::Recycle => Vec::new(), } } @@ -219,20 +247,27 @@ fn truncate(text: &str) -> &str { .map_or(text, |(end, _)| &text[..end]) } +// The connection's whole context, each a different part of it. +#[allow(clippy::too_many_arguments)] fn read( socket: &mut Socket, number: usize, tokens: &mut BTreeSet, book_threads: &[Sender], commands: &Receiver, - closes: &Sender, + notices: &Sender, + silent_after: Duration, stop: &AtomicBool, ) -> anyhow::Result { let mut last_ping = Instant::now(); + let mut last_heard = Instant::now(); loop { let message = socket.read(); // Stamp as soon as the frame is off the wire, before any processing. let received_at = Utc::now(); + if message.is_ok() { + last_heard = Instant::now(); + } match message { Ok(Message::Text(text)) => { if !route(received_at, text.as_str(), number, book_threads) { @@ -246,7 +281,7 @@ fn read( .as_ref() .is_some_and(|frame| frame.code == CloseCode::Again) { - _ = closes.send(number); + _ = notices.send(Notice::TooSlow(number)); } anyhow::bail!("closed by the server: {frame:?}") } @@ -258,11 +293,23 @@ fn read( if stop.load(Ordering::Relaxed) { return Ok(Ended::Stopped); } + if last_heard.elapsed() >= silent_after { + _ = notices.send(Notice::Silent(number)); + anyhow::bail!("silent for {} s", silent_after.as_secs_f64()); + } if last_ping.elapsed() >= PING_EVERY { socket.send(Message::text("PING"))?; last_ping = Instant::now(); } for command in commands.try_iter() { + if command == Command::Recycle { + // Not waited on: whatever still comes would arrive from a + // connection already down for the book threads. The commands + // behind it are applied before the reconnect. + _ = socket.close(None); + _ = socket.flush(); + return Ok(Ended::Recycled); + } for message in apply(tokens, &command) { socket.send(Message::text(message))?; } @@ -351,9 +398,18 @@ mod tests { .unwrap(); let stop = Arc::new(AtomicBool::new(false)); let client_stop = Arc::clone(&stop); - let (closes, _closes_rx) = unbounded(); - let client = - thread::spawn(move || run(&url, 0, &[events_tx], &commands_rx, &closes, &client_stop)); + let (notices, _notices_rx) = unbounded(); + let client = thread::spawn(move || { + run( + &url, + 0, + &[events_tx], + &commands_rx, + ¬ices, + SILENT_AFTER, + &client_stop, + ) + }); let connections = server.join().unwrap(); stop.store(true, Ordering::Relaxed); @@ -368,6 +424,103 @@ mod tests { } } + /// A local server that accepts two connections of `total` tokens, each + /// handled by `serve` once subscribed, and answers what the client + /// subscribed on each. + fn serving_twice( + total: usize, + serve: fn(&mut WebSocket), + ) -> (String, thread::JoinHandle>>>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("ws://{}/ws/market", listener.local_addr().unwrap()); + let server = thread::spawn(move || { + (0..2) + .map(|_| { + let (stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut socket = tungstenite::accept(stream).unwrap(); + let chunks = subscribed(&mut socket, total); + serve(&mut socket); + chunks + }) + .collect() + }); + (url, server) + } + + /// Runs a connection of `tokens` against `url` until `until` returns, + /// then stops it: the notices it sent. + fn client( + url: String, + tokens: Vec, + silent_after: Duration, + until: impl FnOnce(&Sender), + ) -> Vec { + let (events_tx, _events_rx) = unbounded(); + let (commands_tx, commands_rx) = unbounded(); + commands_tx.send(Command::Subscribe(tokens)).unwrap(); + let stop = Arc::new(AtomicBool::new(false)); + let client_stop = Arc::clone(&stop); + let (notices, notices_rx) = unbounded(); + let client = thread::spawn(move || { + run( + &url, + 0, + &[events_tx], + &commands_rx, + ¬ices, + silent_after, + &client_stop, + ) + }); + until(&commands_tx); + stop.store(true, Ordering::Relaxed); + client.join().unwrap(); + notices_rx.try_iter().collect() + } + + #[test] + fn a_silent_connection_reconnects_with_its_tokens() { + let tokens: Vec = (0..10).map(|token| token.to_string()).collect(); + // Silent: reads what comes (the keepalives) and answers nothing, + // until the client closes. + let (url, server) = serving_twice(tokens.len(), |socket| while socket.read().is_ok() {}); + let notices = client(url, tokens.clone(), Duration::from_millis(500), |_| { + thread::sleep(Duration::from_millis(2_500)); + }); + let connections = server.join().unwrap(); + assert_eq!(connections.len(), 2); + assert!(connections.iter().all(|chunks| chunks.concat() == tokens)); + assert_eq!( + notices[..3], + [Notice::Up(0), Notice::Silent(0), Notice::Down(0)] + ); + assert_eq!(notices[3], Notice::Up(0)); + } + + #[test] + fn a_recycle_closes_and_subscribes_every_token_again() { + let tokens: Vec = (0..10).map(|token| token.to_string()).collect(); + // Answers the keepalives, as the exchange does, until closed. + let (url, server) = serving_twice(tokens.len(), |socket| { + while let Ok(message) = socket.read() { + if message.is_text() { + _ = socket.send(Message::text("PONG")); + } + } + }); + let notices = client(url, tokens.clone(), SILENT_AFTER, |commands| { + thread::sleep(Duration::from_millis(300)); + commands.send(Command::Recycle).unwrap(); + thread::sleep(Duration::from_millis(1_800)); + }); + let connections = server.join().unwrap(); + assert!(connections.iter().all(|chunks| chunks.concat() == tokens)); + assert_eq!(notices, [Notice::Up(0), Notice::Down(0), Notice::Up(0)]); + } + /// The recording's first frame holds the opening books of 40 tokens, 20 /// markets: each book goes to its market's book thread, with its share of /// the frame's bytes. @@ -433,7 +586,7 @@ mod tests { [op("subscribe", &["a", "b"])] ); assert_eq!( - sent(apply(&mut tokens, &Command::Resync("a".into()))), + sent(apply(&mut tokens, &Command::Resubscribe("a".into()))), [op("unsubscribe", &["a"]), op("subscribe", &["a"])] ); let unsubscribe = Command::Unsubscribe(vec!["a".into()]); @@ -441,7 +594,7 @@ mod tests { sent(apply(&mut tokens, &unsubscribe)), [op("unsubscribe", &["a"])] ); - assert!(apply(&mut tokens, &Command::Resync("a".into())).is_empty()); + assert!(apply(&mut tokens, &Command::Resubscribe("a".into())).is_empty()); assert_eq!(tokens.into_iter().collect::>(), ["b"]); } }