From 515e85941d945ce4ca8120ebff3c018c18709ec4 Mon Sep 17 00:00:00 2001 From: Pascal Perrenoud Date: Thu, 12 Nov 2026 15:05:25 +0100 Subject: [PATCH] feat(latency): latency experiments, LATENCY.md, Kalshi's slow-connection recycle Extension 4. `engineering-challenge latency EXCHANGE OUT.csv` opens 16 connections to the same busy markets and writes every copy of every change; `scripts/latency/analyse.py` gives each connection's speed, whether it persists, what keeping the fastest would gain, lateness by address, and lag with the clock's offset. LATENCY.md writes up four runs: on Polymarket speed persists but is worth under a millisecond, at the median only; on Kalshi's demo keeping the fastest pair lowers p99 by 28% and 42%, the threshold agreed before the runs. So the collector now recycles a Kalshi connection whose mean lateness stays, 3 minutes running, in the slowest quarter and above twice everyone's, ahead of its hourly turn. Kalshi's rule measures each duplicate's lateness; the orchestrator judges once a minute. A duplicate's lag in Polymarket's minute line is now its "lateness". Roadmap milestone 7, roadmap/07. --- .gitignore | 1 + CLAUDE.md | 18 +- Cargo.lock | 27 ++ Cargo.toml | 2 + GLOSSARY.md | 13 + LATENCY.md | 211 ++++++++++++ docs/notes.md | 18 + docs/project.md | 7 +- docs/roadmap.md | 27 +- roadmap/07. latency experiments.md | 361 ++++++++++++++++++++ scripts/ci.sh | 5 + scripts/latency/analyse.py | 258 ++++++++++++++ scripts/latency/ruff.toml | 15 + scripts/latency/test_analyse.py | 203 +++++++++++ src/core/feed.rs | 17 + src/core/orchestrator.rs | 242 ++++++++++++- src/core/socket.rs | 37 +- src/kalshi/feed.rs | 130 ++++++- src/kalshi/listing.rs | 22 +- src/latency.rs | 524 +++++++++++++++++++++++++++++ src/main.rs | 9 +- src/polymarket/feed.rs | 40 +-- 22 files changed, 2132 insertions(+), 55 deletions(-) create mode 100644 LATENCY.md create mode 100644 roadmap/07. latency experiments.md create mode 100644 scripts/latency/analyse.py create mode 100644 scripts/latency/ruff.toml create mode 100644 scripts/latency/test_analyse.py create mode 100644 src/latency.rs diff --git a/.gitignore b/.gitignore index b83d222..4d26375 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target/ +__pycache__/ diff --git a/CLAUDE.md b/CLAUDE.md index 1270ee1..38d4fe2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,10 +20,11 @@ outside this repo. - `GLOSSARY.md`: the terms (symbol, connection, load, first one wins, duplicate, held, rescued, follow, resync, loss, subscription gap, split, quiet, resubscribe, silent, recycle, sequence number, clock, Snapshot, - Delta). + Delta, lateness, lag, speed). - `src/core/`: what is the same for every exchange (ADR 0004). `orchestrator.rs`: what is tracked, which connections carry each market, - their balancing and their recycling; `socket.rs`: a connection (read, + their balancing and their recycling (hourly, and sooner for a Kalshi + connection that stays among the slowest); `socket.rs`: a connection (read, stamp, decode, route; reconnects when dropped, silent, lost a message or recycled); `feed.rs`: a book thread's shell (Snapshots, moves, the quiet watch) and the `Rule` trait; `mod.rs`: the `Exchange`, `Session`, `Market` @@ -38,6 +39,11 @@ outside this repo. `src/bin/reconcile.rs`: the output check, reading the log's cause lines (`src/cause.rs`). `tests/fixtures/`: recordings of the live feeds that the tests replay. +- `src/latency.rs` + `scripts/latency/analyse.py`: the latency probe + (`engineering-challenge latency EXCHANGE OUT.csv`, until SIGINT: one CSV + line per copy of a change on each of 16 connections) and its analysis + (speed, persistence, racing, by address, lag); `LATENCY.md` is the + writeup (roadmap/07). - `src/util/` and `sbe_schemas/`: the template's SBE machinery, used as given so output stays readable by the template's `AlignedSbeReader`. Our code is the rest of `src/`. @@ -102,6 +108,14 @@ and let the user settle it. Claude owns the data model. - **Live data beats the docs**: Polymarket's market channel has rules its docs don't state (`docs/notes.md`); check a protocol assumption on a recording of the real feed (`tests/fixtures/record.py`) before relying on it. +- **The workspace can't choose what the exchange sees**: a NAT rewrites + every connection's source port (local 41000 left as 36996, then as 37086 + a minute later; `curl --local-port P https://ifconfig.me/all.json`), and + there is no IPv6 route (connections to an IPv6 address fail with "Network + is unreachable"). The probe uses IPv4 addresses only. +- **Python checks run through `uvx`**: no ruff or pyflakes is installed; + `scripts/ci.sh` runs `uvx ruff@0.15.0` (config in + `scripts/latency/ruff.toml`). ## Workflow differences diff --git a/Cargo.lock b/Cargo.lock index 662102d..216db85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -314,6 +314,8 @@ dependencies = [ "serde_json", "serde_yaml_ng", "sha1", + "sntpc", + "sntpc-net-std", "tungstenite", "ureq", ] @@ -657,6 +659,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "miniloop" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad14b72c9fa5a663d069c582d576e5336995e3fafe7d0965d2bba3db1f98c40c" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -998,6 +1006,25 @@ version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" +[[package]] +name = "sntpc" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "294278b134cd7d06d7f22eca970aebe38707864c080f66547f9ad26327534af3" +dependencies = [ + "cfg-if", + "miniloop", +] + +[[package]] +name = "sntpc-net-std" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335b2e533977c45a8c909c2297c659942aee18f2319aae29ee6038271d8d0131" +dependencies = [ + "sntpc", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" diff --git a/Cargo.toml b/Cargo.toml index 583dd35..cdcd9b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,8 @@ rustls-pki-types = { version = "1", features = ["std"] } serde = { version = "1", features = ["derive"] } serde_json = "1" sha1 = "0.11" +sntpc = { version = "0.11", features = ["sync", "std"] } +sntpc-net-std = "1.3" tungstenite = { version = "0.30", features = ["rustls-tls-webpki-roots"] } ureq = "3" diff --git a/GLOSSARY.md b/GLOSSARY.md index 04c8622..146f366 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -64,6 +64,19 @@ _Avoid_: Source, primary, standby, failover (no connection has a rank) **Duplicate**: A change another connection already delivered. +**Lateness**: +How long after a change's first copy a connection delivered it; 0 for the +connection that delivered first. Needs no clock. +_Avoid_: Lag, delay + +**Lag**: +A change's receive time minus the exchange's timestamp for it. It carries +the local clock's offset. +_Avoid_: Latency (the general idea, not a number) + +**Speed**: +A connection's median lateness over a minute. + **Verified**: Said of a symbol whose book gives the exchange's book hash after every change. It stops being verified when a change fits on no connection in time. diff --git a/LATENCY.md b/LATENCY.md new file mode 100644 index 0000000..0116688 --- /dev/null +++ b/LATENCY.md @@ -0,0 +1,211 @@ +# Latency experiments + +Extension 4 of the challenge: ways to get the exchanges' changes sooner, +tried on Polymarket and Kalshi. Every trick the brief names (connecting +first after maintenance, spamming connections until a fast one comes, +changing the TCP source port) rests on one question: **is a connection that +is fast now still fast later?** If its speed is noise from message to +message, no choice of connection pays. That is what was measured, from a +server in Switzerland, on 2026-11-12. + +## Method + +`engineering-challenge latency EXCHANGE OUT.csv` (`src/latency.rs`) opens +16 connections to the same 100 busy markets, spread evenly over the +websocket host's addresses, and writes one line per copy of a change: which +connection delivered it, from which address, and when (stamped right after +the socket read, as the collector does). `scripts/latency/analyse.py` +matches the copies of each change across connections. + +- **Lateness**: how long after a change's first copy a connection + delivered it. It compares receive times on one machine, so it needs no + clock. A connection's **speed** is its median lateness over a minute. +- **Lag**: receive time minus the exchange's timestamp. It carries the + local clock's error, so the probe asks `time.cloudflare.com` (SNTP) for + the offset at the start and end of each run: −0.25 to +0.69 ms, with + 3 to 4 ms round trips. The clock is not what limits the figures below. +- **Markets**: on Polymarket the 100 most traded over 24 hours (Gamma's + `volume24hr`), 200 tokens. Kalshi's demo reports no traded amount, so + the markets that changed most in a minute are kept: in the first run + among the listing's first 30,000 open ones, in the second among every + open one (267,691), after the listing's order had moved and the first + 30,000 held one market that changed. +- **Runs**: 10 minutes each, alternated (Polymarket, Kalshi, Polymarket, + Kalshi), the server's load between 0.4 and 2.7 on 12 cores before each. + Kalshi is its demo: no production key. The demo's servers are in AWS + us-west-2 (Oregon) and production's in us-east-2 (Ohio) (AWS's published + ranges), so how far the demo's figures hold for production is unknown. +- **Agreed before the runs**: the collector changes only if, on an exchange, + speed persists (rank correlation of the connections' speeds from one + minute to the next above about 0.5, read on the run's mean) and keeping + the 2 fastest connections + of the first minute lowers the p99 of the pair's lateness (the sooner of + its two copies, as first one wins takes it) by at least 20% against a + pair at random, in both runs. + +## Does a connection's speed persist? + +| Run | Changes | Rank correlation, minute to minute (mean) | Speeds in a minute | +|---|---|---|---| +| Polymarket 1 | 450,096 | 0.91 to 0.99 (0.96) | 0.2 to 2.5 ms | +| Polymarket 2 | 541,933 | 0.96 to 1.00 (0.98) | 0.04 to 3.3 ms | +| Kalshi demo 1 | 540,574 | 0.47 to 0.86 (0.66) | 0 to 177 ms | +| Kalshi demo 2 | 600,046 | 0.67 to 0.91 (0.78) | 0.04 to 136 ms | + +**On Polymarket, yes, almost perfectly.** A connection keeps its place for +the whole run: in the second run connection 6 was 0.04 to 0.1 ms behind the +first copy in 7 of its 9 minutes, connection 3 about 3 ms in all 9. But the +spread is a few milliseconds, and the slow moments are shared: the p99 of +lateness is 25 to 54 ms on every address alike, and the 4 fastest +connections of the first minute had a p99 barely below the others' after +it (32 against 43 ms in the first run, 21 against 22 in the second), bursts +that delay all 16 at once. Speed seems to be decided when a connection opens (four +connections to each address, fast and slow ones on all four), and it can +change mid-way: connection 11 of the first run went from 0.35 to 1 ms in +its fifth minute. + +**On Kalshi's demo, partly.** Connections differ by tens of milliseconds +up to 177, and the ranks hold less well (0.66 and 0.78; two minutes of the +first run fell to 0.47 and 0.49, so the threshold is met on the mean, not +in every minute). Some connections settle at +0 for minutes (12 from its third minute in the first run): they became the +one that delivers first. + +## Racing: keep the fastest + +Choosing after the fact is what opening many connections and keeping the +fast ones would do, provided speed is set when a connection opens; the +persistence above says it largely is. The 2 fastest of the first minute, +against a pair at random (all 120 pairs, averaged), over the minutes +after (8, and 9 in Kalshi's second run: the first run's stop was timed from +its start, so its listing and count left 9 min 52 s of probing; the +second's came 10 minutes after its count, 10 min 2 s): + +| Run | Kept pair, median / p99 | Pair at random, median / p99 | p99 lower by | +|---|---|---|---| +| Polymarket 1 | 0.20 / 8.79 ms | 0.60 / 12.43 ms | 29% | +| Polymarket 2 | 0.00 / 13.96 ms | 0.95 / 13.38 ms | −4% | +| Kalshi demo 1 | 20.15 / 233.59 ms | 28.91 / 324.56 ms | 28% | +| Kalshi demo 2 | 0.16 / 164.25 ms | 9.24 / 283.20 ms | 42% | + +No change was missed by any pair. **Polymarket: racing works at the median +and not in the tail.** The kept pair is ahead of a random one by under a +millisecond every time, but the p99 gain vanished in the second run: the +tail is the bursts every connection shares. The threshold is not met, and +the collector stays as it is: two connections per market, chosen without +regard to speed. **Kalshi's demo: racing works in both.** The p99 falls +by 28% and 42%, the median from 29 to 20 ms and from 9 to 0.2 ms: there, +a connection's speed is worth choosing, and the threshold is met. Whether +production behaves alike is unknown. + +## Does the address matter? + +Polymarket's websocket host resolves to 4 CloudFront addresses (and 4 IPv6 +ones, unreachable from this server), Kalshi demo's to 2. Lateness by +address, all copies of a run: + +| Run | Medians by address | p99s by address | +|---|---|---| +| Polymarket 1 | 0.90 to 1.04 ms | 33 to 54 ms | +| Polymarket 2 | 1.08 to 1.98 ms | 25 to 36 ms | +| Kalshi demo 1 | 40.6 and 45.2 ms | 503 and 548 ms | +| Kalshi demo 2 | 16.5 and 23.9 ms | 406 and 418 ms | + +The address matters much less than the connection. On Polymarket, +`143.204.55.29` had the lowest median in both runs, by 0.1 and 0.5 ms, but +every address carried both fast and slow connections, and the spread inside +an address (0.3 to 2.5 ms in the first run) is larger than between them. +On Kalshi's demo the faster of the two addresses changed between runs. +Whatever sets a connection's speed is behind the address, assumed to be +the CDN's or the load balancer's backend, chosen per connection. The collector connects only to the first address its lookup +returns (`src/core/socket.rs`); spreading over all of them would change +nothing measured here. + +## Lag: how far behind the exchange + +The first copy's lag, corrected by the clock's offset: + +| Run | Median | p99 | +|---|---|---| +| Polymarket 1 | 13.70 ms | 20.21 ms | +| Polymarket 2 | 14.55 ms | 25.37 ms | +| Kalshi demo 1 | 94.03 ms | 285.20 ms | +| Kalshi demo 2 | 106.31 ms | 382.30 ms | + +These include the exchange's own time between stamping a change and sending +it, which can't be told apart from the network from here. Kalshi's demo in +Oregon is about 80 ms further than Polymarket, which is one hop to a +CloudFront edge (where Polymarket's own servers are was not checked). On +Polymarket that is far above what racing wins (under a millisecond). On +Kalshi's demo racing wins more in the tail (90 and 119 ms at p99, as much +as the median lag), but at the median it wins 9 ms, a tenth of the lag. + +## Not tried, and why + +- **First connection after maintenance.** Kalshi's docs announce + maintenance on Thursdays from 3:00 to 5:00 AM ET (08:00 to 10:00 UTC in + winter); the sprint ran after one. The test: run the probe at the + window's end and an hour later, and compare the speeds of the connections + opened first. Polymarket announces no window. +- **The TCP source port.** It can't be chosen from here: a NAT between the + server and the internet rewrites every connection's port (local 41000 + left as 36996, and as 37086 a minute later). Testing it needs a host with + a public address: open connections from fixed ports, reconnect from the + same ones, and see whether each port's speed returns. Given that speed + does not follow the address, a load balancer hashing on the port is the + likeliest reason it could. +- **Location.** Assumed the largest gain by far: lag is 14 ms to Polymarket + and 94 ms to Kalshi's demo from Switzerland, against a millisecond won by + racing. A server in the exchange's AWS region (us-east-2 for Kalshi + production) would remove most of the lag; not tried, it needs a second + machine. +- **IPv6.** This server has no IPv6 route. From a host with one, the + probe would keep Polymarket's 4 IPv6 addresses in its round-robin (it + leaves them out) and the address table would say whether they differ. +- **Kernel tuning** (busy polling, receive buffers, pinning threads). The + receive time is stamped at the read, the machine's load was low, and + nothing above points at the kernel. The test: the same probe run with + `SO_BUSY_POLL` set and each connection's thread pinned, alternated with + runs without, comparing the first copies' lag on an idle machine. +- **Our own added latency**, from the read to the output write: not + measured. It doesn't change when a change is stamped. The test: stamp + each output again as it is written, and report the difference. +- **Kalshi production**: no key. The probe runs on it as `latency kalshi` + with production's two key files. + +## Verdict + +- **Connections differ, and keep their speed**: strongly on Polymarket + (rank correlation 0.96 and 0.98), less on Kalshi's demo (0.66 and 0.78). + Nothing we could choose at connect sets it (the address doesn't): it is + drawn per connection, mostly when it opens, though it can shift mid-way + (Kalshi's connection 4 went from 75 to 3 ms over its second run). +- **On Polymarket it is worth under a millisecond.** Keeping the fastest + pair wins at the median, not in the tail, where bursts delay every + connection alike. The collector keeps its connections as they are. +- **On Kalshi's demo it is worth tens of milliseconds**, in the tail too + (p99 28% and 42% lower with the fastest pair kept). This met the + threshold agreed before the runs, so the collector now recycles a Kalshi + connection that stays slow for 3 minutes running, ahead of its hourly + turn: its replacement's speed is drawn afresh. Each duplicate tells how + late its connection was against the other's copy (`src/kalshi/feed.rs`); + once a minute the orchestrator ranks the connections by their mean + lateness, and one is slow if it is in the slowest quarter and more than + twice the mean of all (`src/core/orchestrator.rs`). The mean, not the + median: with two copies of each change, a connection first more than + half the time has a median of 0. Without the factor of two, a slowest + quarter always exists: a first version, by median alone, recycled 6 + connections in 20 minutes, one a minute, connections no slower than the + rest. **In 20 minutes of the collector on the demo (270,000 markets on + 120 connections, load 5.7 at the start), no connection was slow by that + rule**, so none was recycled for it; the hourly recycle went on (33), and + the output reconciled (25 mismatches, each a logged loss). The probe's 16 + connections, on the 100 busiest markets, differed far more than the + collector's; assumed to be because each of the collector's shares its + markets with many others and is judged against all their copies, which + evens the means out. So the mechanism is tested but was not seen + to fire live, and its gain on the collector was not measured; how the + demo's figures carry to production, which runs elsewhere, is unknown. +- **Distance weighs more**: 14 ms of lag to Polymarket, about 100 ms to + Kalshi's demo, against a median gain from racing under 1 ms and of 9 ms. + Colocation is the lever; it was out of reach here. diff --git a/docs/notes.md b/docs/notes.md index a3f2eb6..710d341 100644 --- a/docs/notes.md +++ b/docs/notes.md @@ -313,6 +313,24 @@ is the demo's. (2026-11-11, 19:29 UTC). - **Maintenance (docs):** Thursdays 3:00 to 5:00 AM ET; connections drop. +## Latency, from the server (observed 2026-11-12) + +- **The server's clock is synced**: SNTP against `time.cloudflare.com` read + −0.25 to +0.69 ms, 3 to 4 ms round trips (sprint 07's probe). +- **What the exchange sees isn't ours to choose**: a NAT rewrites every + connection's source port (not even stable for one local port), and there + is no IPv6 route; Polymarket's websocket resolves to 4 IPv6 addresses + beside its 4 IPv4 ones (CloudFront). +- **Where the hosts are** (AWS's published ranges): Kalshi production's + websocket in us-east-2, the demo's two addresses in us-west-2; lag about + 14 ms to Polymarket, about 100 ms to Kalshi's demo. +- **Polymarket repeats a change on one connection**, rarely: 22 of 29,922 + changes came twice on every connection in a minute. +- **Kalshi's listing order moves**: its first 30,000 open markets held the + 98 busiest at 11:12 and none of them at 11:50. Pick by counting every + market, not by the listing's first pages. The demo reports + `volume_24h_fp` 0.00 (267,691 open markets, a full listing in 75 s). + ## Clock accuracy bounds time accuracy `received_at` comes from the local clock, so it carries the clock's error. On 2026-11-04 the dev machine ran ~30 ms slow (`sntp`: +30.6 ms ± 13 ms), which diff --git a/docs/project.md b/docs/project.md index 8c2bb36..ef6cb99 100644 --- a/docs/project.md +++ b/docs/project.md @@ -150,7 +150,8 @@ schema debate). As settled in sprints 04 and 05: └──────┬───▲───┘ ──── recycle ────────▶ connections Add / Move / Remove (a market and │ │ load reports, each minute its connections), to its thread │ │ (bytes/s per market); markets - │ │ quiet twice on a connection + │ │ quiet twice on a connection; + │ │ lateness per connection (Kalshi) ┌──────▼───┴───┐ connection 0..N ──events───▶ │ book threads │ ──Snapshot | Delta──▶ writer (read, stamp, parse, route) │ 0..15 │ @@ -160,7 +161,9 @@ schema debate). As settled in sprints 04 and 05: - The **orchestrator** owns what is tracked (each market's two connections, the timers) and decides: placing, moving, splitting, and recycling one - connection at a time, about hourly. It never blocks on HTTP. + connection at a time, about hourly, and sooner a Kalshi connection whose + mean lateness stays above twice everyone's (sprint 07, LATENCY.md). It + never blocks on HTTP. - The two **fetchers** only page the exchange's market listing on request and stream what they read. - A **connection** reads, stamps the receive time, parses, and routes each diff --git a/docs/roadmap.md b/docs/roadmap.md index 7a8b185..1656330 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -77,12 +77,19 @@ abstraction is *extracted* once a second exchange exists, not designed up front. higher than sprint 05's; Kalshi on its demo (about 245,000 markets on 110 connections), production not run for lack of a key (sprint 06's Outcome). -7. **Extension 4 — latency experiments + writeup** — *not started* - Experiment with faster connections (first-after-maintenance, connection - racing, source-port tricks) on both exchanges; document findings in a - markdown file, with the local clock's offset measured alongside (the dev - machine ran ~30 ms slow; see notes.md). +7. **Extension 4 — latency experiments + writeup** — *done (2026-11-12)* + Is a connection that is fast now still fast later? 16 connections to the + same markets, ranked by how long after a change's first copy each + delivers it (no clock needed), with the clock's offset measured by SNTP; + written up in [../LATENCY.md](../LATENCY.md). + Sprint: [../roadmap/07. latency experiments.md](../roadmap/07.%20latency%20experiments.md). *Ends:* experiments documented. + Result: speed persists on both exchanges; keeping the fastest wins under + a millisecond on Polymarket (not in the tail), tens of milliseconds on + Kalshi's demo, where the collector now recycles a connection whose mean + lateness stays above twice everyone's (none was, in 20 live minutes). + The source port can't be chosen here (a NAT rewrites it); first after + maintenance wasn't tried. 8. **Polish + submission** — *not started* Final review, README for the solution (the template's README still says the @@ -138,3 +145,13 @@ abstraction is *extracted* once a second exchange exists, not designed up front. - Telling Kalshi's book at a market's close, and a connection's book merely ahead of the other's stream, from a real loss (sprint 06 logs all three as a loss; the Snapshot is right either way). +- The first connection after Kalshi's maintenance (Thursdays 08:00 to + 10:00 UTC in winter): the probe at the window's end against an hour later + (sprint 07 ran after one). +- The TCP source port's effect on speed, from a host with a public address + (here a NAT rewrites it). +- The slow recycle's gain on Kalshi production, measured on the collector + (sprint 07 measured racing with the probe, on the demo only). +- Judging the connections' speed at the book threads' wall-clock minute, + not on the orchestrator's own timer (a judgment near the crossing sees + partial samples; sprint 07's review). diff --git a/roadmap/07. latency experiments.md b/roadmap/07. latency experiments.md new file mode 100644 index 0000000..6ea0471 --- /dev/null +++ b/roadmap/07. latency experiments.md @@ -0,0 +1,361 @@ +# 07. Latency experiments + +**Status: done (2026-11-12)** + +Extension 4 of the brief: "experimentation into how to [get faster +connections] on various exchanges like Polymarket or Kalshi", written up in a +markdown file. Correctness stays the first priority, so the collector +changes only if the data clears a threshold agreed here. The question under +every trick the brief names (racing connections, the source port, being +first after maintenance) is one: **is a connection that is fast now still +fast later?** If speed is noise per message, no choice of connection pays. +Terms: [../GLOSSARY.md](../GLOSSARY.md) (lateness, lag, speed, duplicate). + +## The probe + +`src/latency.rs`, a subcommand of the collector (`engineering-challenge +latency `), kept in the repo +because `LATENCY.md` cites its numbers. It is not the collector: no book, +no rule, no reconnect. A subcommand, not `src/bin/latency.rs` as first +agreed, because what it reuses (`core`, `polymarket`, `kalshi`, `log!`) +belongs to the collector's binary, not the library; moving them was +rejected as a large diff that buys nothing else. + +- **Markets:** the same 100 on every connection, few enough that our own + load stays out of the measurement. Polymarket: the 100 live markets with + the highest `volume24hr` (Gamma sorts by it: `order=volume24hr&ascending=false`, + checked), all their tokens. Kalshi: the demo reports `volume_24h_fp` 0.00 + (all of the first 1,000 open markets, checked), so the probe counts every + open market's changes for a minute, 30,000 per connection, and keeps the + 100 with the most; it stops if fewer than 10 changed. Production would be + picked the same way, one rule for both Kalshi deployments. Supersedes the + pick by traded amount on Kalshi, because the demo has none; and then the + count over the listing's first 30,000 only, because the listing's order + moves: the 98 busiest of 11:12 were past its first 40,000 at 11:50, and + the second Kalshi run counted 1 market that changed. +- **Connections:** 16 per run, one plain thread each, tungstenite over a + `TcpStream` the probe opens to an address it chooses, round-robin over the + exchange's IPv4 addresses (the workspace has no IPv6 route: the smoke + run's connections to Polymarket's IPv6 ones failed with "Network is + unreachable"; Polymarket's websocket has 4 IPv4 ones, the Kalshi demo's 2, + production's 1, checked 2026-11-12; the collector connects only to the + first, `src/core/socket.rs:184`). It reuses `Exchange::request` (Kalshi's + signed handshake), the `Session`'s subscribe, keepalive and decode + (`src/core/mod.rs:83` to `89`), and the collector's connect with the + address given (split out of `src/core/socket.rs`'s `connect`); a refused + handshake (429) is tried again through `core::retried`, 3 tries 1 s apart. The receive time is stamped + right after the read, as in the collector. A connection that drops is + logged and stays down: a run is 10 minutes. +- **Output:** one CSV line per copy of a change, raw: connection, address, + symbol, the change's identity, receive time (µs), exchange timestamp (ms). + The analysis matches copies across connections; the probe computes + nothing. +- **A change's identity** across connections: on Polymarket the symbol and + the change's book hash, side, price and size (the rule's own copy test, + `Held::key`, `src/polymarket/feed.rs:89`); on Kalshi the market, + `clock`, side, price and delta, plus the tuple's occurrence number on its + connection, since 2 to 4 changes can share a clock, some identical + (`docs/notes.md`). +- **Clock:** an SNTP query to `time.cloudflare.com` at the start and end of + each run, logged; lag is reported raw and corrected. `sntpc` 0.11 + (`sync`, `std`) with `sntpc-net-std` 1.3, tried: offset +0.5 ms, round + trip 3 ms from the server (2026-11-12). + +## The analysis + +`scripts/latency/analyse.py`, standard library only (`statistics.correlation` +with `method="ranked"` is Spearman, Python 3.13 here). From a run's CSV: + +- each copy's lateness, a server's repeat on one connection counted once; + a change a connection never delivered while it was up (from its first + copy until it went down) counts as missed by it; +- each connection's speed per minute, and its missed count; +- **persistence:** the rank correlation of the 16 speeds between each + minute and the next; and the 4 fastest of minute 1 against the other 12 + over the full minutes after it, at the median and p99 of lateness; +- **racing:** the 2 fastest of minute 1 kept, against 2 at random (averaged + over every pair of the 16), over the full minutes after it, at the + median and p99 of lateness, with each pair's missed changes; a run too + thin to compare is said so; +- **by address:** lateness per edge address; +- **lag:** per exchange, raw and corrected by the offset. + +## The runs + +Two 10-minute runs per exchange, alternated (Polymarket, Kalshi, +Polymarket, Kalshi), `/proc/loadavg` recorded before each and the run put +off while the load is high. The second run repeats the first, which the +adoption threshold needs. Raw CSVs and logs stay out of the repo, in +`../runs/latency/`. + +## The writeup + +`LATENCY.md` at the repo root, a page or two: the method (the probe, +lateness against lag, the clock's offset), one section per experiment +(question, a small table per exchange, verdict), and the ideas not tried, +each with why and how one would: first after maintenance, the source port, +location, kernel tuning, IPv6, our own added latency. Kalshi's numbers are +labelled the demo's. What was not measured is written "assumed". + +## Decisions + +- **Lateness, not lag, ranks connections.** Lateness compares copies of one + change on one machine and needs no clock; lag carries the clock's offset + and the exchange's own stamping. Rejected: ranking by lag. +- **Raw CSV, analysis apart.** A new question is asked of a run without + running it again; live runs are what cost time on a shared server. + Rejected: summaries computed by the probe. +- **Racing is computed from the persistence run**, not run as an + open-close loop: choosing after the fact shows the same gain as long as a + connection's speed is set when it connects, which is what persistence + tests. Rejected: a separate racing run. +- **The source port is not tested.** At least one NAT between the workspace + and the internet rewrites it: local ports 40123, 40124 and 41000 to 41003 + left as 50990, 53878, 36996, 48950, 51760, 60434, and 41000 again a minute + later as 37086 (`ifconfig.me`, 2026-11-12). The probe binds no port. + Supersedes the scope's "same (address, port) pairs" run. +- **Adoption threshold**, agreed before the data: the collector changes on + an exchange only if speed persists (rank correlation above about 0.5 + across the run) and the 2 picked connections' p99 lateness is at least + 20% below the random pair's, in both runs. The change would then be to + recycle a connection whose speed stayed among the slowest for several + minutes (the stretch goal). Otherwise the writeup says the threshold was + not met. Met on Kalshi's demo (28% and 42%), not on Polymarket (29%, then + -4%). +- **Recycling the slow connections (Kalshi)**, agreed after the runs, for + showing the loop from data to code and as one more self-correcting + mechanism, knowing the evidence is the demo's: Kalshi's rule keeps the + first copy's receive time and connection of its last 128 positions per + symbol, forgotten when a fresh book replaces ours; a duplicate records + its lateness for its connection, and 0 for the first copy's (the later + stamp is the late one, whichever reached the book thread first); a + third copy, while a market moves, is not measured. Each book thread reports the samples at the minute + (`Report::Lateness`); Polymarket's rule reports none, so nothing changes + there. The orchestrator, once a minute, ranks the connections with at + least 100 samples and up for 10 minutes by their mean lateness; those + of the slowest quarter whose mean is more than twice the mean of every + judged copy count a slow minute, the rest start over; one slow 3 + minutes in a row is recycled at once, ahead of the hourly order, at most + one a minute and under the hourly recycle's rule (one at a time, every + connection up, the last up an interval ago). Rejected: building it in + the core for both exchanges, since Polymarket's p99 doesn't follow + speed. The mean, not the median, and the factor of two, supersede the + first design (by median, the slowest quarter alone), because of the + review: a slowest quarter always exists, so equal connections were + recycled about once a minute; and with two copies of each change, a + connection first more than half the time has a median of 0, so a median + says how often it comes second, not how late, and twice the median of + medians is about 0. The mean holds both, and the tail where Kalshi's + gain was. The probe's speed (the median, GLOSSARY.md) is unchanged. +- **Kalshi on the demo.** No production key; the code is the same, only + whether the demo behaves like production is unknown. +- **`lags` becomes `lateness`** in Polymarket's rule (Kalshi's measures + none), with the log text "second copies … late" read as "duplicates' + lateness … at the median"; nothing parses that line (`src/cause.rs`, + `src/reconcile.rs`). +- **Our own added latency** (receive time to output write) is not measured: + the receive time is stamped before any queue, and the brief's extension + is about the connection. + +## Tasks + +- [x] Rename `lags` to `lateness` in `src/polymarket/feed.rs`, comments + and log text included (Kalshi's rule measures none); the rules' tests + pass unchanged. +- [x] The probe's change identities, for both exchanges. +- [x] Test: a Polymarket change on two connections gets the same identity; + one differing only in its hash does not. +- [x] Test: two identical Kalshi changes under one clock on one connection + get two identities, the same two as on another connection. +- [x] The probe's market pick (Polymarket by `volume24hr`, Kalshi by a + minute's count) and its connections (address round-robin, request, + subscribe, keepalive, stamping, CSV), with the SNTP offset at start + and end. +- [x] A 1-minute smoke run of the probe on each exchange: 16 connections + up, every connection's lines in the CSV, the offset logged. +- [x] `scripts/latency/analyse.py`: lateness, missed, speed per minute, + persistence, racing, by address, lag. +- [x] Test (`scripts/latency/test_analyse.py`, from `scripts/ci.sh`): on a + small CSV worked out by hand, the first copy's lateness is 0, a change + one connection lacks counts as missed by it, and the speeds, ranks + and racing figures are the hand-computed ones. +- [x] The four runs, alternated, load and offset recorded. +- [x] `LATENCY.md`. +- [x] Stretch (threshold met on Kalshi): Kalshi's rule records each + duplicate's lateness and the first copy's 0, reported at the minute. +- [x] Test: a duplicate's lateness is reported for its connection and 0 + for the first copy's; a change with one copy reports nothing. +- [x] The tracker's judgment and the orchestrator's recycle of a slow + connection, logged. +- [x] Test: slowest for 3 minutes is recycled; slowest 2 minutes then not + starts over; a connection up less than 10 minutes or with too few + samples is not judged; never one while another recycle is under way. +- [x] A 20-minute Kalshi demo run (a connection is judged from its 10th + minute, recycled after 3 slow ones), reconciled, its log showing the slow + recycles. +- [x] `CLAUDE.md` (the probe, the new terms, the NAT trap) and + `docs/roadmap.md` brought up to date. + +From the review of `d722446`: + +- [x] A slow recycle waits, as the hourly one does, an interval after the + last connection came up (its opening books), with its test. +- [x] A slow connection is ranked by its mean lateness and recycled only + if that mean is more than twice the mean of every judged copy: equal + connections are left alone, and the hourly recycle isn't starved. + Its test (equal connections, none recycled). +- [x] Kalshi's lateness: a copy stamped before the one applied first + swaps the two (no negative lateness); the remembered positions are + cleared when a fresh book replaces ours. Tests for both. +- [x] Tests: the tracker's mean against samples that differ (a median of + 0, a high mean); two book + threads' samples merged; the 128-position cap; judging refused at + 599 s. +- [x] Tests: Kalshi's identity on two interleaved markets; Polymarket's + told apart by size. +- [x] The stop below 10 counted markets moved into `busiest`, with its test. +- [x] The probe reuses the Kalshi listing's paging and agent + (`src/kalshi/listing.rs`, made `pub(crate)`). +- [x] `RECENT`'s comment names its upgrade path; renames: `counting` (it + means counting is over), `SLOW_SHARE`'s doc, `struct Copy`, + `Tracker::lateness`. +- [x] The analysis: no traceback on thin runs (4 or fewer connections in + the first minute, a kept pair silent after it, one connection); + tests for the p99 against the maximum (200 values), `report()` on + the hand-worked run (racing's gain and the Missed column), racing's + missed counts, and an offset note with no value. +- [x] `LATENCY.md`: the two Kalshi runs' picks; persistence read on the + mean (minutes at 0.47 and 0.49 said); the slowest 177 ms; the + second run's length explained right (the first's stop counted from + start: 9 min 52 s of probing, the second 10 min 2 s); no "two orders + of magnitude" for Kalshi's p99; "behind the address" marked assumed; + the collector run's load 7.3; how one would try IPv6, kernel tuning + and our own added latency. +- [x] This file: minutes 2 to the end, a missed count (while up) rather + than a share, and the third copy unmeasured, in the Design. + +Each test is seen to fail once (the code it protects removed, then put +back). + +## Review of `d722446` (2026-11-12) + +Five reviewers (spec, bugs, tests, smells, security); 2 findings dropped +(the tests were seen to fail; the memory bound is 128 positions a +symbol). Every one below was decided "fix now" (the tasks above) unless +said otherwise. + +- **Bugs:** the slow recycle skipped the hourly one's settling wait; it + never stops (a slowest quarter always exists) and can starve the hourly + one; the analysis crashed on thin runs; lateness could be negative or + measured against a stale position. Later: the judgment runs on the + orchestrator's timer, not the book threads' minute (partial samples make + it more conservative, not unsafe). No: the probe ignores SIGINT during + the listing scan and never notes a silent connection (a tool run by + hand; missed counts were 0 to 5). +- **Tests:** 9 gaps verified one mutant at a time (p99 and the tracker's + median against the maximum, `report()`, samples merged, the cap, racing's + missed, the offset note, Kalshi's identity across markets, Polymarket's + size), the stop below 10 and the 599 s boundary. No: the CSV contract + across the two languages, the IPv4 filter and round-robin, the tie-break, + the loop's wiring (live runs show them). +- **Smells:** the probe's copies of the listing's paging and agent; the + `RECENT` comment; four names. No: `Rule::take_lateness` (a hook on a + trait with two implementations), the analysis's primitive tuples and + repeats, the lateness and median in several places, the probe's match + on the exchange, `latency.rs`'s size. +- **Security:** no vulnerability. No: the docs name the country and the + NAT (part of the findings), unquoted CSV fields (a TLS-authenticated + source), unauthenticated SNTP (only the corrected lag). +- **Spec:** `LATENCY.md`'s figures and claims, the two picks, the mean, + three "how"s, this file's Design. No: the CSV's notes, ruff in CI, the + slow count in the minute line, the listing retry (each serves what was + asked). + +## Not in this sprint + +Polish and the README (M8); the roadmap's possible features; kernel +tuning; anything needing a second machine, another region or a public +address; the first connection after Kalshi's maintenance (next window +Thursday 2026-11-19, 08:00 to 10:00 UTC), unless the sprint is still open +then. + +## Done when + +- The probe and the analysis are on the branch, their tests seen to fail + once. +- Both exchanges' two runs are done and analysed, load and offset recorded. +- `LATENCY.md` gives a verdict per experiment and per exchange, Kalshi + labelled the demo. +- The stretch goal is built and tested if the threshold is met; otherwise + the writeup says it wasn't. +- `scripts/ci.sh` is green. + +## Outcome + +Extension 4 is answered and written up in `LATENCY.md`. `engineering-challenge +latency EXCHANGE OUT.csv` opens 16 connections to the same 100 busy markets +and writes every copy of every change; `scripts/latency/analyse.py` turns a +run into speeds, persistence, racing, lateness by address and lag. Four +10-minute runs, alternated, the clock's offset under 0.7 ms throughout: + +| | Persistence (mean) | Racing, p99 lower by | Lag, median | +|---|---|---|---| +| Polymarket | 0.96, 0.98 | 29%, then −4% | 14 ms | +| Kalshi demo | 0.66, 0.78 | 28%, 42% | 94 to 106 ms | + +On Polymarket speed persists but is worth under a millisecond, at the +median only: the tail is bursts every connection shares. On Kalshi's demo +it met the agreed threshold, so the collector now recycles a Kalshi +connection whose mean lateness stays, 3 minutes running, in the slowest +quarter and above twice everyone's (`src/kalshi/feed.rs` measures, the +orchestrator judges). In 20 live minutes on the demo (120 connections) no +connection was slow by that rule; the output reconciled (25 mismatches, +each a logged loss). + +**Diverged from the plan**, each written into this file when it happened: +the probe is a subcommand, not `src/bin/latency.rs` (what it reuses belongs +to the binary); IPv4 addresses only (no IPv6 route); no source-port run (a +NAT rewrites the port); Kalshi's pick by a minute's count, first over the +listing's first 30,000 markets, then over every open one, after the +listing's order moved and the second run counted one market; the slow +recycle by mean lateness and only past twice the mean, after the review +showed that by median, a slowest quarter alone recycled equal connections +about once a minute. + +**Left out**, and where it went: first after maintenance, the source port +from a public host, the slow recycle's gain on Kalshi production, and +judging at the book threads' minute (`docs/roadmap.md`, possible +features); IPv6, kernel tuning and our own added latency (`LATENCY.md`, +not tried, each with how). Kalshi production: no key, as in sprint 06. + +**Assumed**, not measured: that what sets a connection's speed is the +backend behind the address; that the collector's connections differ less +than the probe's because each is judged against many partners' copies; +that location would gain the most. + +**Reviewed** at `d722446`; the 11 fixes after it are the review's own +tasks, each with a test seen to fail, and had no second review. + +### Retrospective + +- **Went well:** one question under the brief's three tricks made the + experiments few and the writeup short. A threshold agreed before the + data kept the verdict honest: Polymarket's 29% looked like a win until + the repeat run. Raw CSVs let every new question (the pair's median, the + missed counts) be asked of runs already made. Mutating each test's code + one at a time found holes in my test data four times. +- **Went wrong:** the Kalshi pick depended on the listing's order, which + cost a 10-minute run; a pick should not rest on an order the exchange + doesn't promise. I designed the collector's slow rule on the probe's + median without asking what the median means with two copies of each + change: the review and then the fix found it, and the final rule never + fired live, so its value on the demo is unshown. Three chained shell + commands were refused whole in the worktree session, and once that left + a commit amended without its CI run; caught and redone. +- **Tested enough?** The probe's identities, the analysis and the slow + rule are, by removal. Not: the probe's network paths and the + orchestrator loop's wiring (live runs only), and the slow recycle firing + live. +- **Next time:** before reusing a statistic in a new place, work out its + value on that place's data in one line. A live check of a mechanism + should show it acting, or say plainly that it didn't. diff --git a/scripts/ci.sh b/scripts/ci.sh index f8fe321..8740764 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -8,3 +8,8 @@ cd "$(dirname "$0")/.." cargo fmt --check cargo clippy --all-targets -- -D warnings cargo test + +# The latency analysis (scripts/latency/): ruff pinned, run through uvx. +uvx ruff@0.15.0 format --check scripts/latency +uvx ruff@0.15.0 check scripts/latency +python3 -m unittest discover -s scripts/latency diff --git a/scripts/latency/analyse.py b/scripts/latency/analyse.py new file mode 100644 index 0000000..6488755 --- /dev/null +++ b/scripts/latency/analyse.py @@ -0,0 +1,258 @@ +"""The latency probe's CSV, analysed (roadmap/07): lateness, missed changes, +each connection's speed per minute, whether speed persists, what racing +would gain, lateness by address, and lag. Standard library only. + +Usage: python3 scripts/latency/analyse.py RUN.csv + +Terms (GLOSSARY.md): a copy's lateness is how long after the change's first +copy it came; a connection's speed is its median lateness over a minute; +lag is a receive time minus the exchange's timestamp. +""" + +import csv +import itertools +import statistics +import sys +from pathlib import Path + +MINUTE_US = 60_000_000 +# The fastest of the first minute, against the others (persistence). +TOP = 4 +# Connections a symbol has in the collector: first one wins (ADR 0003). +PAIR = 2 + + +class Run: + """A run's changes, each with every connection's first copy of it.""" + + def __init__(self, lines): + self.connections = {} # connection -> address + self.down = {} # connection -> when it went down, µs + self.offsets = [] # the clock's offsets, µs + # (symbol, identity) -> [exchange_ms, {connection: received_us}] + self.changes = {} + rows = [] + for line in lines: + if line.startswith("#"): + self._note(line[1:].strip()) + else: + rows.append(line) + for row in csv.DictReader(rows): + connection = int(row["connection"]) + self.connections[connection] = row["address"] + key = (row["symbol"], row["identity"]) + change = self.changes.setdefault(key, [int(row["exchange_ms"]), {}]) + # A server's repeat on one connection: its first copy counts. + change[1].setdefault(connection, int(row["received_us"])) + self.up_from = {} # connection -> its first copy, µs + for _, copies in self.changes.values(): + for connection, received in copies.items(): + self.up_from[connection] = min(self.up_from.get(connection, received), received) + + def _note(self, note): + fields = note.split(",") + if fields[0] == "down": + self.down[int(fields[1])] = int(fields[2]) + elif fields[0] == "offset" and fields[2]: + self.offsets.append(int(fields[2])) + + def first(self, change): + return min(change[1].values()) + + def minutes(self): + """Each full minute's changes, from the run's first change.""" + firsts = [(self.first(change), change) for change in self.changes.values()] + if not firsts: + return [] + start = min(first for first, _ in firsts) + last = max(first for first, _ in firsts) + count = (last - start) // MINUTE_US # the last, partial, left out + minutes = [[] for _ in range(count)] + for first, change in firsts: + minute = (first - start) // MINUTE_US + if minute < count: + minutes[minute].append(change) + return minutes + + def up(self, connection, at): + """From its first copy (its subscribe done) until it went down.""" + never = float("inf") + return self.up_from.get(connection, never) <= at < self.down.get(connection, never) + + def lateness(self, changes, connections): + """The lateness of the first copy among `connections`, change by + change, behind the first of all; and how many changes none of them + delivered while up.""" + lateness, missed = [], 0 + for change in changes: + first = self.first(change) + copies = [change[1][c] for c in connections if c in change[1]] + if copies: + lateness.append(min(copies) - first) + elif any(self.up(c, first) for c in connections): + missed += 1 + return lateness, missed + + +def quantile(values, q): + """As the collector's log computes it: the value at index n * q.""" + ordered = sorted(values) + return ordered[min(int(len(ordered) * q), len(ordered) - 1)] + + +def speeds(run, minute): + """Each connection's speed over a minute's changes, µs.""" + result = {} + for connection in run.connections: + lateness, _ = run.lateness(minute, [connection]) + if lateness: + result[connection] = statistics.median(lateness) + return result + + +def ranked(speeds_by_connection): + return sorted(speeds_by_connection, key=lambda c: (speeds_by_connection[c], c)) + + +def persistence(run, minutes): + """The rank correlation of the connections' speeds between each minute + and the next.""" + per_minute = [speeds(run, minute) for minute in minutes] + correlations = [] + for this, following in itertools.pairwise(per_minute): + both = sorted(set(this) & set(following)) + if len(both) > 2: + correlations.append( + statistics.correlation( + [this[c] for c in both], + [following[c] for c in both], + method="ranked", + ) + ) + return correlations + + +def summary(lateness): + if not lateness: + return None + return statistics.median(lateness), quantile(lateness, 0.99) + + +def top_against_rest(run, minutes, top=TOP): + """The `top` fastest of the first minute against the others, over the + minutes after it: (median, p99) of their copies' lateness each.""" + order = ranked(speeds(run, minutes[0])) + top, rest = order[:top], order[top:] + later = [change for minute in minutes[1:] for change in minute] + + def copies(connections): + lateness = [] + for connection in connections: + lateness += run.lateness(later, [connection])[0] + return summary(lateness) + + return copies(top), copies(rest) + + +def racing(run, minutes, pair=PAIR): + """The `pair` fastest of the first minute kept, against a pair at + random (every pair, averaged), over the minutes after it: the (median, + p99) of the pair's lateness (first one wins: the sooner of its two + copies), and its missed changes. None if too few connections deliver + after the first minute to compare.""" + order = ranked(speeds(run, minutes[0])) + later = [change for minute in minutes[1:] for change in minute] + picked_lateness, picked_missed = run.lateness(later, order[:pair]) + pairs = [run.lateness(later, list(two)) for two in itertools.combinations(order, pair)] + random = [s for s in (summary(lateness) for lateness, _ in pairs) if s] + picked = summary(picked_lateness) + if len(order) < pair or picked is None or not random: + return None + return ( + picked, + picked_missed, + (statistics.mean(m for m, _ in random), statistics.mean(p for _, p in random)), + statistics.mean(missed for _, missed in pairs), + ) + + +def by_address(run, changes): + """(median, p99) of lateness by address, every copy counted.""" + addresses = {} + for connection, address in run.connections.items(): + addresses.setdefault(address, []).extend(run.lateness(changes, [connection])[0]) + return {address: summary(lateness) for address, lateness in sorted(addresses.items())} + + +def lag(run, changes): + """(median, p99) of the first copies' lag, µs, raw and corrected by the + clock's mean offset (positive: the local clock is behind).""" + raw = [run.first(change) - change[0] * 1000 for change in changes] + offset = statistics.mean(run.offsets) if run.offsets else 0 + return summary(raw), summary([value + offset for value in raw]), offset + + +def ms(micros): + return f"{micros / 1000:.2f}" + + +def report(run): + minutes = run.minutes() + changes = [change for minute in minutes for change in minute] + if not changes: + return f"{len(run.connections)} connections, no change over a full minute" + lines = [ + f"{len(run.connections)} connections, {len(changes)} changes " + f"over {len(minutes)} full minutes", + "", + "| Connection | Address | Speed per minute (ms) | Missed |", + "|---|---|---|---|", + ] + per_minute = [speeds(run, minute) for minute in minutes] + for connection, address in sorted(run.connections.items()): + row = " ".join(ms(s[connection]) if connection in s else "-" for s in per_minute) + _, missed = run.lateness(changes, [connection]) + lines.append(f"| {connection} | {address} | {row} | {missed} |") + if len(minutes) >= 2: + correlations = persistence(run, minutes) + if correlations: + each = " ".join(f"{c:.2f}" for c in correlations) + mean = statistics.mean(correlations) + lines += ["", f"Persistence: rank correlation minute to minute {each}, mean {mean:.2f}"] + top, rest = top_against_rest(run, minutes) + if top and rest: + lines.append( + f"The {TOP} fastest of minute 1 after it: {ms(top[0])} ms median, " + f"{ms(top[1])} ms p99; the others: {ms(rest[0])} ms, {ms(rest[1])} ms" + ) + else: + lines.append(f"The {TOP} fastest of minute 1 against the others: too few connections") + raced = racing(run, minutes) + if raced: + picked, picked_missed, random, random_missed = raced + gain = (random[1] - picked[1]) / random[1] if random[1] else 0 + lines.append( + f"Racing: the {PAIR} fastest of minute 1, {ms(picked[0])} ms median, " + f"{ms(picked[1])} ms p99, {picked_missed} missed; a pair at random, " + f"{ms(random[0])} ms, {ms(random[1])} ms, {random_missed:.1f} missed; " + f"p99 {gain:.0%} lower" + ) + else: + lines.append("Racing: too few connections deliver after minute 1") + lines += ["", "| Address | Median (ms) | p99 (ms) |", "|---|---|---|"] + for address, lateness in by_address(run, changes).items(): + cells = f"{ms(lateness[0])} | {ms(lateness[1])}" if lateness else "- | -" + lines.append(f"| {address} | {cells} |") + raw, corrected, offset = lag(run, changes) + lines += [ + "", + f"Lag of the first copies: raw {ms(raw[0])} ms median, {ms(raw[1])} ms p99; " + f"corrected by the clock's offset ({ms(offset)} ms): " + f"{ms(corrected[0])} ms, {ms(corrected[1])} ms", + ] + return "\n".join(lines) + + +if __name__ == "__main__": + with Path(sys.argv[1]).open() as file: + print(report(Run(file))) diff --git a/scripts/latency/ruff.toml b/scripts/latency/ruff.toml new file mode 100644 index 0000000..e8113e0 --- /dev/null +++ b/scripts/latency/ruff.toml @@ -0,0 +1,15 @@ +# The analysis scripts' lint: every rule, less those that don't fit two +# standalone scripts. +line-length = 100 + +[lint] +select = ["ALL"] +ignore = [ + "D", # docstrings: written where they say something, not on every function + "ANN", # type annotations: two scripts, no type checker run + "T201", # print: the report is printed + "PLR2004", # magic values: the tests compare hand-computed numbers + "PT009", # unittest's assertEqual: the tests use unittest, no pytest + "INP001", # no package: scripts run directly + "COM812", # trailing commas: conflicts with the formatter +] diff --git a/scripts/latency/test_analyse.py b/scripts/latency/test_analyse.py new file mode 100644 index 0000000..944b338 --- /dev/null +++ b/scripts/latency/test_analyse.py @@ -0,0 +1,203 @@ +"""analyse.py on runs small enough to work out by hand. + +Run from `scripts/ci.sh`: `python3 -m unittest discover -s scripts/latency`. +""" + +import unittest + +import analyse + +S = 1_000_000_000_000 # the run's first change, µs +MIN = analyse.MINUTE_US +ADDRESS = {0: "A", 1: "B", 2: "A"} + + +def csv(copies, notes=()): + """`copies`: (connection, change, received_us, exchange_ms).""" + lines = ["connection,address,symbol,identity,received_us,exchange_ms"] + lines += [f"# {note}" for note in notes] + for connection, change, received, exchange_ms in copies: + lines.append(f"{connection},{ADDRESS[connection]},s,{change},{received},{exchange_ms}") + return [line + "\n" for line in lines] + + +# Minute 1: c1 and c2 (connection 2 misses c2); minute 2: c3 and c4; c5 only +# ends minute 2, its own minute partial. Lateness in µs: +# conn 0 conn 1 conn 2 +# c1 0 1000 3000 +# c2 0 2000 - +# c3 1000 0 4000 +# c4 3000 0 5000 +RUN = csv( + [ + (0, "c1", S, (S - 10_000) // 1000), + (1, "c1", S + 1000, 0), + (2, "c1", S + 3000, 0), + # A server's repeat on connection 1: its first copy counts. + (1, "c1", S + 50_000, 0), + (0, "c2", S + 10_000_000, 0), + (1, "c2", S + 10_002_000, 0), + (1, "c3", S + MIN, 0), + (0, "c3", S + MIN + 1000, 0), + (2, "c3", S + MIN + 4000, 0), + (1, "c4", S + MIN + 10_000_000, 0), + (0, "c4", S + MIN + 10_003_000, 0), + (2, "c4", S + MIN + 10_005_000, 0), + (0, "c5", S + 2 * MIN + 5_000_000, 0), + (1, "c5", S + 2 * MIN + 5_000_000, 0), + (2, "c5", S + 2 * MIN + 5_000_000, 0), + ], + notes=["offset,start,400,3000", "offset,end,600,3000"], +) + + +class Analyse(unittest.TestCase): + def setUp(self): + self.run = analyse.Run(RUN) + self.minutes = self.run.minutes() + + def test_full_minutes_only(self): + self.assertEqual([len(minute) for minute in self.minutes], [2, 2]) + + def test_the_first_copy_is_not_late_and_a_repeat_is_ignored(self): + lateness, _ = self.run.lateness(self.minutes[0], [0]) + self.assertEqual(lateness, [0, 0]) + lateness, _ = self.run.lateness(self.minutes[0], [1]) + self.assertEqual(lateness, [1000, 2000]) + + def test_a_change_a_connection_lacks_is_missed_by_it(self): + changes = self.minutes[0] + self.minutes[1] + self.assertEqual([self.run.lateness(changes, [c])[1] for c in (0, 1, 2)], [0, 0, 1]) + + def test_speeds_and_their_persistence(self): + self.assertEqual(analyse.speeds(self.run, self.minutes[0]), {0: 0, 1: 1500, 2: 3000}) + self.assertEqual(analyse.speeds(self.run, self.minutes[1]), {0: 2000, 1: 0, 2: 4500}) + # Ranks (1, 2, 3) then (2, 1, 3). + self.assertEqual(analyse.persistence(self.run, self.minutes), [0.5]) + + def test_the_fastest_of_minute_one_against_the_others(self): + top, rest = analyse.top_against_rest(self.run, self.minutes, top=1) + self.assertEqual(top, (2000, 3000)) + # [0, 0, 4000, 5000]: the median between, p99 the last. + self.assertEqual(rest, (2000, 5000)) + + def test_racing(self): + # Kept: connections 0 and 1, the sooner copy 0 for c3 and c4. At + # random: (0, 1) [0, 0], (0, 2) [1000, 3000] (median 2000, p99 3000), + # (1, 2) [0, 0]. + self.assertEqual( + analyse.racing(self.run, self.minutes, pair=2), ((0, 0), 0, (2000 / 3, 1000), 0) + ) + + def test_by_address(self): + changes = self.minutes[0] + self.minutes[1] + self.assertEqual( + analyse.by_address(self.run, changes), + # A: [0, 0, 1000, 3000] and [3000, 4000, 5000]; B: [1000, 2000, 0, 0]. + {"A": (3000, 5000), "B": (500, 2000)}, + ) + + def test_lag_raw_and_corrected_by_the_mean_offset(self): + raw, corrected, offset = analyse.lag(self.run, self.minutes[0][:1]) + self.assertEqual((raw, corrected, offset), ((10_000, 10_000), (10_500, 10_500), 500)) + + def test_speed_is_the_median_and_racing_keeps_the_fastest(self): + # Minute 1: lateness 0, 0, 0 on connection 0; 0, 0, 3000 on 1 (its + # median 0, its mean 1000); 2000 each on 2. Order 0, 1, 2, so the + # pair kept is (0, 1). Minute 2: connection 1 is now the slowest. + copies = [] + for n, late in enumerate([0, 0, 3000]): + at = S + n * 1_000_000 + copies += [(0, f"a{n}", at, 0), (1, f"a{n}", at + late, 0), (2, f"a{n}", at + 2000, 0)] + copies += [ + (0, "b", S + MIN, 0), + (1, "b", S + MIN + 9000, 0), + (2, "b", S + MIN + 1000, 0), + (0, "end", S + 2 * MIN + 1, 0), + ] + run = analyse.Run(csv(copies)) + minutes = run.minutes() + self.assertEqual(analyse.speeds(run, minutes[0]), {0: 0, 1: 0, 2: 2000}) + # Kept (0, 1): 0. At random: (0, 1) 0, (0, 2) 0, (1, 2) 1000. + picked, _, random, _ = analyse.racing(run, minutes, pair=2) + self.assertEqual((picked, random), ((0, 0), (1000 / 3, 1000 / 3))) + + def test_p99_is_not_the_maximum(self): + self.assertEqual(analyse.quantile(range(200), 0.99), 198) + + def test_the_report_of_the_hand_worked_run(self): + report = analyse.report(self.run) + # Connection 2's speeds, and the change it missed. + self.assertIn("| 2 | A | 3.00 4.50 | 1 |", report) + self.assertIn("rank correlation minute to minute 0.50, mean 0.50", report) + # Three connections: no 4 fastest against the others. + self.assertIn("The 4 fastest of minute 1 against the others: too few connections", report) + # Kept pair p99 0, pairs at random 1000 on average: 100% lower. + self.assertIn("a pair at random, 0.67 ms, 1.00 ms, 0.0 missed; p99 100% lower", report) + + def test_racing_counts_what_a_pair_missed(self): + # Minute 1 ranks 0, 1, 2; after it, one change only connection 2 + # delivers: missed by the kept pair (0, 1), by one pair in three. + run = analyse.Run( + csv( + [ + (0, "a", S, 0), + (1, "a", S + 1000, 0), + (2, "a", S + 2000, 0), + (2, "b", S + MIN, 0), + (0, "c", S + MIN + 1000, 0), + (1, "c", S + MIN + 1000, 0), + (2, "c", S + MIN + 1000, 0), + (0, "end", S + 2 * MIN + 1, 0), + ] + ) + ) + _, picked_missed, _, random_missed = analyse.racing(run, run.minutes(), pair=2) + self.assertEqual((picked_missed, random_missed), (1, 1 / 3)) + + def test_an_offset_the_probe_could_not_measure_is_left_out(self): + run = analyse.Run(csv([], notes=["offset,start,,", "offset,end,600,3000"])) + self.assertEqual(run.offsets, [600]) + + def test_thin_runs_are_reported_without_a_traceback(self): + # One connection over two minutes: nothing to compare. + alone = [(0, "a", S, 0), (0, "b", S + MIN, 0), (0, "end", S + 2 * MIN + 1, 0)] + report = analyse.report(analyse.Run(csv(alone))) + self.assertIn("Racing: too few connections deliver after minute 1", report) + # The kept pair, and every other, silent after minute 1 but one. + silent = [ + (0, "a", S, 0), + (1, "a", S + 1000, 0), + (2, "a", S + 2000, 0), + (2, "end", S + 2 * MIN + 1, 0), + ] + analyse.report(analyse.Run(csv(silent))) + + def test_a_run_with_no_change_is_reported_as_such(self): + self.assertEqual( + analyse.report(analyse.Run(csv([]))), "0 connections, no change over a full minute" + ) + + def test_a_connection_is_missing_only_while_up(self): + run = analyse.Run( + csv( + [ + # c1 before connection 2's first copy, c3 after it went + # down: neither is missed by it. + (0, "c1", S, 0), + (2, "c2", S + 1000, 0), + (0, "c2", S + 1000, 0), + (0, "c3", S + 3000, 0), + (0, "c4", S + MIN + 1, 0), + ], + notes=[f"down,2,{S + 2000}"], + ) + ) + self.assertEqual(run.lateness(run.minutes()[0], [2])[1], 0) + # Without the note, c3 is missed. + run.down.clear() + self.assertEqual(run.lateness(run.minutes()[0], [2])[1], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/core/feed.rs b/src/core/feed.rs index 74ad513..38d46ec 100644 --- a/src/core/feed.rs +++ b/src/core/feed.rs @@ -48,8 +48,15 @@ pub enum Report { market: String, connection: usize, }, + /// Each connection's lateness samples over the minute, µs (roadmap/07): + /// the orchestrator recycles a connection that stays among the slowest. + Lateness(Lateness), } +/// Connection → its copies' lateness this minute, µs: how long after the +/// first copy of a change it delivered its own (0 for the first). +pub type Lateness = HashMap>; + /// Each market's load (market id → bytes per second on one connection), /// every market of the book thread measured for at least [`MIN_MEASURED`]. pub type Loads = HashMap; @@ -148,6 +155,12 @@ pub trait Rule: Send + 'static { /// Logs the minute line, the shell's figures in it. fn log_minute(&mut self, minute: &Minute); + + /// The connections' lateness since the last minute, for the orchestrator; + /// none from a rule that doesn't judge its connections' speed. + fn take_lateness(&mut self) -> Lateness { + Lateness::new() + } } /// The shell's figures for the minute line. @@ -622,6 +635,10 @@ impl Feed { } } let mut effects = vec![Effect::Report(Report::Loads(loads))]; + let lateness = self.rule.take_lateness(); + if !lateness.is_empty() { + effects.push(Effect::Report(Report::Lateness(lateness))); + } let Some(received_at) = shell.last_received_at else { return effects; }; diff --git a/src/core/orchestrator.rs b/src/core/orchestrator.rs index 84f0b98..dda0390 100644 --- a/src/core/orchestrator.rs +++ b/src/core/orchestrator.rs @@ -14,7 +14,7 @@ use chrono::{DateTime, TimeDelta, Utc}; use crossbeam::channel::{Sender, select, unbounded}; use engineering_challenge::book::Output; -use super::feed::{self, BookCommand, Feed, Loads, Report}; +use super::feed::{self, BookCommand, Feed, Lateness, Loads, Report}; use super::socket::{self, Command, Notice}; use super::{BOOK_THREADS, Exchange, Finding, Market, Received, Request, book_thread}; @@ -52,6 +52,22 @@ const RECENT_MARGIN: TimeDelta = TimeDelta::minutes(15); /// Each connection is recycled about this often, one at a time, to shed what /// accumulates on the exchange's side of it. const RECYCLE_EVERY: Duration = Duration::from_secs(60 * 60); +/// A connection slow (among the slowest quarter by mean lateness, and more +/// than [`SLOW_FACTOR`] times the mean of every copy judged) this many +/// minutes in a row is recycled ahead of its turn: the next one is drawn +/// afresh. Kalshi's demo kept the 2 fastest of 16 connections 28% and 42% +/// lower at p99 (LATENCY.md); only Kalshi's rule reports lateness. The mean, +/// not the median: with two copies of each change, a connection first more +/// than half the time has a median of 0 (roadmap/07). +const SLOW_MINUTES: u32 = 3; +/// The slowest quarter is looked at: 1 in this many connections judged. +const SLOW_SHARE: usize = 4; +/// Equal connections are left alone: a slowest quarter always exists. +const SLOW_FACTOR: i64 = 2; +/// A connection is judged once up this long, past its opening books, and +/// with this many samples in the minute. +const JUDGED_AFTER: Duration = Duration::from_secs(10 * 60); +const JUDGED_SAMPLES: usize = 100; /// So SIGINT is seen while no timer is due. const WAKE_EVERY: Duration = Duration::from_secs(1); @@ -98,7 +114,7 @@ pub fn run( let mut queue: VecDeque> = VecDeque::new(); let mut next_deal = Instant::now(); let (mut added, mut removed, mut moved, mut splits) = (0, 0, 0, 0); - let (mut recycled, mut quiet_moves, mut silent) = (0, 0, 0); + let (mut recycled, mut slow_recycled, mut quiet_moves, mut silent) = (0, 0, 0, 0); // The minute checks cover the first scan from its start. let mut recent_since = Utc::now() - RECENT_MARGIN; let mut recent_asked_at: Option> = None; @@ -139,17 +155,27 @@ pub fn run( let moves = tracker.rebalance(now); moved += moves.len(); commands.extend(moves); + if let Some((connection, mean)) = tracker.judge_speeds(now) { + log!( + "{name}: orchestrator: connection {connection} among the slowest for \ + {SLOW_MINUTES} minutes (mean lateness {:.1} ms), recycled", + mean as f64 / 1_000.0 + ); + threads.recycle(connection); + recycled += 1; + slow_recycled += 1; + } let busiest = tracker.connections.iter().map(|one| one.load).max(); log!( "{name}: orchestrator: {} markets on {} connections; {added} added, {removed} \ - removed, {moved} moved ({quiet_moves} quiet), {splits} splits, {recycled} recycled, \ - {silent} silent this minute; busiest connection at {} bytes/s", + removed, {moved} moved ({quiet_moves} quiet), {splits} splits, {recycled} recycled \ + ({slow_recycled} slow), {silent} silent this minute; busiest connection at {} bytes/s", tracker.markets.len(), tracker.connections.len(), busiest.unwrap_or(0) ); (added, removed, moved, splits) = (0, 0, 0, 0); - (recycled, quiet_moves, silent) = (0, 0, 0); + (recycled, slow_recycled, quiet_moves, silent) = (0, 0, 0, 0); } // Recycling starts once the startup deal is sent. if first_scan.is_none() && queue.is_empty() { @@ -237,6 +263,7 @@ pub fn run( }, recv(reports) -> report => match report { Ok(Report::Loads(loads)) => tracker.report(&loads), + Ok(Report::Lateness(samples)) => tracker.add_lateness(samples), Ok(Report::Quiet { market, connection }) => { if let Some(command) = tracker.quiet(&market, connection, Instant::now()) { log!( @@ -418,6 +445,8 @@ struct ConnectionState { up: bool, /// When it last came up: the one longest ago is recycled next. connected_at: Option, + /// Minutes in a row among the slowest. + slow_minutes: u32, } /// Which connections carry each market and what it weighs, and which markets @@ -435,6 +464,8 @@ struct Tracker { next_recycle: Option, /// The connection recycled and not yet up again. recycling: Option, + /// The book threads' lateness samples since the last judgment. + lateness: Lateness, } impl Tracker { @@ -447,6 +478,7 @@ impl Tracker { random: Random(seed | 1), next_recycle: None, recycling: None, + lateness: Lateness::new(), } } @@ -725,6 +757,81 @@ impl Tracker { Some(oldest) } + /// A book thread's lateness samples, by connection. + fn add_lateness(&mut self, samples: Lateness) { + for (connection, samples) in samples { + self.lateness.entry(connection).or_default().extend(samples); + } + } + + /// Once a minute: ranks the connections judged by their mean lateness + /// since the last time; those of the slowest 1 in [`SLOW_SHARE`] more + /// than [`SLOW_FACTOR`] times the mean of every judged copy count a slow + /// minute, the others start over. The slowest of those slow for + /// [`SLOW_MINUTES`] is recycled now, with its mean (µs), under + /// [`Tracker::recycle`]'s + /// rule: one at a time, every connection up, none up for less than an + /// interval. + fn judge_speeds(&mut self, now: Instant) -> Option<(usize, i64)> { + let samples = std::mem::take(&mut self.lateness); + let speeds: Vec<(usize, i64, i64)> = samples + .into_iter() + .filter(|(connection, samples)| { + samples.len() >= JUDGED_SAMPLES + && self.connections.get(*connection).is_some_and(|state| { + state.up + && state + .connected_at + .is_some_and(|at| now.duration_since(at) >= JUDGED_AFTER) + }) + }) + .map(|(connection, samples)| { + let sum: i64 = samples.iter().sum(); + (connection, sum, samples.len() as i64) + }) + .collect(); + let (sum, count) = speeds + .iter() + .fold((0, 0), |(sum, count), &(_, s, n)| (sum + s, count + n)); + let overall = sum.checked_div(count).unwrap_or(0); + let mut speeds: Vec<(usize, i64)> = speeds + .into_iter() + .map(|(connection, sum, count)| (connection, sum / count)) + .collect(); + // The slowest first; ties by number, so a judgment doesn't depend on + // the map's order. + speeds.sort_by_key(|&(connection, mean)| (Reverse(mean), connection)); + let quarter = speeds.len() / SLOW_SHARE; + let slow: Vec<(usize, i64)> = speeds[..quarter] + .iter() + .copied() + .filter(|&(_, mean)| mean > SLOW_FACTOR * overall) + .collect(); + for (number, state) in self.connections.iter_mut().enumerate() { + state.slow_minutes = if slow.iter().any(|&(connection, _)| connection == number) { + state.slow_minutes + 1 + } else { + 0 + }; + } + // As for the hourly recycle: one at a time, every connection up, and + // the last to come up past its opening books. + let settled = self + .connections + .iter() + .filter_map(|state| state.connected_at) + .max() + .is_none_or(|last| now.duration_since(last) >= self.recycle_interval()); + if self.recycling.is_some() || self.connections.iter().any(|state| !state.up) || !settled { + return None; + } + let &(connection, speed) = slow + .iter() + .find(|&&(connection, _)| self.connections[connection].slow_minutes >= SLOW_MINUTES)?; + self.recycling = Some(connection); + Some((connection, speed)) + } + fn scan_started(&mut self) { self.unseen = Some(self.markets.keys().cloned().collect()); } @@ -1103,6 +1210,131 @@ mod tests { assert_eq!(tracker.recycle(at(7_201)), Some(1)); } + /// Twelve connections, up from `start`. + fn twelve_up(start: Instant) -> Tracker { + let mut tracker = tracker(); + tracker.connections = vec![ConnectionState::default(); 12]; + for connection in 0..12 { + tracker.up(connection, start); + } + tracker + } + + /// A minute's samples, `count` per connection: 1 ms each, but for those + /// `slow` names, at their ms. + fn minute(tracker: &mut Tracker, slow: &[(usize, i64)], count: usize) { + let samples = (0..12) + .map(|connection| { + let ms = slow + .iter() + .find(|&&(slow, _)| slow == connection) + .map_or(1, |&(_, ms)| ms); + (connection, vec![ms * 1_000; count]) + }) + .collect(); + tracker.add_lateness(samples); + } + + #[test] + fn a_connection_among_the_slowest_three_minutes_running_is_recycled() { + let start = Instant::now(); + let at = |seconds: u64| start + Duration::from_secs(seconds); + let mut tracker = twelve_up(start); + // The slowest quarter is three of twelve. 3 and 7 are slow every + // minute, 5 for two minutes and then not. + minute(&mut tracker, &[(3, 90), (5, 50), (7, 40)], 100); + assert_eq!(tracker.judge_speeds(at(601)), None); + minute(&mut tracker, &[(3, 90), (5, 50), (7, 40)], 100); + assert_eq!(tracker.judge_speeds(at(661)), None); + // 3 and 7 both reach three minutes: one recycle a minute, the slowest. + minute(&mut tracker, &[(3, 90), (7, 40), (9, 30)], 100); + assert_eq!(tracker.judge_speeds(at(721)), Some((3, 90_000))); + assert_eq!(tracker.connections[5].slow_minutes, 0); + // While 3 recycles, 7 stays slow but isn't recycled. + minute(&mut tracker, &[(3, 90), (7, 40), (9, 30)], 100); + assert_eq!(tracker.judge_speeds(at(781)), None); + assert_eq!(tracker.connections[7].slow_minutes, 4); + tracker.down(3); + tracker.up(3, at(790)); + // 3 is fresh: slow, but not judged. 7 waits, as the hourly recycle + // would, for 3's opening books: an interval (an hour / 12) after it + // came up. + minute(&mut tracker, &[(3, 90), (7, 40), (9, 30)], 100); + assert_eq!(tracker.judge_speeds(at(841)), None); + assert_eq!(tracker.connections[3].slow_minutes, 0); + minute(&mut tracker, &[(3, 90), (7, 40), (9, 30)], 100); + assert_eq!(tracker.judge_speeds(at(1_089)), None); + minute(&mut tracker, &[(3, 90), (7, 40), (9, 30)], 100); + assert_eq!(tracker.judge_speeds(at(1_090)), Some((7, 40_000))); + } + + #[test] + fn the_mean_of_every_book_threads_samples_judges() { + let start = Instant::now(); + let at = |seconds: u64| start + Duration::from_secs(seconds); + let mut tracker = twelve_up(start); + for seconds in [601, 661, 721] { + // Two book threads report, 50 samples each for every connection. + // Connection 3's are 0 but for 10 copies 1 s late: a median of + // 0, a mean of 100 ms; the others' 10 ms each. + for thread in 0..2 { + let mut samples: Lateness = (0..12).map(|c| (c, vec![10_000; 50])).collect(); + let late = if thread == 0 { 10 } else { 0 }; + let mut three = vec![1_000_000; late]; + three.resize(50, 0); + samples.insert(3, three); + tracker.add_lateness(samples); + } + let judged = tracker.judge_speeds(at(seconds)); + assert_eq!(judged, (seconds == 721).then_some((3, 100_000))); + } + } + + #[test] + fn equal_connections_are_left_alone() { + let start = Instant::now(); + let at = |seconds: u64| start + Duration::from_secs(seconds); + let mut tracker = twelve_up(start); + // A slowest quarter exists every minute (3, 7 and 9, at 15 to 20 ms + // against 10), but none is more than twice the mean of all. + for seconds in [601, 661, 721, 781] { + let mut samples: Lateness = (0..12).map(|c| (c, vec![10_000; 100])).collect(); + for (connection, ms) in [(3, 20), (7, 18), (9, 15)] { + samples.insert(connection, vec![ms * 1_000; 100]); + } + tracker.add_lateness(samples); + assert_eq!(tracker.judge_speeds(at(seconds)), None); + } + assert_eq!(tracker.connections[3].slow_minutes, 0); + } + + #[test] + fn a_connection_is_judged_only_up_long_enough_and_with_enough_samples() { + let start = Instant::now(); + let at = |seconds: u64| start + Duration::from_secs(seconds); + // Too early: slow minutes before the tenth say nothing, to the + // second. + let mut tracker = twelve_up(start); + for seconds in [420, 480, 540, 599] { + minute(&mut tracker, &[(3, 90), (5, 50)], 100); + assert_eq!(tracker.judge_speeds(at(seconds)), None); + assert_eq!(tracker.connections[3].slow_minutes, 0); + } + // Too few samples: never judged, so the others share the slowest. + let mut tracker = twelve_up(start); + for seconds in [601, 661, 721] { + minute(&mut tracker, &[(3, 90), (5, 50)], 99); + assert_eq!(tracker.judge_speeds(at(seconds)), None); + } + // A connection down holds every recycle, as the hourly one's rule. + let mut tracker = twelve_up(start); + tracker.down(0); + for seconds in [601, 661, 721, 781] { + minute(&mut tracker, &[(3, 90), (5, 50)], 100); + assert_eq!(tracker.judge_speeds(at(seconds)), None); + } + } + #[test] fn a_market_quiet_again_moves_off_that_connection_unless_it_moved_lately() { let mut tracker = tracker(); diff --git a/src/core/socket.rs b/src/core/socket.rs index e0b9687..125f0cd 100644 --- a/src/core/socket.rs +++ b/src/core/socket.rs @@ -6,7 +6,7 @@ use std::collections::BTreeSet; use std::io::ErrorKind; -use std::net::{TcpStream, ToSocketAddrs}; +use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; use std::sync::atomic::{AtomicBool, Ordering}; use std::thread; use std::time::{Duration, Instant}; @@ -25,7 +25,7 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); /// so the reads and the writes below (keepalive, commands, close) share this /// thread. const READ_TIMEOUT: Duration = Duration::from_millis(200); -const PING_EVERY: Duration = Duration::from_secs(10); +pub(crate) const PING_EVERY: Duration = Duration::from_secs(10); const RECONNECT_AFTER: Duration = Duration::from_secs(1); const CLOSE_TIMEOUT: Duration = Duration::from_secs(2); /// Nothing received this long, not even a keepalive's answer or a ping: the @@ -56,7 +56,7 @@ pub enum Notice { Silent(usize), } -type Socket = WebSocket>; +pub(crate) type Socket = WebSocket>; /// Runs until `stop` is set or a book thread is gone, reconnecting (and /// subscribing its symbols again) whenever the connection drops, falls silent @@ -183,10 +183,29 @@ fn connect( .to_socket_addrs()? .next() .ok_or_else(|| anyhow::anyhow!("{host} has no address"))?; + let mut socket = open(request, address)?; + for message in session.opening() { + socket.send(Message::text(message))?; + } + if !symbols.is_empty() { + for message in session.subscribe(&symbols.iter().collect::>()) { + socket.send(Message::text(message))?; + } + } + Ok(socket) +} + +/// The TCP connect to `address` and the TLS + websocket handshake of +/// `request`, each bounded by [`CONNECT_TIMEOUT`]; reads then time out after +/// [`READ_TIMEOUT`]. +pub(crate) fn open( + request: tungstenite::handshake::client::Request, + address: SocketAddr, +) -> anyhow::Result { let stream = TcpStream::connect_timeout(&address, CONNECT_TIMEOUT)?; stream.set_read_timeout(Some(CONNECT_TIMEOUT))?; stream.set_write_timeout(Some(CONNECT_TIMEOUT))?; - let (mut socket, _response) = tungstenite::client_tls(request, stream).map_err(|error| { + let (socket, _response) = tungstenite::client_tls(request, stream).map_err(|error| { // A refusal's status says why: 401 for a key the exchange refuses. match error { tungstenite::HandshakeError::Failure(tungstenite::Error::Http(response)) => { @@ -201,14 +220,6 @@ fn connect( _ => anyhow::bail!("unexpected TLS stream"), }; stream.set_read_timeout(Some(READ_TIMEOUT))?; - for message in session.opening() { - socket.send(Message::text(message))?; - } - if !symbols.is_empty() { - for message in session.subscribe(&symbols.iter().collect::>()) { - socket.send(Message::text(message))?; - } - } Ok(socket) } @@ -392,6 +403,6 @@ fn close( /// A read timeout shows up as `WouldBlock` on macOS and `TimedOut` elsewhere; /// tungstenite keeps a partly read frame, so the next read resumes it. -fn timed_out(error: &std::io::Error) -> bool { +pub(crate) fn timed_out(error: &std::io::Error) -> bool { matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) } diff --git a/src/kalshi/feed.rs b/src/kalshi/feed.rs index 901b726..d7ddf28 100644 --- a/src/kalshi/feed.rs +++ b/src/kalshi/feed.rs @@ -9,7 +9,7 @@ //! A connection that lost a message reconnects (its `seq`, in //! `src/kalshi/mod.rs`); the book then comes fresh. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, VecDeque}; use chrono::{DateTime, Utc}; use engineering_challenge::book::Book; @@ -17,7 +17,7 @@ use engineering_challenge::cause::{self, Cause}; use super::Market; use super::wire::{BookMessage, ChangeMessage, Event}; -use crate::core::feed::{self, Effect, Minute, Rule, Shell, is_up}; +use crate::core::feed::{self, Effect, Lateness, Minute, Rule, Shell, is_up}; use crate::core::socket::Command; /// A clock, and a rank among the changes of that clock. @@ -26,6 +26,14 @@ type Position = (u64, u32); /// A book's rank: it holds every change of its clock. const BOOK: u32 = u32::MAX; +/// Changes remembered per symbol for their second copy's lateness: two +/// connections deliver a market's changes 2 ms apart at the median, 344 ms +/// at worst (docs/notes.md), far fewer changes than this. +// ponytail: a second copy more than 128 changes behind goes unmeasured, and +// the lookup is linear; a map by position, emptied as the book passes, +// would lift both. +const RECENT: usize = 128; + /// A symbol's state for the rule. #[derive(Debug, Default)] pub struct Clocked { @@ -37,6 +45,9 @@ pub struct Clocked { on: BTreeMap, /// The connection a fresh book was asked from: its answer replaces ours. asked: Option, + /// Its last [`RECENT`] changes applied: where, from which connection, + /// when; a change's second copy takes its out. + recent: VecDeque<(Position, usize, DateTime)>, } type Symbol = feed::Symbol; @@ -64,6 +75,7 @@ struct Counts { #[derive(Default)] pub struct Clock { counts: Counts, + lateness: Lateness, } impl Clock { @@ -113,6 +125,9 @@ impl Clock { symbol.book = book; symbol.exchange_timestamp = message.sent_at; symbol.rule.at = Some(position); + // A position before it may come again, from a stream behind: it would + // be measured against a copy from before the book. + symbol.rule.recent.clear(); // A first book answers whichever connection it was asked from. if asked || cause.is_none() { symbol.rule.asked = None; @@ -154,6 +169,22 @@ impl Clock { match symbol.rule.at { Some(at) if position <= at => { self.counts.duplicates += 1; + let recent = &mut symbol.rule.recent; + let index = recent.iter().position(|&(at, ..)| at == position); + if let Some((_, first, first_at)) = index.and_then(|index| recent.remove(index)) + && let Some(late) = (received_at - first_at).num_microseconds() + { + // The book thread applies copies in the order they reach + // it, not in the order they were stamped: the later + // stamp is the late one. + let (late_on, first_on) = if late < 0 { + (first, connection) + } else { + (connection, first) + }; + self.lateness.entry(late_on).or_default().push(late.abs()); + self.lateness.entry(first_on).or_default().push(0); + } return; } // Its stream was already past the book, which went back to an @@ -166,6 +197,13 @@ impl Clock { _ => {} } symbol.rule.at = Some(position); + if symbol.rule.recent.len() == RECENT { + symbol.rule.recent.pop_front(); + } + symbol + .rule + .recent + .push_back((position, connection, received_at)); let quantity = symbol.book.add(change.side, change.price, change.delta); let quantity = if quantity < 0 { // A level can't hold less than nothing: ours was wrong. @@ -316,6 +354,10 @@ impl Rule for Clock { } } + fn take_lateness(&mut self) -> Lateness { + std::mem::take(&mut self.lateness) + } + fn log_minute(&mut self, minute: &Minute) { let counts = &self.counts; log!( @@ -462,6 +504,90 @@ mod tests { assert_eq!(feed.rule.counts.duplicates, 1); } + #[test] + fn a_duplicate_reports_its_lateness_and_the_first_copy_zero_at_the_minute() { + let mut feed = both_on("M"); + feed.from(1, 5, change("M", 11, Side::Buy, 400_000, 2_000_000)); + feed.from(0, 9, change("M", 11, Side::Buy, 400_000, 2_000_000)); + // Only one connection delivers it: no copy to measure against. + feed.from(1, 20, change("M", 12, Side::Buy, 400_000, 1_000_000)); + let reported: Vec<_> = feed + .tick(at(0) + TimeDelta::minutes(2)) + .into_iter() + .filter_map(|effect| match effect { + Effect::Report(feed::Report::Lateness(lateness)) => Some(lateness), + _ => None, + }) + .collect(); + assert_eq!(reported, [HashMap::from([(0, vec![4_000]), (1, vec![0])])]); + // Taken: the next minute starts empty. + assert!(feed.rule.take_lateness().is_empty()); + } + + #[test] + fn the_copy_stamped_later_is_the_late_one_whichever_is_applied_first() { + let mut feed = both_on("M"); + // Connection 1's copy reaches the book thread first, but connection + // 0's was stamped 4 ms before it. + feed.from(1, 9, change("M", 11, Side::Buy, 400_000, 2_000_000)); + feed.from(0, 5, change("M", 11, Side::Buy, 400_000, 2_000_000)); + assert_eq!( + feed.rule.take_lateness(), + HashMap::from([(0, vec![0]), (1, vec![4_000])]) + ); + } + + #[test] + fn a_fresh_book_forgets_the_copies_before_it() { + let mut feed = both_on("M"); + feed.from(1, 5, change("M", 11, Side::Buy, 400_000, 2_000_000)); + // A fresh book past ours, and different: it replaces ours. + let replaced = feed.from(0, 6, book("M", 12, &[(400_000, 1_000_000)], &[])); + assert_eq!(snapshots(&replaced), 1); + // A change behind it, from a stream behind, an hour later: a + // duplicate, not measured against the copy of before the book. + feed.from(0, 3_600_000, change("M", 11, Side::Buy, 400_000, 2_000_000)); + assert_eq!(feed.rule.counts.duplicates, 1); + assert!(feed.rule.take_lateness().is_empty()); + } + + #[test] + fn a_symbol_remembers_its_last_128_changes_only() { + let mut feed = feed_of(&["M"], &[0]); + feed.from(0, 1, book("M", 10, &[(400_000, 5_000_000)], &[])); + // One connection: no change has a second copy to take it out. + for clock in 11..211 { + feed.from(0, 2, change("M", clock, Side::Buy, 400_000, 1_000)); + } + let recent = &feed.shell.symbols["M"].rule.recent; + assert_eq!(recent.len(), RECENT); + // The oldest went first. + assert_eq!( + recent.front().map(|&(position, ..)| position), + Some((83, 1)) + ); + } + + #[test] + fn a_third_copy_while_a_market_moves_is_not_measured() { + let mut feed = feed_of(&["M"], &[0, 1, 2]); + for connection in [0, 1, 2] { + feed.from(connection, 1, book("M", 10, &[(400_000, 5_000_000)], &[])); + } + for (connection, millis) in [(1, 5), (0, 9), (2, 30)] { + feed.from( + connection, + millis, + change("M", 11, Side::Buy, 400_000, 2_000_000), + ); + } + // The first copy counts once, against the second. + assert_eq!( + feed.rule.take_lateness(), + HashMap::from([(0, vec![4_000]), (1, vec![0])]) + ); + } + #[test] fn every_change_of_one_clock_is_applied_once() { let mut feed = both_on("M"); diff --git a/src/kalshi/listing.rs b/src/kalshi/listing.rs index 420c624..07cf99f 100644 --- a/src/kalshi/listing.rs +++ b/src/kalshi/listing.rs @@ -25,11 +25,7 @@ const OPENING_AHEAD: TimeDelta = TimeDelta::hours(3); /// A fetcher thread: answers each request until the orchestrator is gone. pub fn run(listing: &str, requests: Receiver, findings: Sender>) { - let agent = ureq::Agent::new_with_config( - ureq::Agent::config_builder() - .timeout_global(Some(TIMEOUT)) - .build(), - ); + let agent = agent(); let markets = format!("{listing}/markets"); for request in requests { let sent = match request { @@ -42,6 +38,22 @@ pub fn run(listing: &str, requests: Receiver, findings: Sender ureq::Agent { + ureq::Agent::new_with_config( + ureq::Agent::config_builder() + .timeout_global(Some(TIMEOUT)) + .build(), + ) +} + +/// Every open market's ticker, in the listing's order (the latency probe's +/// candidates, roadmap/07). +pub(crate) fn open_tickers(listing: &str) -> anyhow::Result> { + let query = [("status", "open".to_string()), mve_excluded()]; + let markets = every_page(&agent(), &format!("{listing}/markets"), &query)?; + Ok(markets.into_iter().map(|market| market.ticker).collect()) +} + /// `Err` once the orchestrator is gone. type Sent = Result<(), crossbeam::channel::SendError>>; diff --git a/src/latency.rs b/src/latency.rs new file mode 100644 index 0000000..33f4bc3 --- /dev/null +++ b/src/latency.rs @@ -0,0 +1,524 @@ +//! The latency probe (roadmap/07): [`CONNECTIONS`] connections to the same +//! busy markets, spread over the exchange's addresses, each copy of each +//! change written as a CSV line: which connection delivered it, from which +//! address, when. It computes nothing: `scripts/latency/analyse.py` matches +//! the copies across connections. No book, no rule, no reconnect: a +//! connection that drops stays down, and the CSV says when. +//! Usage: `engineering-challenge latency EXCHANGE OUTPUT`, until SIGINT. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::net::{SocketAddr, ToSocketAddrs, UdpSocket}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; +use std::time::{Duration, Instant}; + +use chrono::Utc; +use crossbeam::channel::{Sender, unbounded}; +use engineering_challenge::Side; +use serde::Deserialize; +use tungstenite::Message; + +use crate::core::socket::{self, PING_EVERY}; +use crate::core::{Exchange, Session, Undecoded, retried}; +use crate::kalshi::{self, Kalshi}; +use crate::polymarket::{self, Polymarket}; + +/// Connections per run: enough to rank, few enough to share the server. +const CONNECTIONS: usize = 16; +/// Markets every connection carries: few, so our own load stays out of the +/// measurement. +const MARKETS: usize = 100; +/// Kalshi's demo reports no traded amount: every open market's changes are +/// counted for [`COUNTING`], this many on each connection (30,000 met no +/// limit, docs/notes.md). +const COUNTED_PER_CONNECTION: usize = 30_000; +const COUNTING: Duration = Duration::from_secs(60); +/// Fewer markets changed while counting: the run would measure nothing. +const ENOUGH_MARKETS: usize = 10; +/// A refused handshake (429 when many open at once) is tried again. +const CONNECT_TRIES: u32 = 3; +const CONNECT_RETRY_AFTER: Duration = Duration::from_secs(1); +const SNTP_SERVER: &str = "time.cloudflare.com:123"; +const GAMMA_MARKETS: &str = "https://gamma-api.polymarket.com/markets"; +const HTTP_TIMEOUT: Duration = Duration::from_secs(30); + +const USAGE: &str = "usage: engineering-challenge latency OUTPUT"; + +/// One copy of a change, as a connection delivered it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ChangeCopy { + symbol: String, + /// The same on every connection for one change, different for two. + identity: String, + exchange_ms: i64, +} + +/// How an exchange's events become copies. +trait Copies: Exchange { + /// What a connection remembers to tell identical changes apart. + type Seen: Default; + + fn copies(seen: &mut Self::Seen, event: &Self::Event) -> Vec; +} + +impl Copies for Polymarket { + type Seen = (); + + /// A price change's entries, each told apart by its symbol and the book + /// hash after it, with its side, price and size: the rule's own copy + /// test (`Held::key`). + fn copies(_: &mut (), event: &polymarket::wire::Event) -> Vec { + let polymarket::wire::Event::PriceChange(message) = event else { + return Vec::new(); + }; + let Ok(exchange_ms) = message.timestamp.parse() else { + return Vec::new(); + }; + message + .price_changes + .iter() + .map(|change| ChangeCopy { + symbol: change.asset_id.clone(), + identity: format!( + "{}/{}/{}/{}", + change.hash, change.side, change.price, change.size + ), + exchange_ms, + }) + .collect() + } +} + +/// A change's side, price and delta. +type Content = (Side, i64, i64); + +/// By market: its last clock, and how often each change came under it. +#[derive(Default)] +struct KalshiSeen(HashMap)>); + +impl Copies for Kalshi { + type Seen = KalshiSeen; + + /// A change, told apart by its market, clock, side, price and delta, and + /// by how many such came before it under that clock on this connection: + /// 2 to 4 changes can share a clock, some identical (docs/notes.md). + fn copies(seen: &mut KalshiSeen, event: &kalshi::wire::Event) -> Vec { + let kalshi::wire::Event::Change(change) = event else { + return Vec::new(); + }; + let (clock, counts) = seen.0.entry(change.ticker.clone()).or_default(); + // A market's clock never goes back: the older ones' counts are done. + if *clock != change.clock { + *clock = change.clock; + counts.clear(); + } + let count = counts + .entry((change.side, change.price, change.delta)) + .or_default(); + *count += 1; + vec![ChangeCopy { + symbol: change.ticker.clone(), + identity: format!( + "{}/{}/{}/{}/{}", + change.clock, change.side, change.price, change.delta, count + ), + exchange_ms: change.at.timestamp_millis(), + }] + } +} + +/// What the writer gets. +enum Line { + Copy { + connection: usize, + address: SocketAddr, + received_us: i64, + copy: ChangeCopy, + }, + /// A line starting with `#`: the clock's offset, a connection down. + Note(String), +} + +pub fn run(mut args: impl Iterator) -> anyhow::Result<()> { + let exchange = args.next().ok_or_else(|| anyhow::anyhow!(USAGE))?; + let output = args.next().ok_or_else(|| anyhow::anyhow!(USAGE))?; + let stop = Arc::new(AtomicBool::new(false)); + let on_sigint = Arc::clone(&stop); + ctrlc::set_handler(move || on_sigint.store(true, Ordering::Relaxed))?; + match exchange.as_str() { + "polymarket" => { + let polymarket = Polymarket { + url: polymarket::MARKET_CHANNEL_URL.to_string(), + }; + probe(&polymarket, polymarket_symbols()?, &output, &stop) + } + "kalshi" | "kalshi-demo" => { + let key = kalshi::sign::Key::from_env()?; + let kalshi = if exchange == "kalshi" { + Kalshi::production(key) + } else { + Kalshi::demo(key) + }; + let symbols = kalshi_symbols(&kalshi, &stop)?; + probe(&kalshi, symbols, &output, &stop) + } + _ => anyhow::bail!("unknown exchange {exchange:?}; {USAGE}"), + } +} + +/// Runs the connections until `stop`, writing every copy to `output`. +fn probe( + exchange: &E, + symbols: Vec, + output: &str, + stop: &AtomicBool, +) -> anyhow::Result<()> { + let addresses = addresses(exchange)?; + log!( + "latency: {} symbols on {CONNECTIONS} connections over {addresses:?}", + symbols.len() + ); + let mut out = BufWriter::new(File::create(output)?); + let (lines, received) = unbounded(); + let writer = thread::spawn(move || -> anyhow::Result<()> { + writeln!( + out, + "connection,address,symbol,identity,received_us,exchange_ms" + )?; + for line in received { + match line { + Line::Copy { + connection, + address, + received_us, + copy, + } => writeln!( + out, + "{connection},{address},{},{},{received_us},{}", + copy.symbol, copy.identity, copy.exchange_ms + )?, + Line::Note(note) => writeln!(out, "# {note}")?, + } + } + out.flush()?; + Ok(()) + }); + note_offset(&lines, "start"); + thread::scope(|scope| { + for number in 0..CONNECTIONS { + let address = addresses[number % addresses.len()]; + let (symbols, lines) = (&symbols, lines.clone()); + scope.spawn(move || connection(exchange, number, address, symbols, stop, &lines)); + } + }); + note_offset(&lines, "end"); + drop(lines); + writer + .join() + .map_err(|_| anyhow::anyhow!("the writer thread panicked"))??; + log!("latency: output closed"); + Ok(()) +} + +/// Every IPv4 address the websocket's host resolves to, in a fixed order. +/// IPv6 ones are left out: the workspace has no IPv6 route ("Network is +/// unreachable", 2026-11-12). +fn addresses(exchange: &E) -> anyhow::Result> { + let request = exchange.request()?; + let host = request + .uri() + .host() + .ok_or_else(|| anyhow::anyhow!("{} has no host", request.uri()))?; + let port = request.uri().port_u16().unwrap_or(443); + let addresses: BTreeSet = (host, port) + .to_socket_addrs()? + .filter(SocketAddr::is_ipv4) + .collect(); + anyhow::ensure!(!addresses.is_empty(), "{host} has no IPv4 address"); + Ok(addresses.into_iter().collect()) +} + +/// One connection, to `address`, until `stop` or until it drops. +fn connection( + exchange: &E, + number: usize, + address: SocketAddr, + symbols: &[String], + stop: &AtomicBool, + lines: &Sender, +) { + let name = E::NAME; + let down = |why: String| { + log!("{name}: latency connection {number} to {address}: {why}"); + let at = Utc::now().timestamp_micros(); + _ = lines.send(Line::Note(format!("down,{number},{at}"))); + }; + let opened = retried(CONNECT_TRIES, CONNECT_RETRY_AFTER, || { + socket::open(exchange.request()?, address) + }); + let mut socket = match opened { + Ok(socket) => socket, + Err(error) => return down(format!("connect failed: {error}")), + }; + let mut session = exchange.session(); + let symbols: Vec<&String> = symbols.iter().collect(); + for message in [session.opening(), session.subscribe(&symbols)].concat() { + if let Err(error) = socket.send(Message::text(message)) { + return down(format!("subscribe failed: {error}")); + } + } + log!("{name}: latency connection {number} to {address}: connected"); + let mut seen = E::Seen::default(); + let mut last_ping = Instant::now(); + while !stop.load(Ordering::Relaxed) { + let message = socket.read(); + // Stamped as soon as the frame is off the wire, as the collector does. + let received_us = Utc::now().timestamp_micros(); + match message { + Ok(Message::Text(text)) => match session.decode(text.as_str()) { + Ok(events) => { + for copy in events.iter().flat_map(|event| E::copies(&mut seen, event)) { + let line = Line::Copy { + connection: number, + address, + received_us, + copy, + }; + if lines.send(line).is_err() { + return; + } + } + } + Err(Undecoded::Unreadable(error)) => { + log!("{name}: latency connection {number}: unreadable frame ({error})") + } + Err(Undecoded::Lost(why)) => return down(format!("lost a message: {why}")), + }, + Ok(Message::Close(frame)) => return down(format!("closed by the server: {frame:?}")), + Ok(_) => {} + Err(tungstenite::Error::Io(error)) if socket::timed_out(&error) => {} + Err(error) => return down(format!("connection lost: {error}")), + } + if last_ping.elapsed() >= PING_EVERY { + if let Some(keepalive) = session.keepalive() + && let Err(error) = socket.send(Message::text(keepalive)) + { + return down(format!("keepalive failed: {error}")); + } + last_ping = Instant::now(); + } + } + _ = socket.close(None); + _ = socket.flush(); +} + +/// The local clock's offset from [`SNTP_SERVER`], as a note: `offset,, +/// ,`; a failed query is noted and logged, not +/// fatal. +fn note_offset(lines: &Sender, when: &str) { + let note = match offset() { + Ok((offset, roundtrip)) => { + log!("latency: clock offset {offset} µs (round trip {roundtrip} µs)"); + format!("offset,{when},{offset},{roundtrip}") + } + Err(error) => { + log!("latency: clock offset unknown: {error}"); + format!("offset,{when},,") + } + }; + _ = lines.send(Line::Note(note)); +} + +fn offset() -> anyhow::Result<(i64, u64)> { + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.set_read_timeout(Some(Duration::from_secs(2)))?; + let server = SNTP_SERVER + .to_socket_addrs()? + .find(SocketAddr::is_ipv4) + .ok_or_else(|| anyhow::anyhow!("{SNTP_SERVER} has no IPv4 address"))?; + let context = sntpc::NtpContext::new(sntpc::StdTimestampGen::default()); + let result = sntpc::sync::get_time( + server, + &sntpc_net_std::UdpSocketWrapper::new(socket), + context, + ) + .map_err(|error| anyhow::anyhow!("SNTP: {error:?}"))?; + Ok((result.offset(), result.roundtrip())) +} + +fn agent() -> ureq::Agent { + ureq::Agent::config_builder() + .timeout_global(Some(HTTP_TIMEOUT)) + .build() + .into() +} + +/// The tokens of the [`MARKETS`] live markets most traded over 24 hours. +fn polymarket_symbols() -> anyhow::Result> { + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct GammaMarket { + /// A JSON array inside a string. + clob_token_ids: Option, + } + let limit = MARKETS.to_string(); + let body = agent() + .get(GAMMA_MARKETS) + .query("closed", "false") + .query("active", "true") + .query("order", "volume24hr") + .query("ascending", "false") + .query("limit", &limit) + .call()? + .body_mut() + .read_to_string()?; + let markets: Vec = serde_json::from_str(&body)?; + let tokens: Vec = markets + .iter() + .filter_map(|market| market.clob_token_ids.as_deref()) + .filter_map(|ids| serde_json::from_str::>(ids).ok()) + .flatten() + .collect(); + anyhow::ensure!(!tokens.is_empty(), "Gamma listed no tokens"); + Ok(tokens) +} + +/// The [`MARKETS`] markets with the most changes in [`COUNTING`], among the +/// listing's open ones. Not the listing's first pages: its order moves (the +/// 98 busiest of 11:12 were past its first 40,000 at 11:50, 2026-11-12). +fn kalshi_symbols(kalshi: &Kalshi, stop: &AtomicBool) -> anyhow::Result> { + let candidates = kalshi::listing::open_tickers(&kalshi.listing)?; + let chunks: Vec<&[String]> = candidates.chunks(COUNTED_PER_CONNECTION).collect(); + log!( + "latency: counting the changes of {} Kalshi markets on {} connections for {} s", + candidates.len(), + chunks.len(), + COUNTING.as_secs() + ); + let addresses = addresses(kalshi)?; + let counted = AtomicBool::new(false); + let (lines, received) = unbounded(); + let mut changes: BTreeMap = BTreeMap::new(); + thread::scope(|scope| { + for (number, chunk) in chunks.iter().enumerate() { + let (address, counted, lines) = (addresses[number % addresses.len()], &counted, &lines); + scope.spawn(move || connection(kalshi, number, address, chunk, counted, lines)); + } + let deadline = Instant::now() + COUNTING; + while Instant::now() < deadline && !stop.load(Ordering::Relaxed) { + for line in received.try_iter() { + if let Line::Copy { copy, .. } = line { + *changes.entry(copy.symbol).or_default() += 1; + } + } + thread::sleep(Duration::from_millis(100)); + } + counted.store(true, Ordering::Relaxed); + }); + busiest(changes, MARKETS) +} + +/// The `n` symbols with the most changes, the most first; an error if fewer +/// than [`ENOUGH_MARKETS`] changed at all: the run would measure nothing. +fn busiest(changes: BTreeMap, n: usize) -> anyhow::Result> { + anyhow::ensure!( + changes.len() >= ENOUGH_MARKETS, + "only {} markets changed in {} s: nothing to measure", + changes.len(), + COUNTING.as_secs() + ); + let mut ranked: Vec<(String, u64)> = changes.into_iter().collect(); + ranked.sort_by_key(|(_, changes)| std::cmp::Reverse(*changes)); + Ok(ranked + .into_iter() + .take(n) + .map(|(symbol, _)| symbol) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn polymarket(text: &str) -> Vec { + polymarket::wire::decode(text) + .unwrap() + .iter() + .flat_map(|event| Polymarket::copies(&mut (), event)) + .collect() + } + + fn price_change(hash: &str, size: &str) -> String { + format!( + r#"{{"market":"0xm","price_changes":[{{"asset_id":"a","price":"0.29","size":"{size}","side":"SELL","hash":"{hash}","best_bid":"0.18","best_ask":"0.19"}}],"timestamp":"1793802284739","event_type":"price_change"}}"# + ) + } + + #[test] + fn a_polymarket_change_has_one_identity_on_every_connection_its_hash_and_size_tell_two_apart() { + let first = polymarket(&price_change("2d0c", "2306.33")); + assert_eq!(first.len(), 1); + assert_eq!(first[0].symbol, "a"); + assert_eq!(first[0].exchange_ms, 1_793_802_284_739); + // The same frame on another connection: the same copy. + assert_eq!(polymarket(&price_change("2d0c", "2306.33")), first); + // Only the hash, or only the size, differs: another change. + for other in [price_change("c000", "2306.33"), price_change("2d0c", "10")] { + assert_ne!(polymarket(&other)[0].identity, first[0].identity); + } + } + + fn kalshi_change(seq: u64, ticker: &str, clock: u64) -> String { + format!( + r#"{{"type":"orderbook_delta","sid":1,"seq":{seq},"msg":{{"market_ticker":"{ticker}","market_id":"01a1","clock":{clock},"price_dollars":"0.8100","delta_fp":"-1.50","side":"no","ts":"2026-11-11T18:36:37.012Z","ts_ms":1794422197012}},"sending_ts_ms":1794422197023}}"# + ) + } + + /// One connection's copies of `frames`, each a (ticker, clock), in order. + fn kalshi(frames: &[(&str, u64)]) -> Vec { + let mut session = kalshi::Session::default(); + let mut seen = KalshiSeen::default(); + (1..) + .zip(frames) + .flat_map(|(seq, &(ticker, clock))| { + session.decode(&kalshi_change(seq, ticker, clock)).unwrap() + }) + .flat_map(|event| Kalshi::copies(&mut seen, &event)) + .map(|copy| copy.identity) + .collect() + } + + #[test] + fn identical_kalshi_changes_under_one_clock_are_two_changes_on_every_connection() { + // Two identical changes under clock 7, then the same again under 8. + let one = kalshi(&[("M", 7), ("M", 7), ("M", 8)]); + assert_eq!(one.len(), 3); + assert_ne!(one[0], one[1]); + assert_ne!(one[2], one[0]); + // A new clock counts from one again. + assert!(one[2].ends_with("/1"), "{}", one[2]); + // Another connection: the same three changes. + assert_eq!(kalshi(&[("M", 7), ("M", 7), ("M", 8)]), one); + } + + #[test] + fn each_kalshi_market_counts_its_identical_changes_on_its_own() { + // Another market's change between M's two identical ones. + let copies = kalshi(&[("M", 7), ("N", 9), ("M", 7)]); + assert_ne!(copies[0], copies[2]); + assert!(copies[2].ends_with("/2"), "{}", copies[2]); + } + + #[test] + fn the_busiest_come_first_and_too_few_stop_the_run() { + // Ten markets, "m9" with the most changes, "m8" next. + let mut changes: BTreeMap = (0..10).map(|n| (format!("m{n}"), n)).collect(); + assert_eq!(busiest(changes.clone(), 2).unwrap(), ["m9", "m8"]); + // Nine: nothing to measure. + changes.remove("m0"); + assert!(busiest(changes, 2).is_err()); + } +} diff --git a/src/main.rs b/src/main.rs index 11f0244..6b029c2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,8 @@ //! resync), all writing through one writer (SBE). //! Usage: `engineering-challenge EXCHANGE [OUTPUT]`, EXCHANGE one of //! `polymarket`, `kalshi` or `kalshi-demo`, OUTPUT defaulting to stdout; logs -//! go to stderr. +//! go to stderr. `engineering-challenge latency ...` runs the latency probe +//! instead (`src/latency.rs`). use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -22,6 +23,7 @@ macro_rules! log { mod core; mod kalshi; +mod latency; mod polymarket; use engineering_challenge::book::Output; @@ -29,11 +31,14 @@ use engineering_challenge::book::Output; /// `AlignedSbeReader::open` reads nothing else. const CHUNK: usize = 64 << 20; -const USAGE: &str = "usage: engineering-challenge [OUTPUT]"; +const USAGE: &str = "usage: engineering-challenge [OUTPUT], or latency OUTPUT"; fn main() -> anyhow::Result<()> { let mut args = std::env::args().skip(1); let exchange = args.next().ok_or_else(|| anyhow::anyhow!(USAGE))?; + if exchange == "latency" { + return latency::run(args); + } let output = args.next().unwrap_or_else(|| "/dev/stdout".to_string()); // Before anything starts: a key that can't be read stops the run here. match exchange.as_str() { diff --git a/src/polymarket/feed.rs b/src/polymarket/feed.rs index 272329c..1b9dd06 100644 --- a/src/polymarket/feed.rs +++ b/src/polymarket/feed.rs @@ -105,7 +105,7 @@ struct RecentChange { /// The level's quantity before it (0 if absent): what undoes it. previous: i64, hash: Hash, - /// When its first copy came: a later copy's lag is measured from it. + /// When its first copy came: a later copy's lateness is measured from it. received_at: DateTime, /// How many copies came: one, of a symbol on two connections, is a change /// the other missed. @@ -123,7 +123,7 @@ pub struct Chained { /// By connection: the changes it delivered that don't fit yet. held: BTreeMap>, /// When the book last moved: held changes older than this, on a - /// connection whose changes still don't fit, are that connection's lag. + /// connection whose changes still don't fit, are late on that connection. advanced_at: DateTime, /// The connection a fresh book was asked from, and whether it is written /// whatever it holds: the connection followed changed. @@ -332,7 +332,7 @@ pub struct Chain { /// The symbols with held changes: their ceilings are checked. waiting: BTreeSet, /// How long after its first copy each duplicate came this minute, µs. - lags: Vec, + lateness: Vec, } impl Chain { @@ -730,7 +730,7 @@ impl Chain { change, received_at, &mut self.counts, - &mut self.lags, + &mut self.lateness, effects, ); if symbol.rule.held.is_empty() { @@ -746,7 +746,7 @@ impl Chain { /// A verified symbol's change: dropped if the book has it, else held on /// its connection until a tail of its held changes fits; each fit may let /// another connection's held changes fit in turn. - // The rule's fields it needs (the counts, the lags), borrowed apart from + // The rule's fields it needs (the counts, the lateness), borrowed apart from // the symbol, which its callers hold out of the shell. #[allow(clippy::too_many_arguments)] fn first_one_wins( @@ -756,7 +756,7 @@ impl Chain { change: Held, now: DateTime, counts: &mut Counts, - lags: &mut Vec, + lateness: &mut Vec, effects: &mut Vec, ) { if let Some(first) = symbol @@ -767,8 +767,8 @@ impl Chain { { counts.duplicates += 1; first.copies = first.copies.saturating_add(1); - if let Some(lag) = (change.received_at - first.received_at).num_microseconds() { - lags.push(lag); + if let Some(late) = (change.received_at - first.received_at).num_microseconds() { + lateness.push(late); } return; } @@ -952,11 +952,11 @@ impl Rule for Chain { } fn log_minute(&mut self, minute: &Minute) { - self.lags.sort_unstable(); - let lag = |quantile: f64| { - let index = ((self.lags.len() as f64 * quantile) as usize) - .min(self.lags.len().saturating_sub(1)); - self.lags + self.lateness.sort_unstable(); + let late = |quantile: f64| { + let index = ((self.lateness.len() as f64 * quantile) as usize) + .min(self.lateness.len().saturating_sub(1)); + self.lateness .get(index) .map_or(0.0, |µs| micros as f64 / 1_000.0) }; @@ -965,8 +965,8 @@ impl Rule for Chain { "polymarket: book thread {}: {} symbols; so far {} changes written ({} verified), {} duplicates, \ {} rescued, {} held dropped, {} chains broken, {} resyncs, {} full books behind ours, \ {} subscription gaps, {} losses, {} switches, {} quiet subscriptions resubscribed, \ - {} markets reported quiet again, {} reconnect books judged ({} losses); second copies \ - {:.1} ms late at the median, {:.1} ms at p99; this minute {} changes received, at most \ + {} markets reported quiet again, {} reconnect books judged ({} losses); duplicates' \ + lateness {:.1} ms at the median, {:.1} ms at p99; this minute {} changes received, at most \ {} events waited", minute.number, minute.symbols, @@ -985,12 +985,12 @@ impl Rule for Chain { minute.quiet_reported, counts.reconnect_books, counts.reconnect_losses, - lag(0.5), - lag(0.99), + late(0.5), + late(0.99), minute.received, minute.peak_waiting ); - self.lags.clear(); + self.lateness.clear(); } } @@ -1709,7 +1709,7 @@ mod tests { both.rule.counts ); assert_eq!(both.rule.counts.losses, 0); - assert!(!both.rule.lags.is_empty() || both.rule.counts.duplicates == 0); + assert!(!both.rule.lateness.is_empty() || both.rule.counts.duplicates == 0); } /// The first copy's receive time is the one written. @@ -2711,6 +2711,8 @@ mod tests { Effect::Send(..) => false, Effect::Report(Report::Loads(loads)) => loads.contains_key(&markets[0].condition_id), Effect::Report(Report::Quiet { market, .. }) => *market == markets[0].condition_id, + // Polymarket's rule reports none. + Effect::Report(Report::Lateness(_)) => true, }; assert!(!effects.iter().any(mentions)); // The minute Snapshots of the 38 others, twice.