feat(polymarket): track every live market over load-balanced lanes

An orchestrator keeps the tracked set equal to Gamma's live markets (a scan at
startup and every hour, new and closed markets every minute, reaching back 15
minutes for Gamma's indexing delay) and spreads them over lanes, each a
connection and a book thread, all writing through the one writer. Lanes report
each market's load in bytes per second; the orchestrator puts new markets on
the least-loaded lane and moves the busiest markets off lanes over a 300 KB/s
budget, busiest lane first. Differing books our own subscribe explains are
subscription gaps, not losses, and every first book after a move is logged,
so reconcile's mismatches are all accounted for in the log. A 75-minute live
run tracked ~195k markets (~390k symbols) in 89 lanes.
This commit is contained in:
pascal committed 2026-11-11 06:28:25 +00:00
1 parent adb7fee8d8
commit 8114a51dc2
18 files changed
+2826 -320

No files matched your search

+542 -105
View File
@@ -1,32 +1,58 @@
//! The book thread: parses market-channel frames, folds them into one book per
//! symbol, checks them against Polymarket's book hash, resyncs when a check
//! fails, and emits Deltas and Snapshots. The rules are in
//! `roadmap/02. book pipeline.md`; the hash formula in `docs/notes.md`.
//! fails, and emits Deltas and Snapshots. The protocol facts and the hash
//! formula are in `docs/notes.md`, the terms (loss, resync, subscription gap,
//! load) in `GLOSSARY.md`.
use std::collections::{BTreeMap, VecDeque};
use std::collections::{BTreeMap, HashMap, VecDeque};
use std::time::Duration;
use chrono::{DateTime, DurationRound, TimeDelta, Utc};
use crossbeam::channel::{Receiver, RecvTimeoutError, Sender};
use crossbeam::channel::{Receiver, Sender, never, select};
use engineering_challenge::Side;
use engineering_challenge::util::dtos::challenge_orderbook::PriceLevelUpdateDomain;
use serde::{Deserialize, Serialize};
use sha1::{Digest, Sha1};
use super::Market;
use super::socket::Frame;
use super::socket::{Command, Frame};
use engineering_challenge::book::{Book, Output, format_fixed, parse_fixed};
/// A market measured for less than this waits for the next load report: over
/// a few seconds, its opening books alone would read as a burst.
const MIN_MEASURED: TimeDelta = TimeDelta::seconds(30);
/// How far back a full book is recognised as behind our stream: the lag seen
/// live is a few ms, a handful of changes.
const RECENT_CHANGES: usize = 64;
/// What the book thread does with a frame: write output, or ask the socket
/// thread for a resync.
/// What the book thread does with a frame or a command: write output, ask its
/// connection for something, or report its load to the orchestrator.
#[derive(Debug)]
pub enum Effect {
Write(Output),
Resync(String),
Send(Command),
Report(Loads),
}
/// Each market's load (`conditionId` → bytes per second), every market of the
/// lane measured for at least [`MIN_MEASURED`].
pub type Loads = HashMap<String, u64>;
/// A lane's loads since its last report, for the orchestrator.
#[derive(Debug)]
pub struct LoadReport {
pub lane: usize,
pub loads: Loads,
}
/// What the orchestrator tells a lane's book thread.
#[derive(Debug)]
pub enum LaneCommand {
Add(Market),
/// An Add for a market moved here from another lane.
MoveIn(Market),
Remove(Market),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -34,7 +60,7 @@ enum State {
/// No book yet since subscribing: changes are dropped until it comes.
AwaitingBook,
/// Its last full book matched its hash: every change is checked.
Live,
Verified,
/// A check failed and a fresh book is asked for: changes are applied and
/// written unchecked until it comes. After a trade, it usually equals ours.
Resyncing,
@@ -53,6 +79,12 @@ struct Symbol {
/// Its last [`RECENT_CHANGES`] changes: a full book carrying the hash of
/// one of them is the exchange's state after it.
recent_changes: VecDeque<RecentChange>,
/// Whether a change has been verified since its opening book: until one
/// is, a differing fresh book is the gap our subscribe opened.
verified_since_book: bool,
/// Moved in from another lane, no book yet: its first book holds the
/// move's subscription gap, which only the old lane's book would show.
moved_in: bool,
// The other inputs of the book hash, as the exchange writes them.
market: String,
tick_size: String,
@@ -62,6 +94,30 @@ struct Symbol {
}
impl Symbol {
fn new(market: &Market) -> Self {
Self {
state: State::AwaitingBook,
book: Book::default(),
exchange_timestamp: DateTime::UNIX_EPOCH,
open_batch: None,
recent_changes: VecDeque::new(),
verified_since_book: false,
moved_in: false,
market: String::new(),
tick_size: String::new(),
last_trade_price: String::new(),
min_order_size: market.min_order_size.clone(),
neg_risk: market.neg_risk,
}
}
/// A change matched the exchange's hash: counted in `verified_changes`,
/// and the book was right since its opening book.
fn verified(&mut self, verified_changes: &mut u64) {
*verified_changes += 1;
self.verified_since_book = true;
}
/// Asks for a fresh book, unless one is already coming.
fn resync(&mut self, token: &str, effects: &mut Vec<Effect>) {
if matches!(self.state, State::AwaitingBook | State::Resyncing) {
@@ -77,7 +133,7 @@ impl Symbol {
self.state = State::Resyncing;
}
self.open_batch = None;
effects.push(Effect::Resync(token.to_string()));
effects.push(Effect::Send(Command::Resync(token.to_string())));
}
/// Whether `book` is ours as it was after one of our recent changes: ours
@@ -102,6 +158,29 @@ impl Symbol {
}
}
/// A market's frame bytes since `since`.
struct Measure {
bytes: u64,
since: DateTime<Utc>,
}
impl Measure {
/// Bytes per second since `since`, then starts again from `now`; none
/// before [`MIN_MEASURED`], the bytes kept for the next report.
fn take(&mut self, now: DateTime<Utc>) -> Option<u64> {
let measured = now - self.since;
if measured < MIN_MEASURED {
return None;
}
let load = self.bytes * 1_000 / measured.num_milliseconds() as u64;
*self = Measure {
bytes: 0,
since: now,
};
Some(load)
}
}
/// A change as applied, with the exchange's hash of the book after it.
struct RecentChange {
side: Side,
@@ -113,17 +192,25 @@ struct RecentChange {
}
pub struct Feed {
/// Its lane's number, for the logs.
lane: usize,
symbols: BTreeMap<String, Symbol>,
/// Bytes of frames per market since its last load report.
measures: HashMap<String, Measure>,
/// The receive time of the last frame applied: the time a Snapshot's book
/// is exact as of.
last_received_at: Option<DateTime<Utc>>,
next_snapshots: Option<DateTime<Utc>>,
/// The next wall-clock minute: its Snapshots, load report and log line.
next_minute_at: Option<DateTime<Utc>>,
verified_changes: u64,
resyncs: u64,
/// Full books that were a state we had already passed (see `on_book`).
books_behind: u64,
/// Fresh books ahead of our stream by changes our resync kept from us.
resync_gaps: u64,
/// Fresh books holding changes our own subscribe kept from us.
subscription_gaps: u64,
/// First books of markets moved in, each logged: whether one holds a
/// subscription gap only the old lane's book would tell.
books_after_moves: u64,
/// Full books whose levels differed from ours otherwise.
losses: u64,
/// The most frames waiting for the book thread since the last minute log:
@@ -132,42 +219,56 @@ pub struct Feed {
}
impl Feed {
pub fn new(markets: &[Market]) -> Self {
let symbols = markets
.iter()
.flat_map(|market| {
market.tokens.iter().map(|token| {
let symbol = Symbol {
state: State::AwaitingBook,
book: Book::default(),
exchange_timestamp: DateTime::UNIX_EPOCH,
open_batch: None,
recent_changes: VecDeque::new(),
market: String::new(),
tick_size: String::new(),
last_trade_price: String::new(),
min_order_size: market.min_order_size.clone(),
neg_risk: market.neg_risk,
};
(token.clone(), symbol)
})
})
.collect();
pub fn new(lane: usize) -> Self {
Self {
symbols,
lane,
symbols: BTreeMap::new(),
measures: HashMap::new(),
last_received_at: None,
next_snapshots: None,
next_minute_at: None,
verified_changes: 0,
resyncs: 0,
books_behind: 0,
resync_gaps: 0,
subscription_gaps: 0,
books_after_moves: 0,
losses: 0,
peak_waiting: 0,
}
}
pub fn tokens(&self) -> Vec<String> {
self.symbols.keys().cloned().collect()
/// Tracks a market's symbols, then asks the connection to subscribe them:
/// a book can't arrive for a symbol not yet known. Its load is measured
/// from `now`.
pub fn add_market(&mut self, market: &Market, now: DateTime<Utc>) -> Vec<Effect> {
for token in &market.tokens {
self.symbols.insert(token.clone(), Symbol::new(market));
}
let measure = Measure {
bytes: 0,
since: now,
};
self.measures.insert(market.condition_id.clone(), measure);
vec![Effect::Send(Command::Subscribe(market.tokens.clone()))]
}
/// [`Feed::add_market`] for a market moved in from another lane.
pub fn move_in(&mut self, market: &Market, now: DateTime<Utc>) -> Vec<Effect> {
let effects = self.add_market(market, now);
for token in &market.tokens {
self.symbols.get_mut(token).expect("just added").moved_in = true;
}
effects
}
/// Forgets a market's symbols and unsubscribes them; nothing is written
/// for them. Their frames still on the way are ignored, as any unknown
/// token's.
pub fn remove_market(&mut self, market: &Market) -> Vec<Effect> {
for token in &market.tokens {
self.symbols.remove(token);
}
self.measures.remove(&market.condition_id);
vec![Effect::Send(Command::Unsubscribe(market.tokens.clone()))]
}
pub fn on_frame(&mut self, received_at: DateTime<Utc>, text: &str) -> Vec<Effect> {
@@ -186,51 +287,74 @@ impl Feed {
}
},
};
// A frame is one market's change, or books of several: each book takes
// its share.
let share = (text.len() / events.len().max(1)) as u64;
for event in events {
let market = match &event {
Event::Book(book) => Some(&book.market),
Event::PriceChange(change) => Some(&change.market),
Event::TickSizeChange { market, .. } | Event::LastTradePrice { market } => {
market.as_ref()
}
Event::Other => None,
};
if let Some(measure) = market.and_then(|market| self.measures.get_mut(market)) {
measure.bytes += share;
}
match event {
Event::Book(book) => self.on_book(received_at, book, &mut effects),
Event::PriceChange(change) => {
self.on_price_change(received_at, change, &mut effects)
}
Event::TickSizeChange { asset_id } => {
Event::TickSizeChange { asset_id, .. } => {
if let Some(symbol) = self.symbols.get_mut(&asset_id) {
log!("polymarket: {asset_id}: tick size changed, resync");
symbol.resync(&asset_id, &mut effects);
}
}
// Among them `last_trade_price`: the hash takes a new last
// trade price before this event, and not always its value, so
// a fresh book brings it instead (docs/notes.md).
Event::Other => {}
// The hash takes a new last trade price before this event,
// and not always its value, so a fresh book brings it instead
// (docs/notes.md).
Event::LastTradePrice { .. } | Event::Other => {}
}
}
self.resyncs += count_resyncs(&effects[ticked..]);
effects
}
/// Once `now` reaches the next wall-clock minute: emits the minute
/// Snapshots, and asks again for the fresh books still not come.
/// Once `now` reaches the next wall-clock minute: reports the minute's
/// load, emits the minute Snapshots, and asks again for the fresh books
/// still not come.
pub fn tick(&mut self, now: DateTime<Utc>) -> Vec<Effect> {
let next = *self.next_snapshots.get_or_insert_with(|| next_minute(now));
let next = *self.next_minute_at.get_or_insert_with(|| next_minute(now));
if now < next {
return Vec::new();
}
self.next_snapshots = Some(next_minute(now));
self.next_minute_at = Some(next_minute(now));
let loads = self
.measures
.iter_mut()
.filter_map(|(market, measure)| Some((market.clone(), measure.take(now)?)))
.collect();
let mut effects = vec![Effect::Report(loads)];
let Some(received_at) = self.last_received_at else {
return Vec::new();
return effects;
};
log!(
"polymarket: so far {} changes verified, {} resyncs, {} full books behind ours, \
{} resync gaps, {} losses; at most {} frames waited this minute",
"polymarket: lane {}: {} symbols; so far {} changes verified, {} resyncs, {} full books behind ours, \
{} subscription gaps, {} books after moves, {} losses; at most {} frames waited this minute",
self.lane,
self.symbols.len(),
self.verified_changes,
self.resyncs,
self.books_behind,
self.resync_gaps,
self.subscription_gaps,
self.books_after_moves,
self.losses,
self.peak_waiting
);
self.peak_waiting = 0;
let mut effects = Vec::new();
for (token, symbol) in &mut self.symbols {
if symbol.state != State::AwaitingBook {
effects.push(Effect::Write(symbol.book.snapshot(
@@ -249,8 +373,8 @@ impl Feed {
/// A full book replaces ours, brought forward first if it is behind our
/// stream, and is written as a Snapshot when it is the symbol's first
/// since subscribing, or its levels differ from ours (a resync gap or a
/// loss). Then it is checked against its own hash, as received.
/// since subscribing, or its levels differ from ours (a subscription gap
/// or a loss). Then it is checked against its own hash, as received.
fn on_book(
&mut self,
received_at: DateTime<Utc>,
@@ -285,6 +409,13 @@ impl Feed {
let verified = hash(&token, symbol, &book, &message.timestamp) == message.hash;
let first = symbol.state == State::AwaitingBook;
if first && symbol.moved_in {
// The check matches it with the Snapshot below if that differs
// from the old lane's last.
symbol.moved_in = false;
self.books_after_moves += 1;
log!("polymarket: {token}: first book after a move, a subscription gap if it differs");
}
let mut book = book;
let book_time = exchange_timestamp;
let mut exchange_timestamp = exchange_timestamp;
@@ -312,17 +443,24 @@ impl Feed {
brought_forward = true;
}
let differs = !first && book != symbol.book;
// A fresh book answering our resync can also be ahead: between our
// unsubscribe and subscribe the server sent us none of its changes.
let gap = differs
&& !brought_forward
// A subscribe hides a few ms of changes (docs/notes.md): a fresh book
// answering our resync can be ahead of our stream, since between our
// unsubscribe and subscribe the server sent none of its changes; and
// so can any book before a change was verified since the opening one.
let opening = !symbol.verified_since_book;
let resync = !brought_forward
&& complete
&& symbol.state == State::Resyncing
&& book_time >= symbol.exchange_timestamp;
let kind = if gap { "resync gap" } else { "loss" };
let gap = differs && (opening || resync);
let kind = match (gap, opening) {
(false, _) => "loss",
(true, true) => "subscription gap (opening)",
(true, false) => "subscription gap (resync)",
};
if differs {
if gap {
self.resync_gaps += 1;
self.subscription_gaps += 1;
} else {
self.losses += 1;
}
@@ -347,7 +485,7 @@ impl Feed {
}
if verified {
symbol.state = State::Live;
symbol.state = State::Verified;
} else if complete {
if symbol.state != State::Unverifiable {
log!(
@@ -355,10 +493,10 @@ impl Feed {
);
}
symbol.state = State::Unverifiable;
} else if matches!(symbol.state, State::AwaitingBook | State::Live) {
// Its last trade price may be stale: a fresh book carries it. Live
// first, since it has a book now, which resync() requires.
symbol.state = State::Live;
} else if matches!(symbol.state, State::AwaitingBook | State::Verified) {
// Its last trade price may be stale: a fresh book carries it.
// Verified first, since it has a book now, which resync() requires.
symbol.state = State::Verified;
symbol.resync(&token, effects);
}
}
@@ -384,7 +522,7 @@ impl Feed {
.take_if(|(_, batch_hash)| *batch_hash != change.hash)
{
if hash(&token, symbol, &symbol.book, &timestamp) == batch_hash {
self.verified_changes += 1;
symbol.verified(&mut self.verified_changes);
} else {
symbol.resync(&token, effects);
}
@@ -416,9 +554,9 @@ impl Feed {
hash: change.hash.clone(),
});
symbol.exchange_timestamp = exchange_timestamp;
if symbol.state == State::Live {
if symbol.state == State::Verified {
if hash(&token, symbol, &symbol.book, &message.timestamp) == change.hash {
self.verified_changes += 1;
symbol.verified(&mut self.verified_changes);
} else {
symbol.open_batch = Some((message.timestamp.clone(), change.hash));
}
@@ -435,8 +573,8 @@ impl Feed {
}
}
/// The book thread: runs until the socket thread is gone, then drops its
/// senders so the writer thread finishes too.
/// A lane's book thread: runs until its connection is gone, then drops its
/// senders so the writer thread finishes once every lane has.
// ponytail: unbounded channels never block the socket read nor drop a frame;
// the ceiling is memory if a consumer stays behind. The minute log's peak of
// frames waiting makes that visible; bounded channels are a constructor change
@@ -444,20 +582,41 @@ impl Feed {
pub fn run(
mut feed: Feed,
frames: Receiver<Frame>,
commands: Receiver<LaneCommand>,
outputs: Sender<Output>,
resyncs: Sender<String>,
connection: Sender<Command>,
reports: Sender<LoadReport>,
) {
let no_commands = never();
let mut orchestrator_gone = false;
loop {
let until_next_minute = (next_minute(Utc::now()) - Utc::now())
.to_std()
.unwrap_or(Duration::ZERO);
let effects = match frames.recv_timeout(until_next_minute) {
Ok((received_at, text)) => {
feed.peak_waiting = feed.peak_waiting.max(frames.len());
feed.on_frame(received_at, text.as_str())
}
Err(RecvTimeoutError::Timeout) => feed.tick(Utc::now()),
Err(RecvTimeoutError::Disconnected) => return,
let commands = if orchestrator_gone {
&no_commands
} else {
&commands
};
let effects = select! {
recv(frames) -> frame => match frame {
Ok((received_at, text)) => {
feed.peak_waiting = feed.peak_waiting.max(frames.len());
feed.on_frame(received_at, text.as_str())
}
Err(_) => return,
},
recv(commands) -> command => match command {
Ok(LaneCommand::Add(market)) => feed.add_market(&market, Utc::now()),
Ok(LaneCommand::MoveIn(market)) => feed.move_in(&market, Utc::now()),
Ok(LaneCommand::Remove(market)) => feed.remove_market(&market),
// It is stopping: the connection's end still decides ours.
Err(_) => {
orchestrator_gone = true;
Vec::new()
}
},
default(until_next_minute) => feed.tick(Utc::now()),
};
for effect in effects {
match effect {
@@ -466,8 +625,15 @@ pub fn run(
return log!("polymarket: the writer thread is gone, stopping");
}
}
// The socket thread may be gone already, while stopping.
Effect::Resync(token) => _ = resyncs.send(token),
// The connection may be gone already, while stopping, and the
// orchestrator too.
Effect::Send(command) => _ = connection.send(command),
Effect::Report(loads) => {
_ = reports.send(LoadReport {
lane: feed.lane,
loads,
})
}
}
}
}
@@ -531,11 +697,19 @@ enum Events {
enum Event {
Book(BookMessage),
PriceChange(PriceChangeMessage),
/// Its wire shape was never observed: only the symbol is read, and it
/// resyncs, since the fresh book carries the new tick size.
/// Only the symbol and its market are read, and it resyncs, since the fresh book carries
/// the new tick size. The server sends each one twice: the second finds
/// the symbol resyncing and asks for nothing (docs/notes.md).
/// `market` was on every one seen; without it, only the load misses it.
TickSizeChange {
market: Option<String>,
asset_id: String,
},
LastTradePrice {
market: Option<String>,
},
/// Not counted in any load: the others come only with
/// `custom_feature_enabled`, which we don't ask for.
#[serde(other)]
Other,
}
@@ -560,6 +734,7 @@ struct LevelMessage {
#[derive(Deserialize)]
struct PriceChangeMessage {
market: String,
timestamp: String,
price_changes: Vec<ChangeMessage>,
}
@@ -600,7 +775,7 @@ fn next_minute(now: DateTime<Utc>) -> DateTime<Utc> {
fn count_resyncs(effects: &[Effect]) -> u64 {
effects
.iter()
.filter(|effect| matches!(effect, Effect::Resync(_)))
.filter(|effect| matches!(effect, Effect::Send(Command::Resync(_))))
.count() as u64
}
@@ -615,6 +790,17 @@ mod tests {
use super::*;
use crate::polymarket::fixture_markets;
/// A lane's feed holding `markets`, as the orchestrator's Adds leave it
/// at the start of the recording.
fn feed_of(markets: &[Market]) -> Feed {
let mut feed = Feed::new(0);
let start = fixture_frames()[0].0;
for market in markets {
feed.add_market(market, start);
}
feed
}
/// The recorded live feed for the fixture's markets (tests/fixtures/README.md).
fn fixture_frames() -> Vec<(DateTime<Utc>, String)> {
#[derive(Deserialize)]
@@ -653,8 +839,8 @@ mod tests {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Resync(token) => Some(token.as_str()),
Effect::Write(_) => None,
Effect::Send(Command::Resync(token)) => Some(token.as_str()),
_ => None,
})
.collect()
}
@@ -685,7 +871,7 @@ mod tests {
/// the recording must replay without a single loss.
#[test]
fn replay_has_no_loss() {
let mut feed = Feed::new(&fixture_markets());
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &fixture_frames());
assert_eq!(feed.losses, 0);
assert!(
@@ -712,7 +898,7 @@ mod tests {
let markets = fixture_markets();
// The full replay, noting the change frames whose symbols are all
// checked when they come.
let mut control = Feed::new(&markets);
let mut control = feed_of(&markets);
let mut candidates = Vec::new();
let control_effects: Vec<Vec<Effect>> = frames
.iter()
@@ -723,7 +909,7 @@ mod tests {
&& !tokens.is_empty()
&& tokens
.iter()
.all(|token| control.symbols[token].state == State::Live)
.all(|token| control.symbols[token].state == State::Verified)
{
candidates.push(index);
}
@@ -733,7 +919,7 @@ mod tests {
let lost = candidates[0];
let lost_tokens = changed_tokens(&frames[lost].1);
let mut feed = Feed::new(&markets);
let mut feed = feed_of(&markets);
replay(&mut feed, &frames[..lost]);
let (caught, token) = frames
.iter()
@@ -766,7 +952,7 @@ mod tests {
let effects = feed.on_frame(frames[caught].0, &book.to_string());
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!(feed.losses, losses + 1);
assert_eq!(feed.symbols[&token].state, State::Live);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
/// A feed replayed past a mid-stream book and its symbol's next change, and
@@ -788,7 +974,7 @@ mod tests {
let moved_on = (stale + 1..frames.len())
.find(|&index| changed_tokens(&frames[index].1).contains(&token))
.unwrap();
let mut feed = Feed::new(&fixture_markets());
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &frames[..=moved_on]);
(feed, token, frames[moved_on].0, frames[stale].1.clone())
}
@@ -839,25 +1025,26 @@ mod tests {
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(
(feed.books_behind, feed.resync_gaps, feed.losses),
(feed.books_behind, feed.subscription_gaps, feed.losses),
(1, 0, 0)
);
assert_eq!(feed.symbols[&token].book, ours);
assert_eq!(feed.symbols[&token].state, State::Live);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
/// Between our unsubscribe and subscribe the server sends no changes, so
/// the fresh book can hold some we never got: a resync gap, not a loss.
/// the fresh book can hold some we never got: a subscription gap, not a
/// loss.
#[test]
fn a_fresh_book_ahead_of_our_stream_is_a_resync_gap() {
fn a_fresh_book_ahead_of_our_stream_is_a_subscription_gap() {
let frames = fixture_frames();
let now = frames.last().unwrap().0;
let mut feed = Feed::new(&fixture_markets());
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &frames);
let (token, symbol) = feed
.symbols
.iter_mut()
.find(|(_, symbol)| symbol.state == State::Live)
.find(|(_, symbol)| symbol.state == State::Verified)
.unwrap();
let token = token.clone();
symbol.resync(&token, &mut Vec::new());
@@ -870,9 +1057,9 @@ mod tests {
let effects = feed.on_frame(now, &book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!((feed.resync_gaps, feed.losses), (1, 0));
assert_eq!((feed.subscription_gaps, feed.losses), (1, 0));
assert_eq!(feed.symbols[&token].book, ahead);
assert_eq!(feed.symbols[&token].state, State::Live);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
/// The level test must not mistake our wrong book for a newer one: we
@@ -941,7 +1128,7 @@ mod tests {
for market in &mut markets {
market.min_order_size = "6".to_string();
}
let mut feed = Feed::new(&markets);
let mut feed = feed_of(&markets);
let frames = fixture_frames();
let opening = feed.on_frame(frames[0].0, &frames[0].1);
assert_eq!(snapshot_symbols(&opening).len(), 40);
@@ -963,7 +1150,7 @@ mod tests {
#[test]
fn frames_that_are_not_events_are_skipped() {
let mut feed = Feed::new(&fixture_markets());
let mut feed = feed_of(&fixture_markets());
let at = fixture_frames()[0].0;
for text in [
"PONG",
@@ -976,18 +1163,268 @@ mod tests {
}
}
/// A market's opening books, as a subscribe brings them.
fn opening_books(frames: &[(DateTime<Utc>, String)], market: &Market) -> String {
let opening: Vec<serde_json::Value> = serde_json::from_str(&frames[0].1).unwrap();
let books: Vec<_> = opening
.iter()
.filter(|book| {
market
.tokens
.iter()
.any(|token| book["asset_id"] == **token)
})
.collect();
serde_json::to_string(&books).unwrap()
}
#[test]
fn a_moved_market_logs_its_first_books_once() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut feed = feed_of(&markets[1..]);
replay(&mut feed, &frames[1..100]);
let effects = feed.move_in(&markets[0], frames[100].0);
assert!(matches!(
effects.as_slice(),
[Effect::Send(Command::Subscribe(tokens))] if *tokens == markets[0].tokens
));
let books = opening_books(&frames, &markets[0]);
let effects = feed.on_frame(frames[100].0, &books);
assert_eq!(snapshot_symbols(&effects).len(), 2);
assert_eq!(feed.books_after_moves, 2);
// Later books are the lane's own business.
feed.on_frame(frames[101].0, &books);
assert_eq!(feed.books_after_moves, 2);
assert_eq!((feed.subscription_gaps, feed.losses), (0, 0));
}
#[test]
fn a_market_added_mid_run_is_subscribed_and_its_first_book_written() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut feed = feed_of(&markets[1..]);
replay(&mut feed, &frames[1..100]);
let effects = feed.add_market(&markets[0], frames[100].0);
assert!(matches!(
effects.as_slice(),
[Effect::Send(Command::Subscribe(tokens))] if *tokens == markets[0].tokens
));
let effects = feed.on_frame(frames[100].0, &opening_books(&frames, &markets[0]));
let mut written = snapshot_symbols(&effects);
written.sort();
let mut tokens: Vec<&str> = markets[0].tokens.iter().map(String::as_str).collect();
tokens.sort();
assert_eq!(written, tokens);
assert!(
markets[0]
.tokens
.iter()
.all(|token| feed.symbols[token].state == State::Verified)
);
}
#[test]
fn a_removed_market_is_unsubscribed_and_written_no_more() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut feed = feed_of(&markets);
replay(&mut feed, &frames[..1]);
let removed = &markets[0].tokens;
let effects = feed.remove_market(&markets[0]);
assert!(matches!(
effects.as_slice(),
[Effect::Send(Command::Unsubscribe(tokens))] if tokens == removed
));
// The rest of the recording crosses two minutes: no Delta, Snapshot or
// resync names a removed token.
let effects = replay(&mut feed, &frames[1..]);
let mentions = |effect: &Effect| match effect {
Effect::Write(Output::Snapshot(snapshot)) => removed.contains(&snapshot.symbol),
Effect::Write(Output::Delta(delta)) => removed.contains(&delta.symbol),
Effect::Send(Command::Resync(token)) => removed.contains(token),
Effect::Send(_) => false,
Effect::Report(loads) => loads.contains_key(&markets[0].condition_id),
};
assert!(!effects.iter().any(mentions));
// The minute Snapshots of the 38 others, twice.
assert_eq!(snapshot_symbols(&effects).len(), 38 * 2);
}
/// Seen live: our subscribe hides changes made just before the opening
/// book reaches us, so the first fresh book differs at a level none of our
/// changes touched, before any change was verified.
#[test]
fn a_differing_book_before_any_verified_change_is_a_subscription_gap() {
let frames = fixture_frames();
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &frames[..1]);
let (token, symbol) = feed
.symbols
.iter()
.find(|(_, symbol)| symbol.state == State::Verified)
.unwrap();
let token = token.clone();
assert!(!symbol.verified_since_book);
let mut exchange = symbol.book.clone();
exchange.set(Side::Buy, 1_000, 123_000_000);
let book = fresh_book(&token, symbol, &exchange);
let effects = feed.on_frame(frames[1].0, &book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!((feed.subscription_gaps, feed.losses), (1, 0));
assert_eq!(feed.symbols[&token].book, exchange);
}
/// Once a change has been verified since the opening book, our book was
/// right then: a fresh book that differs is a loss.
#[test]
fn a_differing_book_after_a_verified_change_is_a_loss() {
let frames = fixture_frames();
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &frames[..frames.len() / 2]);
let (token, symbol) = feed
.symbols
.iter()
.find(|(_, symbol)| symbol.state == State::Verified && symbol.verified_since_book)
.unwrap();
let token = token.clone();
let mut exchange = symbol.book.clone();
exchange.set(Side::Buy, 1_000, 123_000_000);
let book = fresh_book(&token, symbol, &exchange);
let (gaps, losses) = (feed.subscription_gaps, feed.losses);
let effects = feed.on_frame(frames[frames.len() / 2].0, &book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!((feed.subscription_gaps, feed.losses), (gaps, losses + 1));
assert_eq!(feed.symbols[&token].book, exchange);
}
#[test]
fn each_minute_reports_the_bytes_per_second_of_every_market() {
let minute = "2026-11-10T16:00:00Z".parse::<DateTime<Utc>>().unwrap();
let at = |seconds| minute + TimeDelta::seconds(seconds);
let market = |id: &str| Market {
condition_id: id.to_string(),
tokens: vec![format!("{id}a"), format!("{id}b")],
min_order_size: "5".to_string(),
neg_risk: false,
};
let (busy, quiet, late) = (market("0xbusy"), market("0xquiet"), market("0xlate"));
let mut feed = Feed::new(0);
feed.add_market(&busy, at(0));
// Added halfway through the minute: measured over its 30 s.
feed.add_market(&quiet, at(30));
// Added 15 s before it: kept for the next report.
feed.add_market(&late, at(45));
let change = |id: &str| {
serde_json::json!({"event_type": "price_change", "market": id, "timestamp": "1",
"price_changes": []})
.to_string()
};
let trade = serde_json::json!({"event_type": "last_trade_price", "market": "0xbusy",
"asset_id": "0xbusya", "price": "0.5", "size": "1", "side": "BUY",
"timestamp": "1"})
.to_string();
let tick_size = serde_json::json!({"event_type": "tick_size_change", "market": "0xquiet",
"asset_id": "0xquieta", "old_tick_size": "0.01", "new_tick_size": "0.001",
"timestamp": "1"})
.to_string();
// One frame, two markets: each takes half.
let shared = format!("[{},{}]", change("0xbusy"), change("0xquiet"));
let mut reports = Vec::new();
let mut on_frame = |feed: &mut Feed, seconds, text: &str| {
for effect in feed.on_frame(at(seconds), text) {
if let Effect::Report(loads) = effect {
reports.push(loads);
}
}
};
on_frame(&mut feed, 10, &change("0xbusy"));
on_frame(&mut feed, 20, &trade);
on_frame(&mut feed, 40, &tick_size);
on_frame(&mut feed, 50, &change("0xlate"));
on_frame(&mut feed, 55, &change("0xunknown"));
// The next minute: the first report, then nothing more until the one
// after, which starts again from zero.
on_frame(&mut feed, 60, &shared);
on_frame(&mut feed, 120, &change("0xunknown"));
let bytes = |text: &String| text.len() as u64;
let half = bytes(&shared) / 2;
assert_eq!(
reports,
[
Loads::from([
(
"0xbusy".to_string(),
(bytes(&change("0xbusy")) + bytes(&trade)) / 60
),
("0xquiet".to_string(), bytes(&tick_size) / 30),
]),
Loads::from([
("0xbusy".to_string(), half / 60),
("0xquiet".to_string(), half / 60),
("0xlate".to_string(), bytes(&change("0xlate")) / 75),
]),
]
);
}
#[test]
fn the_recording_reports_every_market_once_measured_long_enough() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut feed = feed_of(&markets);
let reports: Vec<Loads> = replay(&mut feed, &frames)
.into_iter()
.filter_map(|effect| match effect {
Effect::Report(loads) => Some(loads),
_ => None,
})
.collect();
// The recording crosses two minutes, the first too soon after its
// start for any market to be measured.
let start = frames[0].0;
assert!(next_minute(start) - start < MIN_MEASURED);
assert_eq!(reports.len(), 2);
assert!(reports[0].is_empty());
let loads = &reports[1];
assert_eq!(loads.len(), markets.len());
assert!(
markets
.iter()
.all(|market| loads.contains_key(&market.condition_id))
);
assert!(loads.values().sum::<u64>() > 0);
}
#[test]
fn the_output_reconciles() {
// The book and writer threads as main runs them, fed the recording.
let (frames_tx, frames_rx) = crossbeam::channel::unbounded();
let (outputs_tx, outputs_rx) = crossbeam::channel::unbounded();
let (resyncs_tx, _resyncs_rx) = crossbeam::channel::unbounded();
let (connection_tx, _connection_rx) = crossbeam::channel::unbounded();
let (_commands_tx, commands_rx) = crossbeam::channel::unbounded();
let (reports_tx, _reports_rx) = crossbeam::channel::unbounded();
for (received_at, text) in fixture_frames() {
frames_tx.send((received_at, text.into())).unwrap();
}
drop(frames_tx);
let feed = Feed::new(&fixture_markets());
let book = std::thread::spawn(move || run(feed, frames_rx, outputs_tx, resyncs_tx));
let feed = feed_of(&fixture_markets());
let book = std::thread::spawn(move || {
run(
feed,
frames_rx,
commands_rx,
outputs_tx,
connection_tx,
reports_tx,
)
});
let path = std::env::temp_dir().join(format!("reconcile-{}.sbe", std::process::id()));
let written = crate::write(path.to_str().unwrap(), outputs_rx);
book.join().unwrap();