//! A book thread: folds the events its markets' connections send into one book //! per symbol, first one wins: a change is taken from whichever connection //! delivers it first, Polymarket's book hash saying where it belongs //! (ADR 0003). It resyncs when a change fits on no connection in time, and //! emits Deltas and Snapshots. It also looks for what first one wins hides: //! a subscription gone quiet on one connection, each minute, and symbols //! quiet on both, in a reconnect's opening books. The rules are in //! `roadmap/04. redundant connections.md` and //! `roadmap/05. subscription monitoring.md`, the protocol facts and the hash //! formula in `docs/notes.md`, the terms in `GLOSSARY.md`. use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; use std::time::Duration; use chrono::{DateTime, DurationRound, TimeDelta, Utc}; use crossbeam::channel::{Receiver, Sender, never, select}; use engineering_challenge::Side; use engineering_challenge::cause::{self, Cause}; use engineering_challenge::util::dtos::challenge_orderbook::PriceLevelUpdateDomain; use serde::Serialize; use sha1::{Digest, Sha1}; use super::Market; use super::socket::Command; use super::wire::{Arrival, BookMessage, Event, LevelMessage, PriceChangeMessage, Received}; use engineering_challenge::book::{Book, Output, format_fixed, parse_fixed}; /// A market measured for less than this waits for the next load report: over /// a few seconds, its opening books alone would read as a burst. const MIN_MEASURED: TimeDelta = TimeDelta::seconds(30); /// How many of a symbol's last changes are remembered: a slower connection's /// copy is recognised among them (it ran up to 82 behind, docs/notes.md), and /// so is a fresh book behind our stream. const RECENT_CHANGES: usize = 128; /// How long a change may be held while another connection might still deliver /// the one before it: the second delivery lagged up to 0.7 s. const HOLD: TimeDelta = TimeDelta::seconds(1); /// How long once no other connection can: only the rest of a batch is still to /// come, from the same connection (40 ms at worst). const HOLD_ALONE: TimeDelta = TimeDelta::milliseconds(50); /// How often a book thread with held changes wakes to check their ceilings. const EXPIRE_EVERY: Duration = Duration::from_millis(10); /// The most changes a connection holds for a symbol: past it, its ceiling is /// reached at once. A burst (thousands a second at a crypto market's /// 5-minute mark) would otherwise pile up until the ceiling, each one tried /// against the book. const MAX_HELD: usize = 512; /// How many tails of a connection's held changes are tried against the book: /// the whole, and the last few that start on a batch. const TAILS_TRIED: usize = 4; /// When a change's copies are counted: no second copy came later than 0.7 s. const COPIES_BY: TimeDelta = TimeDelta::seconds(2); /// A connection that delivered none of a symbol's changes in a minute while /// another delivered at least this many is quiet for it (sprint 05: crossed /// once in 21 live minutes; at 1, copies landing in the next minute would /// count). const QUIET_AGAINST: u32 = 3; /// A reconnect's opening book is judged only for a symbol unchanged this /// long: a moving symbol's books run ahead of or behind our stream. const STILL_FOR: TimeDelta = TimeDelta::seconds(10); /// What the book thread does with an event or a command: write output, ask a /// connection (by number) for something, or report to the orchestrator. #[derive(Debug)] pub enum Effect { Write(Output), Send(usize, Command), Report(Report), } /// What a book thread tells the orchestrator. #[derive(Debug, PartialEq, Eq)] pub enum Report { Loads(Loads), /// The market stayed quiet on the connection after being resubscribed /// there: its subscription should move. Quiet { market: String, connection: usize, }, } /// Each market's load (`conditionId` → bytes per second on one connection), /// every market of the book thread measured for at least [`MIN_MEASURED`]. pub type Loads = HashMap; /// What the orchestrator tells a book thread. A market's commands all go to /// its one book thread, in order. #[derive(Debug)] pub enum BookCommand { /// A connection just opened, and how to reach it: sent to every book /// thread before any command that names it. Connected(usize, Sender), /// Track the market, on these connections. Add(Market, Vec), /// The market's subscription on `from` goes to `to`. Move { market: Market, from: usize, to: usize, }, Remove(Market), } /// Polymarket's book hash, as bytes: 20 instead of 40 as text, for 128 of /// them per symbol. type Hash = [u8; 20]; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum State { /// No book yet since subscribing, or since every connection carrying it /// went down: changes are dropped until one comes. AwaitingBook, /// Its book gives the exchange's hash after every change: first one wins. Verified, /// A change fitted on no connection in time: this connection's changes /// are applied unchecked, the others' dropped, until a fresh book. Following(usize), /// A complete book didn't match its own hash, so our formula or metadata /// is wrong for it: this connection's changes are applied unchecked, for /// good. Unverifiable(usize), } /// A change that doesn't fit the book yet, kept unwritten. #[derive(Debug, Clone)] struct Held { received_at: DateTime, exchange_timestamp: DateTime, /// As the exchange wrote it: the hash takes it so. timestamp: String, side: Side, price: i64, quantity: i64, hash: Hash, } impl Held { fn same(&self, other: &Held) -> bool { self.key() == other.key() } /// What tells a change from another: two copies have the same. fn key(&self) -> (Hash, Side, i64, i64) { (self.hash, self.side, self.price, self.quantity) } } impl RecentChange { fn is_copy(&self, change: &Held) -> bool { (self.hash, self.side, self.price, self.quantity) == change.key() } } /// A symbol's subscription on one of its connections, as watched for going /// quiet. #[derive(Debug, Clone, Copy)] struct Watch { connection: usize, /// Its changes received from this connection this minute: every one, /// whatever first one wins then made of it. received: u32, /// When it was last subscribed there: a minute it didn't cover whole is /// not judged. since: DateTime, /// When the connection last delivered a complete book of it: its first /// since the connection came up is the reconnect's. book_at: DateTime, } impl Watch { fn new(connection: usize, since: DateTime) -> Self { Self { connection, received: 0, since, book_at: DateTime::UNIX_EPOCH, } } } /// A change as applied, with the exchange's hash of the book after it. struct RecentChange { side: Side, price: i64, quantity: i64, /// The level's quantity before it (0 if absent): what undoes it. previous: i64, hash: Hash, /// When its first copy came: a later copy's lag is measured from it. received_at: DateTime, /// How many copies came: one, of a symbol on two connections, is a change /// the other missed. copies: u8, /// Its copies are counted, or not to be: applied unchecked, the others' /// copies were dropped unseen. counted: bool, } struct Symbol { state: State, book: Book, /// Of the last change applied, or of the book that replaced ours: a change /// older than this is a late copy. exchange_timestamp: DateTime, /// Its last [`RECENT_CHANGES`] changes. recent_changes: VecDeque, /// By connection: the changes it delivered that don't fit yet. held: BTreeMap>, /// When the book last moved: held changes older than this, on a /// connection whose changes still don't fit, are that connection's lag. advanced_at: DateTime, /// The connection a fresh book was asked from, and whether it is written /// whatever it holds: the connection followed changed. asked: Option<(usize, bool)>, /// Whether a change was verified since its first book: a fresh book that /// differs before then shows a subscription gap, not a loss. verified_since_book: bool, /// It had a book when every connection carrying it went down: the /// reconnect's book is compared with it. had_book: bool, /// The connections it is subscribed on (three while a move overlaps). connections: Vec, /// One per connection in `connections`. watches: Vec, /// The connection it was quiet on last minute, and resubscribed. quiet_on: Option, /// The receive time of its last Delta: a held change is stamped no earlier, /// so replaying the output by time keeps its order. last_written: DateTime, // The other inputs of the book hash, as the exchange writes them. market: String, tick_size: String, last_trade_price: String, min_order_size: String, neg_risk: bool, } impl Symbol { fn new(market: &Market, connections: &[usize], now: DateTime) -> Self { Self { state: State::AwaitingBook, book: Book::default(), exchange_timestamp: DateTime::UNIX_EPOCH, recent_changes: VecDeque::new(), held: BTreeMap::new(), advanced_at: DateTime::UNIX_EPOCH, asked: None, verified_since_book: false, had_book: false, connections: connections.to_vec(), watches: connections .iter() .map(|&connection| Watch::new(connection, now)) .collect(), quiet_on: None, last_written: DateTime::UNIX_EPOCH, market: market.condition_id.clone(), tick_size: String::new(), last_trade_price: String::new(), min_order_size: market.min_order_size.clone(), neg_risk: market.neg_risk, } } /// Asks `connection` for a fresh book: unsubscribe and subscribe again. fn ask_fresh_book( &mut self, token: &str, connection: usize, switch: bool, effects: &mut Vec, ) { // A switch already asked stays one until its book comes. let switch = switch || self.asked.is_some_and(|(_, switch)| switch); self.asked = Some((connection, switch)); effects.push(Effect::Send( connection, Command::Resubscribe(token.to_string()), )); } fn watch_mut(&mut self, connection: usize) -> Option<&mut Watch> { self.watches .iter_mut() .find(|watch| watch.connection == connection) } /// The connection quiet for it over the minute begun at `started`: one /// that delivered none of its changes while another delivered at least /// [`QUIET_AGAINST`], both up and subscribed for the whole minute. fn quiet(&self, started: DateTime, up_since: &[Option>]) -> Option { let judged: Vec<&Watch> = self .watches .iter() .filter(|watch| { watch.since <= started && up_since .get(watch.connection) .copied() .flatten() .is_some_and(|up| up <= started) }) .collect(); let busiest = judged.iter().map(|watch| watch.received).max()?; (busiest >= QUIET_AGAINST) .then(|| judged.iter().find(|watch| watch.received == 0)) .flatten() .map(|watch| watch.connection) } /// One of its connections that is up, other than `except`. fn other_up(&self, up: &[bool], except: usize) -> Option { self.connections .iter() .copied() .find(|&connection| connection != except && is_up(up, connection)) } /// Whether `book` is ours as it was after one of our recent changes: ours /// rewound, each change undone. Comparing `book` plus our later changes /// with ours instead would be unsound: a lost change at a price we had /// changed before would be overwritten by that earlier change. fn was_ours(&self, book: &Book) -> bool { let mut rewound = self.book.clone(); self.recent_changes.iter().rev().any(|change| { rewound.set(change.side, change.price, change.previous); rewound == *book }) } /// `book` with our recent changes, from the `from`-th on, applied. fn brought_forward(&self, book: &Book, from: usize) -> Book { let mut book = book.clone(); for change in self.recent_changes.iter().skip(from) { book.set(change.side, change.price, change.quantity); } book } /// Applies a change, `checked` against its hash or not, and remembers it; /// its Delta, unless it leaves the book as it was. fn apply( &mut self, token: &str, change: &Held, now: DateTime, checked: bool, counts: &mut Counts, ) -> Option { let previous = self.book.set(change.side, change.price, change.quantity); if self.recent_changes.len() == RECENT_CHANGES && let Some(mut oldest) = self.recent_changes.pop_front() { Self::count_copies(&mut oldest, self.connections.len(), counts); } self.recent_changes.push_back(RecentChange { side: change.side, price: change.price, quantity: change.quantity, previous, hash: change.hash, received_at: change.received_at, copies: 1, counted: !checked, }); self.exchange_timestamp = change.exchange_timestamp; self.advanced_at = now; if previous == change.quantity { return None; } self.last_written = self.last_written.max(change.received_at); Some(Output::Delta(PriceLevelUpdateDomain { backtest_timestamp: self.last_written, exchange_timestamp: change.exchange_timestamp, price_level: change.price, quantity: change.quantity, side: change.side, symbol: token.to_string(), })) } /// A change delivered by one connection only, of a symbol on several: /// rescued. fn count_copies(change: &mut RecentChange, connections: usize, counts: &mut Counts) { if !change.counted { change.counted = true; if change.copies == 1 && connections > 1 { counts.rescued += 1; } } } /// Counts the copies of its changes old enough for every copy to have /// come. fn count_old_copies(&mut self, now: DateTime, counts: &mut Counts) { let connections = self.connections.len(); for change in &mut self.recent_changes { if now - change.received_at > COPIES_BY { Self::count_copies(change, connections, counts); } } } /// Where the shortest tail of `connection`'s held changes that gives the /// hash of its last one starts: that tail fits; what comes before it is /// already in the book. A tail is whole batches, so only those starting a /// batch are tried: the whole list, and the last [`TAILS_TRIED`]. fn fit(&self, token: &str, connection: usize) -> Option { let held = self.held.get(&connection)?; let last = held.last()?; let batch_starts = (1..held.len()) .rev() .filter(|&start| held[start].hash != held[start - 1].hash) .take(TAILS_TRIED); batch_starts.chain([0]).find(|&start| { let mut book = self.book.clone(); for change in &held[start..] { book.set(change.side, change.price, change.quantity); } hash(token, self, &book, &last.timestamp) == last.hash }) } /// Drops the held changes that the book now holds: copies of applied /// changes, and changes older than its last. fn prune(&mut self) { let (recent, newest) = (&self.recent_changes, self.exchange_timestamp); for held in self.held.values_mut() { held.retain(|change| { change.exchange_timestamp >= newest && !recent.iter().any(|applied| applied.is_copy(change)) }); } self.held.retain(|_, held| !held.is_empty()); } } fn is_up(up: &[bool], connection: usize) -> bool { up.get(connection).copied().unwrap_or(false) } /// A market's frame bytes since `since`, from all of its connections. struct Measure { bytes: u64, since: DateTime, } impl Measure { /// Bytes per second since `since` on one of `connections`, then starts /// again from `now`; none before [`MIN_MEASURED`], the bytes kept for the /// next report. fn take(&mut self, now: DateTime, connections: usize) -> Option { let measured = now - self.since; if measured < MIN_MEASURED { return None; } let load = self.bytes * 1_000 / measured.num_milliseconds() as u64 / connections.max(1) as u64; *self = Measure { bytes: 0, since: now, }; Some(load) } } /// A market of the book thread. struct Tracked { tokens: Vec, measure: Measure, /// A move under way: the connection left once the new one, the second /// here, delivers a book of the market. Until then both carry it, so the /// move costs no change. leaving: Option<(usize, usize)>, } /// What the minute line counts, since the start. #[derive(Debug, Default)] struct Counts { written: u64, verified: u64, duplicates: u64, /// Changes of a symbol on several connections that only one delivered: /// the others missed them. rescued: u64, /// Held changes dropped at their ceiling, the book having moved on /// without them: their connection was behind. held_dropped: u64, chains_broken: u64, resyncs: u64, /// Fresh books that were a state we had already passed. books_behind: u64, subscription_gaps: u64, losses: u64, switches: u64, /// Subscriptions found quiet, each resubscribed. quiet: u64, /// Markets quiet again after their resubscribe: reported for a move. quiet_reported: u64, /// A reconnect's opening books judged, those of still symbols. reconnect_books: u64, /// Of them, books that differed from every state of ours: both /// subscriptions were quiet. reconnect_losses: u64, } pub struct Feed { /// Its book thread's number, for the logs. number: usize, symbols: BTreeMap, /// By `conditionId`. markets: HashMap, /// By connection number: whether it is up. up: Vec, /// By connection number: since when it is up, if it is. up_since: Vec>>, /// When the minute being counted began: its quiet subscriptions are /// judged at its end. None before the first. minute_started: Option>, /// Changes received this minute, from all connections. received_this_minute: u64, /// The symbols with held changes: their ceilings are checked. waiting: BTreeSet, /// The receive time of the last event applied: the time a Snapshot's book /// is exact as of. last_received_at: Option>, /// The next wall-clock minute: its Snapshots, load report and log line. next_minute_at: Option>, counts: Counts, /// How long after its first copy each duplicate came this minute, µs. lags: Vec, /// The most events waiting for the book thread since the last minute log: /// how far behind it fell. peak_waiting: usize, } impl Feed { pub fn new(number: usize) -> Self { Self { number, symbols: BTreeMap::new(), markets: HashMap::new(), up: Vec::new(), up_since: Vec::new(), minute_started: None, received_this_minute: 0, waiting: BTreeSet::new(), last_received_at: None, next_minute_at: None, counts: Counts::default(), lags: Vec::new(), peak_waiting: 0, } } /// Tracks a market's symbols, then asks its connections to subscribe /// them: a book can't arrive for a symbol not yet known. Its load is /// measured from `now`. pub fn add_market( &mut self, market: &Market, connections: &[usize], now: DateTime, ) -> Vec { for token in &market.tokens { self.symbols .insert(token.clone(), Symbol::new(market, connections, now)); } let tracked = Tracked { tokens: market.tokens.clone(), measure: Measure { bytes: 0, since: now, }, leaving: None, }; self.markets.insert(market.condition_id.clone(), tracked); connections .iter() .map(|&connection| Effect::Send(connection, Command::Subscribe(market.tokens.clone()))) .collect() } /// Moves a market's subscription from one connection to another: it /// subscribes on `to` now, and leaves `from` once `to` delivers a book of /// the market. The book stays as it is: no Snapshot, no subscription gap. pub fn move_market( &mut self, market: &Market, from: usize, to: usize, now: DateTime, ) -> Vec { let Some(tracked) = self.markets.get_mut(&market.condition_id) else { return Vec::new(); }; // A move still under way when the next comes: it ends now. let mut effects = match tracked.leaving.take() { Some((left, _)) => self.leave(&market.condition_id, left), None => Vec::new(), }; if let Some(tracked) = self.markets.get_mut(&market.condition_id) { tracked.leaving = Some((from, to)); } for token in &market.tokens { if let Some(symbol) = self.symbols.get_mut(token) && !symbol.connections.contains(&to) { symbol.connections.push(to); symbol.watches.push(Watch::new(to, now)); } } effects.push(Effect::Send(to, Command::Subscribe(market.tokens.clone()))); effects } /// The market's symbols stop being carried by `connection`: it is /// unsubscribed, and a symbol following it follows another. fn leave(&mut self, condition_id: &str, connection: usize) -> Vec { let Some(tracked) = self.markets.get(condition_id) else { return Vec::new(); }; let tokens = tracked.tokens.clone(); let mut effects = vec![Effect::Send( connection, Command::Unsubscribe(tokens.clone()), )]; for token in &tokens { if let Some(symbol) = self.symbols.get_mut(token) { symbol.connections.retain(|&on| on != connection); symbol .watches .retain(|watch| watch.connection != connection); symbol.held.remove(&connection); Self::lose( token, symbol, connection, &self.up, &mut self.counts, &mut effects, ); // A fresh book asked from it would never come: ask another. if let Some((asked, switch)) = symbol.asked && asked == connection && let Some(other) = symbol.other_up(&self.up, connection) { symbol.ask_fresh_book(token, other, switch, &mut effects); self.counts.resyncs += 1; } } } effects } /// Forgets a market's symbols and unsubscribes them; nothing is written /// for them. Their events still on the way are ignored, as any unknown /// token's. pub fn remove_market(&mut self, market: &Market) -> Vec { let mut connections = Vec::new(); for token in &market.tokens { if let Some(symbol) = self.symbols.remove(token) { connections = symbol.connections; } self.waiting.remove(token); } self.markets.remove(&market.condition_id); connections .into_iter() .map(|connection| Effect::Send(connection, Command::Unsubscribe(market.tokens.clone()))) .collect() } /// What a connection sent: an event, or itself coming up or going down. pub fn on_received(&mut self, received: Received) -> Vec { match received { Received::Event(arrival) => self.on_event(arrival), Received::Up(connection) => { // Its first events come after this marker; any other // connection's before it were received earlier. let at = self.last_received_at.unwrap_or(DateTime::UNIX_EPOCH); self.set_up(connection, true, at); Vec::new() } Received::Down(connection) => self.on_down(connection), } } fn set_up(&mut self, connection: usize, up: bool, at: DateTime) { if self.up.len() <= connection { self.up.resize(connection + 1, false); self.up_since.resize(connection + 1, None); } if up && !self.up[connection] { self.up_since[connection] = Some(at); } else if !up { self.up_since[connection] = None; } self.up[connection] = up; } pub fn on_event(&mut self, arrival: Arrival) -> Vec { let (received_at, connection) = (arrival.received_at, arrival.connection); // A connection that delivers is up, its marker possibly still behind. if !is_up(&self.up, connection) { self.set_up(connection, true, received_at); } // Minute Snapshots due before this event hold the books as they were. let mut effects = self.tick(received_at); self.last_received_at = Some(received_at); if let Some(tracked) = arrival .event .market() .and_then(|market| self.markets.get_mut(market)) { tracked.measure.bytes += arrival.bytes; } match arrival.event { Event::Book(book) => self.on_book(received_at, connection, book, &mut effects), Event::PriceChange(change) => { self.on_price_change(received_at, connection, change, &mut effects) } Event::TickSizeChange { asset_id, .. } => { if let Some(symbol) = self.symbols.get_mut(&asset_id) && symbol.state == State::Verified { log!("polymarket: {asset_id}: tick size changed, resync"); Self::break_chain( &asset_id, symbol, connection, received_at, &self.up, &mut self.counts, &mut effects, ); } } // The hash takes a new last trade price before this event, and // not always its value, so a fresh book brings it instead // (docs/notes.md). Event::LastTradePrice { .. } | Event::Other => {} } effects.extend(self.expire(received_at)); effects } /// While no event comes: the minute, and the held changes' ceilings. pub fn idle(&mut self, now: DateTime) -> Vec { let mut effects = self.tick(now); effects.extend(self.expire(now)); effects } /// Whether the book thread must wake soon for held changes. fn holds(&self) -> bool { !self.waiting.is_empty() } fn on_down(&mut self, connection: usize) -> Vec { self.set_up(connection, false, DateTime::UNIX_EPOCH); let mut effects = Vec::new(); for (token, symbol) in &mut self.symbols { if !symbol.connections.contains(&connection) { continue; } symbol.held.remove(&connection); Self::lose( token, symbol, connection, &self.up, &mut self.counts, &mut effects, ); } self.waiting.retain(|token| { self.symbols .get(token) .is_some_and(|symbol| !symbol.held.is_empty()) }); effects } /// `connection` no longer delivers for the symbol (it went down, or the /// symbol left it): a symbol following it follows another, whose fresh /// book is written whatever it holds; one with no connection left up /// awaits a book. fn lose( token: &str, symbol: &mut Symbol, connection: usize, up: &[bool], counts: &mut Counts, effects: &mut Vec, ) { let other = symbol.other_up(up, connection); match (symbol.state, other) { (State::AwaitingBook, _) => {} (State::Following(followed) | State::Unverifiable(followed), Some(other)) if followed == connection => { symbol.state = match symbol.state { State::Unverifiable(_) => State::Unverifiable(other), _ => State::Following(other), }; symbol.ask_fresh_book(token, other, true, effects); counts.resyncs += 1; } (_, None) => { // Nothing carries it: changes are being missed. symbol.state = State::AwaitingBook; symbol.had_book = true; symbol.held.clear(); symbol.asked = None; } _ => {} } } /// Once `now` reaches the next wall-clock minute: reports the loads, /// emits the minute Snapshots, asks again for the fresh books still not /// come, and resubscribes (or reports) the subscriptions quiet over the /// minute that ends. pub fn tick(&mut self, now: DateTime) -> Vec { let next = *self.next_minute_at.get_or_insert_with(|| next_minute(now)); if now < next { return Vec::new(); } self.next_minute_at = Some(next_minute(now)); let started = self.minute_started.replace(now); let mut loads = Loads::new(); for (market, tracked) in &mut self.markets { let connections = tracked .tokens .first() .and_then(|token| self.symbols.get(token)) .map_or(1, |symbol| symbol.connections.len()); if let Some(load) = tracked.measure.take(now, connections) { loads.insert(market.clone(), load); } } let mut effects = vec![Effect::Report(Report::Loads(loads))]; let Some(received_at) = self.last_received_at else { return effects; }; self.log_minute(); let mut reported = BTreeSet::new(); for (token, symbol) in &mut self.symbols { symbol.count_old_copies(now, &mut self.counts); let quiet = started .filter(|_| symbol.state == State::Verified && symbol.asked.is_none()) .and_then(|started| symbol.quiet(started, &self.up_since)); for watch in &mut symbol.watches { watch.received = 0; } match quiet { // Resubscribed already, and still quiet: it should move. Some(connection) if symbol.quiet_on == Some(connection) => { symbol.quiet_on = None; if reported.insert((symbol.market.clone(), connection)) { self.counts.quiet_reported += 1; effects.push(Effect::Report(Report::Quiet { market: symbol.market.clone(), connection, })); } } // The other connection carries it meanwhile: the book that // comes back is ignored, no Snapshot, no subscription gap. Some(connection) => { log!("polymarket: {token}: quiet on connection {connection}, resubscribed"); self.counts.quiet += 1; symbol.quiet_on = Some(connection); if let Some(watch) = symbol.watch_mut(connection) { watch.since = now; } effects.push(Effect::Send( connection, Command::Resubscribe(token.clone()), )); } None => symbol.quiet_on = None, } if symbol.state != State::AwaitingBook { effects.push(Effect::Write(symbol.book.snapshot( token, received_at, symbol.exchange_timestamp, ))); // A change held across the minute is written after it. symbol.last_written = symbol.last_written.max(received_at); } // The answer may have been lost or unreadable: ask again. let again = match (symbol.state, symbol.asked) { (_, Some((connection, _))) => Some(connection), (State::AwaitingBook, None) => symbol .connections .iter() .copied() .find(|&connection| is_up(&self.up, connection)), _ => None, }; if let Some(connection) = again { let switch = symbol.asked.is_some_and(|(_, switch)| switch); symbol.ask_fresh_book(token, connection, switch, &mut effects); self.counts.resyncs += 1; } } effects } fn log_minute(&mut self) { self.lags.sort_unstable(); let lag = |quantile: f64| { let index = ((self.lags.len() as f64 * quantile) as usize) .min(self.lags.len().saturating_sub(1)); self.lags .get(index) .map_or(0.0, |µs| micros as f64 / 1_000.0) }; let counts = &self.counts; log!( "polymarket: book thread {}: {} symbols; so far {} changes written ({} verified), {} duplicates, \ {} rescued, {} held dropped, {} chains broken, {} resyncs, {} full books behind ours, \ {} subscription gaps, {} losses, {} switches, {} quiet subscriptions resubscribed, \ {} markets reported quiet again, {} reconnect books judged ({} losses); second copies \ {:.1} ms late at the median, {:.1} ms at p99; this minute {} changes received, at most \ {} events waited", self.number, self.symbols.len(), counts.written, counts.verified, counts.duplicates, counts.rescued, counts.held_dropped, counts.chains_broken, counts.resyncs, counts.books_behind, counts.subscription_gaps, counts.losses, counts.switches, counts.quiet, counts.quiet_reported, counts.reconnect_books, counts.reconnect_losses, lag(0.5), lag(0.99), self.received_this_minute, self.peak_waiting ); self.lags.clear(); self.peak_waiting = 0; self.received_this_minute = 0; } /// Held changes past their ceiling: dropped if the book moved on without /// them (their connection is behind), else the chain is broken and their /// connection followed. fn expire(&mut self, now: DateTime) -> Vec { let mut effects = Vec::new(); let mut done = Vec::new(); for token in &self.waiting { let Some(symbol) = self.symbols.get_mut(token) else { done.push(token.clone()); continue; }; let due = symbol.held.iter().find_map(|(&connection, held)| { let first = held.first()?; // Another connection that is up and doesn't hold the same // change could still deliver the one before it. let others_can = symbol.connections.iter().any(|&other| { other != connection && is_up(&self.up, other) && symbol .held .get(&other) .and_then(|held| held.first()) .is_none_or(|theirs| !theirs.same(first)) }); let ceiling = if others_can { HOLD } else { HOLD_ALONE }; let due = now - first.received_at > ceiling || held.len() > MAX_HELD; due.then_some((connection, first.received_at)) }); if let Some((connection, since)) = due { if symbol.advanced_at > since { let dropped = symbol.held.remove(&connection).map_or(0, |held| held.len()); self.counts.held_dropped += dropped as u64; } else { Self::break_chain( token, symbol, connection, now, &self.up, &mut self.counts, &mut effects, ); } } if symbol.held.is_empty() { done.push(token.clone()); } } for token in done { self.waiting.remove(&token); } effects } /// No connection's changes fit: `connection`'s are applied unchecked from /// now, and a fresh book is asked from another, so the followed stream is /// not interrupted. fn break_chain( token: &str, symbol: &mut Symbol, connection: usize, now: DateTime, up: &[bool], counts: &mut Counts, effects: &mut Vec, ) { counts.chains_broken += 1; symbol.state = State::Following(connection); for change in symbol.held.remove(&connection).unwrap_or_default() { if let Some(delta) = symbol.apply(token, &change, now, false, counts) { counts.written += 1; effects.push(Effect::Write(delta)); } } symbol.held.clear(); let asked = symbol.other_up(up, connection).unwrap_or(connection); symbol.ask_fresh_book(token, asked, false, effects); counts.resyncs += 1; } /// A full book counts when asked for (a symbol's first, from any of its /// connections, or the answer to a resync), or when it is a reconnect's /// opening book of a symbol unchanged for [`STILL_FOR`]. It replaces /// ours, brought forward first if it is behind our stream, and is written /// as a Snapshot when it is the symbol's first, when it differs from /// ours, or when the connection followed changed. Then it is checked /// against its own hash. A reconnect's book that is ours, or behind /// ours, changes nothing at all. fn on_book( &mut self, received_at: DateTime, connection: usize, message: BookMessage, effects: &mut Vec, ) { let token = message.asset_id; // A move ends once the new connection delivers a book of the market. let leaving = self .markets .get(&message.market) .and_then(|tracked| tracked.leaving) .filter(|&(_, to)| to == connection); if let Some((from, _)) = leaving { if let Some(tracked) = self.markets.get_mut(&message.market) { tracked.leaving = None; } effects.extend(self.leave(&message.market, from)); } let Some(symbol) = self.symbols.get_mut(&token) else { return; }; // Books on subscribe carry every field; those sent mid-stream, after // a trade, omit these two and are never an answer. let complete = message.last_trade_price.is_some(); let asked = symbol.asked.filter(|&(asked, _)| asked == connection); let first = symbol.state == State::AwaitingBook && symbol.connections.contains(&connection); // The connection's first complete book since it came up, for a // symbol subscribed there before: the reconnect's opening book. let up_since = self.up_since.get(connection).copied().flatten(); let reconnect = complete && symbol.watch_mut(connection).is_some_and(|watch| { // A book at the very instant it came up came before it. let opening = up_since.is_some_and(|up| watch.since < up && watch.book_at <= up); watch.book_at = received_at; opening }); // Judged only for a still, verified symbol, nothing held or asked. let reconnect = reconnect && asked.is_none() && symbol.asked.is_none() && symbol.state == State::Verified && symbol.held.is_empty() && received_at - symbol.advanced_at >= STILL_FOR; if !complete || (asked.is_none() && !first && !reconnect) { return; } let (Some(book), Some(exchange_timestamp), Some(book_hash)) = ( parse_book(&message.bids, &message.asks), parse_millis(&message.timestamp), parse_hash(&message.hash), ) else { // Nobody asked for a reconnect's book: unreadable, it is ignored. if reconnect { return; } // A protocol error: ask again. A book that stays unreadable is asked // for once per answer, and its symbol stays unchecked meanwhile. log!("polymarket: {token}: unreadable book, asking for a fresh one"); symbol.ask_fresh_book(&token, connection, false, effects); self.counts.resyncs += 1; return; }; if reconnect { self.counts.reconnect_books += 1; // The exchange's opening books lag by minutes (docs/notes.md): one // that is ours, or a state we passed, leaves ours, and the hash // inputs, as they are. let brought = symbol .recent_changes .iter() .rposition(|change| change.hash == book_hash) .map(|position| symbol.brought_forward(&book, position + 1)); if book == symbol.book || brought.as_ref() == Some(&symbol.book) { return; } // Our chain holds up to our last change, so a book no newer than // it holds nothing we missed, however far back it lags: further // than the changes we remember, too. if exchange_timestamp <= symbol.exchange_timestamp || symbol.was_ours(&book) { self.counts.books_behind += 1; return; } } let inputs = reconnect.then(|| (symbol.tick_size.clone(), symbol.last_trade_price.clone())); if let Some(tick_size) = message.tick_size { symbol.tick_size = tick_size; } if let Some(last_trade_price) = message.last_trade_price { symbol.last_trade_price = last_trade_price; } // Checked as received, before being brought forward below. let verified = hash(&token, symbol, &book, &message.timestamp) == book_hash; if let Some(inputs) = inputs { // A book failing its own hash proves nothing against a verified // one: ours stands, with its hash inputs. if !verified { (symbol.tick_size, symbol.last_trade_price) = inputs; return; } // Newer than ours and no state of ours: both subscriptions were // quiet. The other one is renewed too; its book is not waited for. self.counts.reconnect_losses += 1; for other in symbol.connections.clone() { if other != connection { if let Some(watch) = symbol.watch_mut(other) { watch.since = received_at; } effects.push(Effect::Send(other, Command::Resubscribe(token.clone()))); } } } let switch = asked.is_some_and(|(_, switch)| switch); let compared = !first || symbol.had_book; let mut book = book; let book_time = exchange_timestamp; let mut exchange_timestamp = exchange_timestamp; // The book a resync brings can lag the changes already applied, with a // newer timestamp or last trade price than its levels (docs/notes.md). // It is behind if it is ours as it was a few changes ago: that past // state was the exchange's own, and the changes since are what it // sent us, so ours stands. if compared && book != symbol.book && symbol.was_ours(&book) { self.counts.books_behind += 1; book = symbol.book.clone(); exchange_timestamp = symbol.exchange_timestamp; } else if let Some(position) = symbol .recent_changes .iter() .rposition(|change| change.hash == book_hash) .filter(|_| compared) { // Its hash is that of one of our changes: the exchange's state // after it, which the changes we got since bring to now. book = symbol.brought_forward(&book, position + 1); exchange_timestamp = symbol.exchange_timestamp; } let differs = compared && book != symbol.book; let cause = if switch { Some(Cause::Switch) } else if differs && !reconnect && !symbol.verified_since_book && !symbol.had_book { // A subscribe hides a few ms of changes (docs/notes.md): until a // change is verified, a differing book is that gap. Some(Cause::SubscriptionGap) } else if differs { Some(Cause::Loss) } else { None }; match cause { Some(Cause::Loss) => self.counts.losses += 1, Some(Cause::SubscriptionGap) => self.counts.subscription_gaps += 1, Some(Cause::Switch) => self.counts.switches += 1, None => {} } symbol.book = book; symbol.exchange_timestamp = exchange_timestamp; symbol.advanced_at = received_at; symbol.held.clear(); symbol.asked = None; symbol.had_book = false; if !compared || cause.is_some() { // Our changes before this book no longer lead to ours: rewinding // through them would make up states that never existed. symbol.recent_changes.clear(); let snapshot = symbol .book .snapshot(&token, received_at, exchange_timestamp); effects.push(Effect::Write(snapshot)); symbol.last_written = symbol.last_written.max(received_at); if let Some(cause) = cause { let found = if reconnect { ", found by a reconnect" } else { "" }; let detail = format!( "{found} (its time {}, ours {})", book_time.format("%H:%M:%S%.3f"), exchange_timestamp.format("%H:%M:%S%.3f") ); log!( "polymarket: {}", cause::line(&token, received_at, cause, &detail) ); } } symbol.state = if verified { State::Verified } else { if !matches!(symbol.state, State::Unverifiable(_)) { log!( "polymarket: {token}: a complete book fails its own hash, changes go unchecked" ); } State::Unverifiable(connection) }; self.waiting.remove(&token); } fn on_price_change( &mut self, received_at: DateTime, connection: usize, message: PriceChangeMessage, effects: &mut Vec, ) { let exchange_timestamp = parse_millis(&message.timestamp); for change in message.price_changes { let token = change.asset_id; let Some(symbol) = self.symbols.get_mut(&token) else { continue; }; if let Some(watch) = symbol.watch_mut(connection) { watch.received += 1; } self.received_this_minute += 1; let side = match change.side.as_str() { "BUY" => Some(Side::Buy), "SELL" => Some(Side::Sell), _ => None, }; let (Some(side), Some(price), Some(quantity), Some(exchange_timestamp), Some(hash)) = ( side, parse_fixed(&change.price), parse_fixed(&change.size), exchange_timestamp, parse_hash(&change.hash), ) else { // Another connection's copy can still bring it; if none does, // the next change of this one waits for it in vain. log!( "polymarket: {token}: unreadable change from connection {connection}, dropped" ); continue; }; let change = Held { received_at, exchange_timestamp, timestamp: message.timestamp.clone(), side, price, quantity, hash, }; match symbol.state { State::AwaitingBook => {} State::Following(followed) | State::Unverifiable(followed) => { if followed == connection && let Some(delta) = symbol.apply(&token, &change, received_at, false, &mut self.counts) { self.counts.written += 1; effects.push(Effect::Write(delta)); } } State::Verified => { Self::first_one_wins( &token, symbol, connection, change, received_at, &mut self.counts, &mut self.lags, effects, ); if symbol.held.is_empty() { self.waiting.remove(&token); } else { self.waiting.insert(token); } } } } } /// A verified symbol's change: dropped if the book has it, else held on /// its connection until a tail of its held changes fits; each fit may let /// another connection's held changes fit in turn. // The Feed's fields it needs, borrowed apart from the symbol it holds: // a method on Feed couldn't take `symbol` from `self.symbols` too. #[allow(clippy::too_many_arguments)] fn first_one_wins( token: &str, symbol: &mut Symbol, connection: usize, change: Held, now: DateTime, counts: &mut Counts, lags: &mut Vec, effects: &mut Vec, ) { if let Some(first) = symbol .recent_changes .iter_mut() .find(|applied| applied.is_copy(&change)) { counts.duplicates += 1; first.copies = first.copies.saturating_add(1); if let Some(lag) = (change.received_at - first.received_at).num_microseconds() { lags.push(lag); } return; } // While the chain holds, everything up to the book's state was applied. if change.exchange_timestamp < symbol.exchange_timestamp { counts.duplicates += 1; return; } symbol.held.entry(connection).or_default().push(change); let mut fitted = true; while fitted { fitted = false; let connections: Vec = symbol.held.keys().copied().collect(); for held_on in connections { let Some(start) = symbol.fit(token, held_on) else { continue; }; let held = symbol.held.remove(&held_on).unwrap_or_default(); for change in &held[start..] { counts.verified += 1; if let Some(delta) = symbol.apply(token, change, now, true, counts) { counts.written += 1; effects.push(Effect::Write(delta)); } } symbol.verified_since_book = true; symbol.prune(); fitted = true; break; } } } } /// A book thread: runs until every connection and the orchestrator have /// dropped its channels, then drops its senders so the writer thread finishes /// once every book thread has. // ponytail: unbounded channels never block a socket read nor drop an event; // the ceiling is memory if a consumer stays behind. The minute log's peak of // events waiting makes that visible; bounded channels are a constructor change // if it ever matters. pub fn run( feed: Feed, events: Receiver, commands: Receiver, outputs: Sender, reports: Sender, ) { serve(feed, Vec::new(), events, commands, outputs, reports) } /// [`run`], knowing `connections` already (by number); the orchestrator /// announces the others as it opens them. fn serve( mut feed: Feed, mut connections: Vec>, events: Receiver, commands: Receiver, outputs: Sender, reports: Sender, ) { let no_commands = never(); let mut orchestrator_gone = false; loop { let mut wake = (next_minute(Utc::now()) - Utc::now()) .to_std() .unwrap_or(Duration::ZERO); if feed.holds() { wake = wake.min(EXPIRE_EVERY); } let commands = if orchestrator_gone { &no_commands } else { &commands }; let effects = select! { recv(events) -> received => match received { Ok(received) => { feed.peak_waiting = feed.peak_waiting.max(events.len()); feed.on_received(received) } Err(_) => return, }, recv(commands) -> command => match command { Ok(BookCommand::Connected(number, connection)) => { assert_eq!(number, connections.len(), "connections open in order"); connections.push(connection); Vec::new() } Ok(BookCommand::Add(market, on)) => feed.add_market(&market, &on, Utc::now()), Ok(BookCommand::Move { market, from, to }) => { feed.move_market(&market, from, to, Utc::now()) } Ok(BookCommand::Remove(market)) => feed.remove_market(&market), // It is stopping: the connections' end still decides ours. Err(_) => { orchestrator_gone = true; Vec::new() } }, default(wake) => feed.idle(Utc::now()), }; for effect in effects { match effect { Effect::Write(output) => { if outputs.send(output).is_err() { return log!("polymarket: the writer thread is gone, stopping"); } } // The connection may be gone already, while stopping, and the // orchestrator too. Effect::Send(connection, command) => _ = connections[connection].send(command), Effect::Report(report) => _ = reports.send(report), } } } } /// Polymarket's book hash: SHA-1 of the book as its REST `/book` returns it, /// `hash` emptied, keys in this order (docs/notes.md): `book` with the /// symbol's other inputs. fn hash(token: &str, symbol: &Symbol, book: &Book, timestamp: &str) -> Hash { #[derive(Serialize)] struct Level { price: String, size: String, } #[derive(Serialize)] struct HashedBook<'a> { market: &'a str, asset_id: &'a str, timestamp: &'a str, hash: &'a str, bids: Vec, asks: Vec, min_order_size: &'a str, tick_size: &'a str, neg_risk: bool, last_trade_price: &'a str, } let level = |(&price, &size): (&i64, &i64)| Level { price: format_fixed(price), size: format_fixed(size), }; let hashed = HashedBook { market: &symbol.market, asset_id: token, timestamp, hash: "", // Bids ascending, asks descending: the best level of each side last. bids: book.bids.iter().map(level).collect(), asks: book.asks.iter().rev().map(level).collect(), min_order_size: &symbol.min_order_size, tick_size: &symbol.tick_size, neg_risk: symbol.neg_risk, last_trade_price: &symbol.last_trade_price, }; let json = serde_json::to_string(&hashed).expect("strings and bools always serialize"); Sha1::digest(json.as_bytes()).into() } /// The exchange's 40 lowercase hex characters. fn parse_hash(text: &str) -> Option { if text.len() != 40 { return None; } let mut hash = [0; 20]; for (byte, pair) in hash.iter_mut().zip(text.as_bytes().chunks(2)) { *byte = u8::from_str_radix(std::str::from_utf8(pair).ok()?, 16).ok()?; } Some(hash) } fn parse_book(bids: &[LevelMessage], asks: &[LevelMessage]) -> Option { let side = |levels: &[LevelMessage]| { levels .iter() .map(|level| Some((parse_fixed(&level.price)?, parse_fixed(&level.size)?))) .collect::>>() }; Some(Book { bids: side(bids)?, asks: side(asks)?, }) } /// Polymarket timestamps: milliseconds since the epoch, as a string. fn parse_millis(text: &str) -> Option> { DateTime::from_timestamp_millis(text.parse().ok()?) } fn next_minute(now: DateTime) -> DateTime { now.duration_trunc(TimeDelta::minutes(1)) .expect("a minute divides any UTC time") + TimeDelta::minutes(1) } #[cfg(test)] mod tests { use serde::Deserialize; use super::*; use crate::polymarket::{fixture_markets, wire}; /// A frame's events as `connection` would send them. fn arrivals(received_at: DateTime, connection: usize, text: &str) -> Vec { // The connection logs an unreadable frame and sends nothing. let events = wire::decode(text).unwrap_or_default(); let bytes = (text.len() / events.len().max(1)) as u64; events .into_iter() .filter(|event| event.market().is_some()) .map(|event| Arrival { received_at, connection, bytes, event, }) .collect() } impl Feed { fn on_frame_from( &mut self, connection: usize, received_at: DateTime, text: &str, ) -> Vec { arrivals(received_at, connection, text) .into_iter() .flat_map(|arrival| self.on_event(arrival)) .collect() } fn on_frame(&mut self, received_at: DateTime, text: &str) -> Vec { self.on_frame_from(0, received_at, text) } } /// A frame of the recordings: receive time, connection, text. type Frame = (DateTime, usize, String); fn read_frames(file: &str) -> Vec { #[derive(Deserialize)] struct Line { received_at_us: i64, #[serde(default)] connection: usize, text: String, } let path = format!("{}/tests/fixtures/{file}", env!("CARGO_MANIFEST_DIR")); let mut frames: Vec = std::fs::read_to_string(path) .unwrap() .lines() .map(|line| { let line: Line = serde_json::from_str(line).unwrap(); let received_at = DateTime::from_timestamp_micros(line.received_at_us).unwrap(); (received_at, line.connection, line.text) }) .collect(); frames.sort_by_key(|(received_at, _, _)| *received_at); frames } /// The recorded live feed for the fixture's markets, on one connection /// (tests/fixtures/README.md). fn fixture_frames() -> Vec<(DateTime, String)> { read_frames("polymarket-frames.jsonl") .into_iter() .map(|(received_at, _, text)| (received_at, text)) .collect() } /// The same markets recorded on two connections at once, in receive order. fn two_connections() -> Vec { read_frames("polymarket-frames-two.jsonl") } /// A book thread's feed holding `markets` on `connections`, as the /// orchestrator's Adds leave it at `start`. fn feed_on(markets: &[Market], connections: &[usize], start: DateTime) -> Feed { let mut feed = Feed::new(0); for market in markets { feed.add_market(market, connections, start); } feed } /// The one-connection recording's feed: its markets on connection 0. fn feed_of(markets: &[Market]) -> Feed { feed_on(markets, &[0], fixture_frames()[0].0) } fn replay<'a>( feed: &mut Feed, frames: impl IntoIterator, String)>, ) -> Vec { frames .into_iter() .flat_map(|(received_at, text)| feed.on_frame(*received_at, text)) .collect() } fn replay_both<'a>( feed: &mut Feed, frames: impl IntoIterator, ) -> Vec { frames .into_iter() .flat_map(|(received_at, connection, text)| { feed.on_frame_from(*connection, *received_at, text) }) .collect() } fn resyncs(effects: &[Effect]) -> Vec<(usize, &str)> { effects .iter() .filter_map(|effect| match effect { Effect::Send(connection, Command::Resubscribe(token)) => { Some((*connection, token.as_str())) } _ => None, }) .collect() } fn snapshot_symbols(effects: &[Effect]) -> Vec<&str> { effects .iter() .filter_map(|effect| match effect { Effect::Write(Output::Snapshot(snapshot)) => Some(snapshot.symbol.as_str()), _ => None, }) .collect() } /// A Delta's content: exchange time, side, price, quantity. type Content = (DateTime, Side, i64, i64); /// Per symbol, its Deltas' content, in order, from `since` (exchange /// time) on. fn deltas(effects: &[Effect], since: DateTime) -> BTreeMap> { let mut deltas: BTreeMap<_, Vec<_>> = BTreeMap::new(); for effect in effects { if let Effect::Write(Output::Delta(delta)) = effect && delta.exchange_timestamp >= since { deltas.entry(delta.symbol.clone()).or_default().push(( delta.exchange_timestamp, delta.side, delta.price_level, delta.quantity, )); } } deltas } fn delta_stamps(effects: &[Effect]) -> Vec<(String, DateTime)> { effects .iter() .filter_map(|effect| match effect { Effect::Write(Output::Delta(delta)) => { Some((delta.symbol.clone(), delta.backtest_timestamp)) } _ => None, }) .collect() } /// Whether the changes of frame `index` are lost for good if dropped: no /// later frame touches their levels, and their symbols change again /// later, so a missing change is seen. fn telling_loss(frames: &[T], index: usize, text: impl Fn(&T) -> &str) -> bool { let levels = |text: &str| -> Vec<(String, String, String)> { wire::decode(text) .unwrap_or_default() .into_iter() .flat_map(|event| match event { Event::PriceChange(change) => change.price_changes, _ => Vec::new(), }) .map(|change| (change.asset_id, change.side, change.price)) .collect() }; let lost = levels(text(&frames[index])); let later: Vec<_> = frames[index + 1..] .iter() .flat_map(|frame| levels(text(frame))) .collect(); !lost.is_empty() && lost.iter().all(|level| !later.contains(level)) && lost.iter().all(|(token, _, _)| { later .iter() .filter(|(theirs, _, _)| theirs == token) .count() >= 3 }) } /// The tokens a frame's price changes touch. fn changed_tokens(text: &str) -> Vec { wire::decode(text) .unwrap_or_default() .into_iter() .flat_map(|event| match event { Event::PriceChange(change) => change.price_changes, _ => Vec::new(), }) .map(|change| change.asset_id) .collect() } fn hex(hash: Hash) -> String { hash.iter().map(|byte| format!("{byte:02x}")).collect() } /// A frame of `changes` to `token`'s book `base`, all at `timestamp` (ms), /// carrying the hash of the book after all of them (a batch, if several), /// and that book. fn changes_frame( token: &str, symbol: &Symbol, base: &Book, changes: &[(Side, i64, i64)], timestamp: i64, ) -> (String, Book) { let mut book = base.clone(); for &(side, price, quantity) in changes { book.set(side, price, quantity); } let timestamp = timestamp.to_string(); let hash = hex(hash(token, symbol, &book, ×tamp)); let price_changes: Vec<_> = changes .iter() .map(|&(side, price, quantity)| { serde_json::json!({"asset_id": token, "price": format_fixed(price), "size": format_fixed(quantity), "side": if side == Side::Buy { "BUY" } else { "SELL" }, "hash": hash}) }) .collect(); let frame = serde_json::json!({"event_type": "price_change", "market": symbol.market, "timestamp": timestamp, "price_changes": price_changes}); (frame.to_string(), book) } /// A complete book frame for `book`, as the exchange would send it in /// answer to a resync, 5 ms after the symbol's last change. fn fresh_book(token: &str, symbol: &Symbol, book: &Book) -> String { let at = symbol.exchange_timestamp + TimeDelta::milliseconds(5); book_frame(token, symbol, book, at) } /// A complete book frame for `book`, last changed at `at` (exchange time). fn book_frame(token: &str, symbol: &Symbol, book: &Book, at: DateTime) -> String { let timestamp = at.timestamp_millis().to_string(); let levels = |levels: &BTreeMap| { levels .iter() .map(|(&price, &size)| serde_json::json!({"price": format_fixed(price), "size": format_fixed(size)})) .collect::>() }; serde_json::json!({ "event_type": "book", "market": symbol.market, "asset_id": token, "timestamp": timestamp, "hash": hex(hash(token, symbol, book, ×tamp)), "bids": levels(&book.bids), "asks": levels(&book.asks), "tick_size": symbol.tick_size, "last_trade_price": symbol.last_trade_price, }) .to_string() } /// The first verified symbol with a bid side, and a free bid price below /// 1 for synthetic changes; the next free one is `free + 1_000` or above. fn verified_symbol(feed: &Feed) -> (String, i64) { let (token, symbol) = feed .symbols .iter() .find(|(_, symbol)| symbol.state == State::Verified && symbol.book.bids.len() > 1) .expect("a verified symbol"); let free = (1_000..) .step_by(1_000) .find(|price| { !symbol.book.bids.contains_key(price) && !symbol.book.bids.contains_key(&(price + 1_000)) }) .unwrap(); (token.clone(), free) } // --- one connection: sprint 02's rules still hold --- /// Split batches and trades make checks wait or fail with nothing lost: /// the recording must replay without a single loss. #[test] fn replay_has_no_loss() { let mut feed = feed_of(&fixture_markets()); replay(&mut feed, &fixture_frames()); assert_eq!(feed.counts.losses, 0); assert!( feed.symbols .values() .all(|symbol| !matches!(symbol.state, State::Unverifiable(_))) ); assert!( feed.counts.verified > 1_000, "only {} changes verified", feed.counts.verified ); // Alone, a connection never has a duplicate but the server's repeats. assert!(feed.counts.duplicates < 10, "{:?}", feed.counts); } #[test] fn a_lost_frame_breaks_the_chain_and_its_fresh_book_counts_a_loss() { let frames = fixture_frames(); let markets = fixture_markets(); let mut control = feed_of(&markets); let mut lost = None; for (index, (received_at, text)) in frames.iter().enumerate() { let tokens = changed_tokens(text); if lost.is_none() && index > frames.len() / 4 && !tokens.is_empty() && telling_loss(&frames, index, |(_, text)| text) && tokens.iter().all(|token| { control.symbols[token].state == State::Verified && control.symbols[token].held.is_empty() }) { lost = Some((index, tokens[0].clone())); } control.on_frame(*received_at, text); } let (lost, token) = lost.unwrap(); let mut feed = feed_of(&markets); replay(&mut feed, &frames[..lost]); let chains = feed.counts.chains_broken; let effects = replay(&mut feed, &frames[lost + 1..]); assert!( resyncs(&effects).contains(&(0, token.as_str())), "the loss is never caught" ); assert!(feed.counts.chains_broken > chains); assert_eq!(feed.symbols[&token].state, State::Following(0)); // The fresh book: the exchange's, which our control holds. let book = fresh_book(&token, &feed.symbols[&token], &control.symbols[&token].book); let losses = feed.counts.losses; let effects = feed.on_frame(frames.last().unwrap().0, &book); assert_eq!(snapshot_symbols(&effects), [token.as_str()]); assert_eq!(feed.counts.losses, losses + 1); assert_eq!(feed.symbols[&token].state, State::Verified); } /// A feed half replayed, a verified symbol of it that holds nothing, and a /// complete book of that symbol as it was before its last change: a book /// behind our stream, as a resync's fresh book can be. Its fresh book is /// asked from connection 0. fn feed_past_a_stale_book() -> (Feed, String, DateTime, String) { let frames = fixture_frames(); let mut feed = feed_of(&fixture_markets()); let half = frames.len() / 2; replay(&mut feed, &frames[..half]); let (token, symbol) = feed .symbols .iter_mut() .find(|(_, symbol)| { symbol.state == State::Verified && symbol.held.is_empty() && symbol.recent_changes.len() >= 2 }) .expect("a verified symbol with changes"); let token = token.clone(); let last = symbol.recent_changes.back().unwrap(); let mut before = symbol.book.clone(); before.set(last.side, last.price, last.previous); let before_hash = symbol.recent_changes[symbol.recent_changes.len() - 2].hash; let mut stale: serde_json::Value = serde_json::from_str(&fresh_book(&token, symbol, &before)).unwrap(); stale["hash"] = serde_json::json!(hex(before_hash)); symbol.asked = Some((0, false)); (feed, token, frames[half].0, stale.to_string()) } #[test] fn a_book_behind_our_stream_is_not_a_loss() { let (mut feed, token, now, stale_book) = feed_past_a_stale_book(); let ours = feed.symbols[&token].book.clone(); let effects = feed.on_frame(now, &stale_book); assert!(snapshot_symbols(&effects).is_empty()); assert_eq!(feed.counts.books_behind, 1); assert_eq!(feed.counts.losses, 0); assert_eq!(feed.symbols[&token].book, ours); assert_eq!(feed.symbols[&token].asked, None); } /// Being behind says nothing of our own book: if it was the one lost, the /// book brought forward differs from it. #[test] fn a_book_behind_our_stream_still_catches_our_loss() { let (mut feed, token, now, stale_book) = feed_past_a_stale_book(); let right = feed.symbols[&token].book.clone(); feed.symbols .get_mut(&token) .unwrap() .book .set(Side::Buy, 1, 1); let effects = feed.on_frame(now, &stale_book); assert_eq!(snapshot_symbols(&effects), [token.as_str()]); assert_eq!(feed.counts.losses, 1); assert_eq!(feed.symbols[&token].book, right); } /// Seen live: a resync's fresh book behind by its levels but carrying a /// newer timestamp, so its hash is none of our changes'. Ours stands, and /// nothing stale is written. #[test] fn a_book_behind_by_levels_with_a_newer_timestamp_is_not_a_loss() { let (mut feed, token, now, stale_book) = feed_past_a_stale_book(); let message: BookMessage = serde_json::from_str(&stale_book).unwrap(); let levels = parse_book(&message.bids, &message.asks).unwrap(); let symbol = &feed.symbols[&token]; let book = fresh_book(&token, symbol, &levels); let ours = symbol.book.clone(); let effects = feed.on_frame(now, &book); assert!(snapshot_symbols(&effects).is_empty()); let counts = &feed.counts; assert_eq!( (counts.books_behind, counts.subscription_gaps, counts.losses), (1, 0, 0) ); assert_eq!(feed.symbols[&token].book, ours); assert_eq!(feed.symbols[&token].state, State::Verified); } /// The level test must not mistake our wrong book for a newer one: we /// changed a level, then lost the exchange's next change to it. The fresh /// book plus our earlier change would equal ours; ours rewound never does. #[test] fn a_lost_change_at_a_level_we_had_changed_is_not_taken_for_behind() { let (mut feed, token, now, _) = feed_past_a_stale_book(); let symbol = &feed.symbols[&token]; let last = symbol.recent_changes.back().unwrap(); let mut exchange = symbol.book.clone(); let lost_quantity = last.quantity + 1_000_000; assert_ne!(last.previous, lost_quantity); exchange.set(last.side, last.price, lost_quantity); let book = fresh_book(&token, symbol, &exchange); let effects = feed.on_frame(now, &book); assert_eq!(snapshot_symbols(&effects), [token.as_str()]); assert_eq!(feed.counts.books_behind, 0); assert_eq!(feed.symbols[&token].book, exchange); } /// Seen live: our subscribe hides changes made just before the opening /// book reaches us, so the first fresh book differs at a level none of our /// changes touched, before any change was verified. #[test] fn a_differing_book_before_any_verified_change_is_a_subscription_gap() { let frames = fixture_frames(); let mut feed = feed_of(&fixture_markets()); replay(&mut feed, &frames[..1]); let (token, symbol) = feed .symbols .iter_mut() .find(|(_, symbol)| symbol.state == State::Verified) .unwrap(); let token = token.clone(); assert!(!symbol.verified_since_book); symbol.asked = Some((0, false)); let mut exchange = symbol.book.clone(); exchange.set(Side::Buy, 1_000, 123_000_000); let book = fresh_book(&token, symbol, &exchange); let effects = feed.on_frame(frames[1].0, &book); assert_eq!(snapshot_symbols(&effects), [token.as_str()]); assert_eq!((feed.counts.subscription_gaps, feed.counts.losses), (1, 0)); assert_eq!(feed.symbols[&token].book, exchange); } /// Once a change has been verified since the opening book, our book was /// right then: a fresh book that differs is a loss. #[test] fn a_differing_book_after_a_verified_change_is_a_loss() { let frames = fixture_frames(); let mut feed = feed_of(&fixture_markets()); replay(&mut feed, &frames[..frames.len() / 2]); let (token, symbol) = feed .symbols .iter_mut() .find(|(_, symbol)| symbol.state == State::Verified && symbol.verified_since_book) .unwrap(); let token = token.clone(); symbol.asked = Some((0, false)); let mut exchange = symbol.book.clone(); exchange.set(Side::Buy, 1_000, 123_000_000); let book = fresh_book(&token, symbol, &exchange); let effects = feed.on_frame(frames[frames.len() / 2].0, &book); assert_eq!(snapshot_symbols(&effects), [token.as_str()]); assert_eq!((feed.counts.subscription_gaps, feed.counts.losses), (0, 1)); assert_eq!(feed.symbols[&token].book, exchange); } #[test] fn a_book_with_an_empty_side_parses() { let ask = LevelMessage { price: "0.999".to_string(), size: "5".to_string(), }; let book = parse_book(&[], &[ask]).unwrap(); assert!(book.bids.is_empty()); assert_eq!(book.asks, BTreeMap::from([(999_000, 5_000_000)])); } #[test] fn a_book_failing_its_own_hash_goes_unverifiable_without_resync() { let mut markets = fixture_markets(); for market in &mut markets { market.min_order_size = "6".to_string(); } let mut feed = feed_of(&markets); let frames = fixture_frames(); let opening = feed.on_frame(frames[0].0, &frames[0].1); assert_eq!(snapshot_symbols(&opening).len(), 40); let effects = replay(&mut feed, &frames[1..]); assert_eq!(resyncs(&effects), []); assert!( feed.symbols .values() .all(|symbol| symbol.state == State::Unverifiable(0)) ); assert!( effects .iter() .any(|effect| matches!(effect, Effect::Write(Output::Delta(_)))) ); assert_eq!(feed.counts.verified, 0); } #[test] fn frames_that_are_not_events_are_skipped() { let mut feed = feed_of(&fixture_markets()); let at = fixture_frames()[0].0; for text in [ "PONG", "NO NEW ASSETS\n", "not json", "{\"event_type\":\"best_bid_ask\"}", "[]", ] { assert!(feed.on_frame(at, text).is_empty(), "{text:?}"); } } #[test] fn a_hash_parses_from_its_hex_and_nothing_else() { let text = "b75cdbc9bc53ef345ed7bc6fe7872140992cb182"; assert_eq!(parse_hash(text).map(hex).as_deref(), Some(text)); assert_eq!(parse_hash("b75c"), None); assert_eq!(parse_hash(&"zz".repeat(20)), None); } // --- two connections: first one wins --- /// The exchange time from which both connections had delivered their /// opening books, with a margin: before it, two replays start from books /// taken at different moments. fn settled(frames: &[Frame]) -> DateTime { frames[0].0 + TimeDelta::seconds(5) } #[test] fn two_connections_write_what_one_alone_writes_once() { let frames = two_connections(); let markets = fixture_markets(); let start = frames[0].0; let one_only: Vec = frames .iter() .filter(|(_, connection, _)| *connection == 0) .cloned() .collect(); let mut alone = feed_on(&markets, &[0], start); let alone_effects = replay_both(&mut alone, &one_only); let mut both = feed_on(&markets, &[0, 1], start); let both_effects = replay_both(&mut both, &frames); // One opening Snapshot per symbol, not one per connection. let opening = |effects: &[Effect]| snapshot_symbols(&effects[..effects.len().min(200)]).len(); assert_eq!(opening(&both_effects), 40); assert_eq!(opening(&alone_effects), 40); let since = settled(&frames); let (alone_deltas, both_deltas) = (deltas(&alone_effects, since), deltas(&both_effects, since)); assert!(alone_deltas.values().map(Vec::len).sum::() > 1_000); assert_eq!(alone_deltas, both_deltas); // Every change came twice: the second copy is a duplicate. assert!( both.counts.duplicates > both.counts.verified / 2, "{:?}", both.counts ); assert_eq!(both.counts.losses, 0); assert!(!both.lags.is_empty() || both.counts.duplicates == 0); } /// The first copy's receive time is the one written. #[test] fn each_delta_takes_the_first_receive_time() { let frames = two_connections(); let markets = fixture_markets(); let start = frames[0].0; let mut both = feed_on(&markets, &[0, 1], start); let both_effects = replay_both(&mut both, &frames); for connection in [0, 1] { let one: Vec = frames .iter() .filter(|(_, on, _)| *on == connection) .cloned() .collect(); let mut alone = feed_on(&markets, &[connection], start); let alone_effects = replay_both(&mut alone, &one); // Delta by delta, in order per symbol: never later than one alone. let mut alone_stamps: BTreeMap>> = BTreeMap::new(); for (symbol, at) in delta_stamps(&alone_effects) { alone_stamps.entry(symbol).or_default().push_back(at); } let mut earlier = 0; for (symbol, at) in delta_stamps(&both_effects) { if let Some(theirs) = alone_stamps.get_mut(&symbol).and_then(VecDeque::pop_front) { earlier += usize::from(at < theirs); assert!( at <= theirs + TimeDelta::milliseconds(60), "{symbol}: {at} after {theirs}" ); } } assert!(earlier > 0, "connection {connection} was never slower"); } } /// One change frame of a single verified symbol, past the opening, as both /// connections delivered it: its index on each. fn a_change_on_both( frames: &[Frame], feed_at: impl Fn(usize) -> Feed, ) -> (usize, usize, String) { let quarter = frames.len() / 4; for (index, (_, connection, text)) in frames.iter().enumerate().skip(quarter) { let tokens = changed_tokens(text); if *connection != 0 || tokens.is_empty() || !telling_loss(frames, index, |(_, _, text)| text) { continue; } let feed = feed_at(index); let settled = |token: &String| { feed.symbols[token].state == State::Verified && feed.symbols[token].held.is_empty() }; if !tokens.iter().all(settled) { continue; } if let Some(other) = frames .iter() .enumerate() .skip(quarter) .position(|(_, (_, on, theirs))| *on == 1 && theirs == text) { return (index, other + quarter, tokens[0].clone()); } } panic!("no change on both connections"); } #[test] fn a_change_lost_on_one_connection_is_rescued() { let frames = two_connections(); let markets = fixture_markets(); let start = frames[0].0; let feed_at = |index: usize| { let mut feed = feed_on(&markets, &[0, 1], start); replay_both(&mut feed, &frames[..index]); feed }; let (on_0, _, token) = a_change_on_both(&frames, feed_at); let mut full = feed_on(&markets, &[0, 1], start); let full_effects = replay_both(&mut full, &frames); let mut lossy = feed_on(&markets, &[0, 1], start); let mut lossy_effects = replay_both(&mut lossy, &frames[..on_0]); lossy_effects.extend(replay_both(&mut lossy, &frames[on_0 + 1..])); let since = settled(&frames); assert_eq!( deltas(&full_effects, since), deltas(&lossy_effects, since), "{token}" ); // Every copy has come a minute after the recording: count them. let after = frames.last().unwrap().0 + TimeDelta::minutes(1); full.tick(after); lossy.tick(after); assert_eq!(lossy.counts.losses, 0); assert_eq!(lossy.counts.chains_broken, full.counts.chains_broken); assert!( lossy.counts.rescued > full.counts.rescued, "{:?}", lossy.counts ); } #[test] fn a_change_lost_on_both_breaks_the_chain_and_resyncs_on_the_other() { let frames = two_connections(); let markets = fixture_markets(); let start = frames[0].0; let feed_at = |index: usize| { let mut feed = feed_on(&markets, &[0, 1], start); replay_both(&mut feed, &frames[..index]); feed }; let (on_0, on_1, token) = a_change_on_both(&frames, feed_at); let mut control = feed_on(&markets, &[0, 1], start); replay_both(&mut control, &frames); let mut feed = feed_on(&markets, &[0, 1], start); let kept: Vec = frames .iter() .enumerate() .filter(|(index, _)| *index != on_0 && *index != on_1) .map(|(_, frame)| frame.clone()) .collect(); let chains = control.counts.chains_broken; let effects = replay_both(&mut feed, &kept); assert!(feed.counts.chains_broken > chains, "{:?}", feed.counts); let followed = match feed.symbols[&token].state { State::Following(followed) => followed, state => panic!("{token} is {state:?}"), }; // The fresh book is asked from the connection not followed. assert!( resyncs(&effects).contains(&(1 - followed, token.as_str())), "{:?}", resyncs(&effects) ); assert_eq!(feed.symbols[&token].asked, Some((1 - followed, false))); let book = fresh_book(&token, &feed.symbols[&token], &control.symbols[&token].book); let effects = feed.on_frame_from(1 - followed, frames.last().unwrap().0, &book); assert_eq!(snapshot_symbols(&effects), [token.as_str()]); assert_eq!(feed.counts.losses, 1); assert_eq!(feed.symbols[&token].state, State::Verified); } #[test] fn a_slow_connection_changes_nothing_but_the_duplicates() { let frames = two_connections(); let markets = fixture_markets(); let start = frames[0].0; let mut delayed: Vec = frames .iter() .map(|(at, connection, text)| { let at = if *connection == 1 { *at + TimeDelta::seconds(2) } else { *at }; (at, *connection, text.clone()) }) .collect(); delayed.sort_by_key(|(at, _, _)| *at); let mut slow = feed_on(&markets, &[0, 1], start); let slow_effects = replay_both(&mut slow, &delayed); // Connection 0 alone sets the pace: compare with its own replay. let one_only: Vec = frames .iter() .filter(|(_, connection, _)| *connection == 0) .cloned() .collect(); let mut alone = feed_on(&markets, &[0], start); let alone_effects = replay_both(&mut alone, &one_only); let since = settled(&frames); assert_eq!(deltas(&slow_effects, since), deltas(&alone_effects, since)); assert_eq!( slow.counts.chains_broken, alone.counts.chains_broken, "{:?}", slow.counts ); assert_eq!(slow.counts.losses, 0); assert!(slow.counts.duplicates > 1_000); // Its receive times are connection 0's, symbol by symbol (a held // change waits longer while connection 1 might still bring the one // before it, so symbols interleave differently). let by_symbol = |effects: &[Effect]| { let mut stamps: BTreeMap>> = BTreeMap::new(); for (symbol, at) in delta_stamps(effects) { stamps.entry(symbol).or_default().push(at); } stamps }; assert_eq!(by_symbol(&slow_effects), by_symbol(&alone_effects)); } /// A verified symbol on connections 0 and 1, its book, a free bid price, /// and an exchange time past its last change. fn two_connection_symbol() -> (Feed, String, Book, i64, i64, DateTime) { let frames = two_connections(); let mut feed = feed_on(&fixture_markets(), &[0, 1], frames[0].0); replay_both(&mut feed, &frames); let (token, free) = verified_symbol(&feed); let symbol = &feed.symbols[&token]; let at = frames.last().unwrap().0 + TimeDelta::seconds(1); let millis = symbol.exchange_timestamp.timestamp_millis() + 1_000; let book = symbol.book.clone(); (feed, token, book, free, millis, at) } #[test] fn a_change_held_for_the_one_before_it_is_stamped_after_it() { let (mut feed, token, book, free, millis, at) = two_connection_symbol(); let symbol = &feed.symbols[&token]; let (x, after_x) = changes_frame( &token, symbol, &book, &[(Side::Buy, free, 1_000_000)], millis, ); let (y, after_y) = changes_frame( &token, symbol, &after_x, &[(Side::Buy, free + 1_000, 2_000_000)], millis + 1, ); // Connection 0 missed X: its Y waits. assert!(feed.on_frame_from(0, at, &y).is_empty()); assert!(feed.waiting.contains(&token)); // Connection 1 brings X, 10 ms later: X, then Y. let later = at + TimeDelta::milliseconds(10); let effects = feed.on_frame_from(1, later, &x); assert_eq!( delta_stamps(&effects), [(token.clone(), later), (token.clone(), later)] ); assert_eq!(feed.symbols[&token].book, after_y); // Connection 1's own Y comes after: a duplicate. let duplicates = feed.counts.duplicates; assert!( feed.on_frame_from(1, later + TimeDelta::milliseconds(1), &y) .is_empty() ); assert_eq!(feed.counts.duplicates, duplicates + 1); // Once every copy could have come: X came once, rescued; Y twice. let rescued = feed.counts.rescued; feed.tick(at + TimeDelta::minutes(2)); assert_eq!(feed.counts.rescued, rescued + 1); } /// A burst on a connection that missed the change before it: the held /// changes stop at the cap, the chain breaks, and it is all quick. #[test] fn a_burst_behind_a_missing_change_is_capped() { let (mut feed, token, book, free, millis, at) = two_connection_symbol(); let symbol = &feed.symbols[&token]; let (_, mut after) = changes_frame( &token, symbol, &book, &[(Side::Buy, free, 1_000_000)], millis, ); let mut burst = Vec::new(); for step in 1..=5_000 { let (frame, next) = changes_frame( &token, symbol, &after, &[(Side::Buy, free + 1_000, step * 1_000)], millis + step, ); burst.push(frame); after = next; } let started = std::time::Instant::now(); let mut most = 0; for (step, frame) in burst.iter().enumerate() { feed.on_frame_from(0, at + TimeDelta::microseconds(step as i64), frame); let held = feed.symbols[&token].held.get(&0).map_or(0, Vec::len); most = most.max(held); } assert!(most <= MAX_HELD + 1, "{most} held"); assert_eq!(feed.symbols[&token].state, State::Following(0)); assert!( started.elapsed() < std::time::Duration::from_secs(10), "{:?}", started.elapsed() ); } #[test] fn a_batch_is_written_once_it_fits() { let (mut feed, token, book, free, millis, at) = two_connection_symbol(); let symbol = &feed.symbols[&token]; let changes = [ (Side::Buy, free, 1_000_000), (Side::Buy, free + 1_000, 1_000_000), ]; let (whole, after) = changes_frame(&token, symbol, &book, &changes, millis); // The exchange sends a batch one change per frame, each with the // batch's hash. let value: serde_json::Value = serde_json::from_str(&whole).unwrap(); let part = |index: usize| { let mut value = value.clone(); value["price_changes"] = serde_json::json!([value["price_changes"][index]]); value.to_string() }; assert!(feed.on_frame_from(0, at, &part(0)).is_empty()); let effects = feed.on_frame_from(0, at + TimeDelta::milliseconds(1), &part(1)); assert_eq!(delta_stamps(&effects).len(), 2); assert_eq!(feed.symbols[&token].book, after); assert!(!feed.waiting.contains(&token)); } #[test] fn a_change_older_than_the_book_is_dropped() { let (mut feed, token, book, free, millis, at) = two_connection_symbol(); let symbol = &feed.symbols[&token]; let (old, _) = changes_frame( &token, symbol, &book, &[(Side::Buy, free, 1_000_000)], millis - 60_000, ); let duplicates = feed.counts.duplicates; assert!(feed.on_frame_from(1, at, &old).is_empty()); assert_eq!(feed.counts.duplicates, duplicates + 1); assert!(!feed.waiting.contains(&token)); } #[test] fn held_changes_past_their_ceiling_break_the_chain_when_the_book_stood_still() { let (mut feed, token, book, free, millis, at) = two_connection_symbol(); let symbol = &feed.symbols[&token]; let (_, after_x) = changes_frame( &token, symbol, &book, &[(Side::Buy, free, 1_000_000)], millis, ); let (y, after_y) = changes_frame( &token, symbol, &after_x, &[(Side::Buy, free + 1_000, 2_000_000)], millis + 1, ); // Both missed X and hold the same Y: nobody can bring X, 50 ms. feed.on_frame_from(0, at, &y); feed.on_frame_from(1, at, &y); assert!(feed.idle(at + TimeDelta::milliseconds(40)).is_empty()); let effects = feed.idle(at + TimeDelta::milliseconds(60)); assert_eq!(feed.symbols[&token].state, State::Following(0)); assert_eq!(resyncs(&effects), [(1, token.as_str())]); // Y is written unchecked, on the book that lacks X. assert_eq!(delta_stamps(&effects), [(token.clone(), at)]); assert_ne!(feed.symbols[&token].book, after_y); // Connection 1's changes are dropped while 0 is followed. let (z, _) = changes_frame( &token, &feed.symbols[&token], &after_y, &[(Side::Buy, free, 3_000_000)], millis + 2, ); assert!( feed.on_frame_from(1, at + TimeDelta::milliseconds(70), &z) .is_empty() ); assert_eq!( feed.on_frame_from(0, at + TimeDelta::milliseconds(71), &z) .len(), 1 ); } #[test] fn a_connection_behind_has_its_held_changes_dropped_at_the_ceiling() { let (mut feed, token, book, free, millis, at) = two_connection_symbol(); let symbol = &feed.symbols[&token]; let (x, after_x) = changes_frame( &token, symbol, &book, &[(Side::Buy, free, 1_000_000)], millis, ); // Connection 0 holds a change that will never fit (made on another // book), while connection 1 moves the book on. let mut elsewhere = after_x.clone(); elsewhere.set(Side::Buy, free + 1_000, 1); let (stray, _) = changes_frame( &token, symbol, &elsewhere, &[(Side::Sell, 999_000, 5_000_000)], millis, ); feed.on_frame_from(0, at, &stray); let effects = feed.on_frame_from(1, at + TimeDelta::milliseconds(5), &x); assert_eq!(delta_stamps(&effects).len(), 1); // Connection 1 could still deliver: a second, then dropped. assert!(feed.idle(at + TimeDelta::milliseconds(900)).is_empty()); let effects = feed.idle(at + TimeDelta::milliseconds(1_100)); assert!(effects.is_empty()); assert_eq!(feed.symbols[&token].state, State::Verified); assert_eq!(feed.counts.held_dropped, 1); assert!(!feed.waiting.contains(&token)); } #[test] fn a_book_not_asked_for_changes_nothing() { let (mut feed, token, book, _, _, at) = two_connection_symbol(); let symbol = &feed.symbols[&token]; let mut other = book.clone(); other.set(Side::Buy, 1_000, 5_000_000); let unasked = fresh_book(&token, symbol, &other); assert!(feed.on_frame_from(1, at, &unasked).is_empty()); assert_eq!(feed.symbols[&token].book, book); } #[test] fn one_connection_down_changes_nothing_and_both_down_await_a_book() { let (mut feed, token, book, free, millis, at) = two_connection_symbol(); assert!(feed.on_received(Received::Down(1)).is_empty()); assert_eq!(feed.symbols[&token].state, State::Verified); let symbol = &feed.symbols[&token]; let (x, after_x) = changes_frame( &token, symbol, &book, &[(Side::Buy, free, 1_000_000)], millis, ); // Alone, a change that doesn't fit waits 50 ms, not a second. assert_eq!(feed.on_frame_from(0, at, &x).len(), 1); assert_eq!(feed.symbols[&token].book, after_x); feed.on_received(Received::Down(0)); assert_eq!(feed.symbols[&token].state, State::AwaitingBook); // The reconnect's opening book is compared with ours: equal, nothing. feed.on_received(Received::Up(0)); let same = fresh_book(&token, &feed.symbols[&token], &after_x); assert!(snapshot_symbols(&feed.on_frame_from(0, at, &same)).is_empty()); assert_eq!(feed.symbols[&token].state, State::Verified); // Differing, a loss. feed.on_received(Received::Down(0)); feed.on_received(Received::Down(1)); let mut other = after_x.clone(); other.set(Side::Buy, free, 7_000_000); let differing = fresh_book(&token, &feed.symbols[&token], &other); let losses = feed.counts.losses; assert_eq!( snapshot_symbols(&feed.on_frame_from(1, at, &differing)), [token.as_str()] ); assert_eq!(feed.counts.losses, losses + 1); } /// A frame setting `token`'s bid at `price` to `quantity`, `millis` its /// exchange time, from the book as it is now. fn change(feed: &Feed, token: &str, millis: i64, price: i64, quantity: i64) -> String { let symbol = &feed.symbols[token]; let changes = [(Side::Buy, price, quantity)]; changes_frame(token, symbol, &symbol.book, &changes, millis).0 } fn resyncs_of<'a>(effects: &'a [Effect], token: &str) -> Vec<(usize, &'a str)> { resyncs(effects) .into_iter() .filter(|&(_, theirs)| theirs == token) .collect() } /// From the next wall-clock minute after `at`, which the feed has begun: /// the following minute is counted whole. fn begin_minute(feed: &mut Feed, at: DateTime) -> DateTime { let minute = next_minute(at); feed.idle(minute); minute } /// `count` changes of `token`, from 10 s after `start`, on the /// connections `on` only. fn changes( feed: &mut Feed, token: &str, on: &[usize], start: DateTime, price: i64, count: i64, ) { let millis = feed.symbols[token].exchange_timestamp.timestamp_millis() + 1_000; for step in 0..count { let at = start + TimeDelta::seconds(10 + step); let frame = change(feed, token, millis + step, price, (step + 1) * 1_000_000); for &connection in on { feed.on_frame_from(connection, at, &frame); } } } fn quiet_reports(effects: &[Effect]) -> Vec<(&str, usize)> { effects .iter() .filter_map(|effect| match effect { Effect::Report(Report::Quiet { market, connection }) => { Some((market.as_str(), *connection)) } _ => None, }) .collect() } #[test] fn a_subscription_quiet_on_one_connection_is_resubscribed_then_reported() { let (mut feed, token, _, free, _, at) = two_connection_symbol(); let market = feed.symbols[&token].market.clone(); let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n); begin_minute(&mut feed, at); let (losses, gaps) = (feed.counts.losses, feed.counts.subscription_gaps); // Two changes against none: too few to say. changes(&mut feed, &token, &[0], minute(0), free, 2); let effects = feed.idle(minute(1)); assert!(resyncs_of(&effects, &token).is_empty()); assert_eq!(feed.counts.quiet, 0); // Three: quiet on connection 1, resubscribed there. changes(&mut feed, &token, &[0], minute(1), free, 3); let effects = feed.idle(minute(2)); assert_eq!(resyncs_of(&effects, &token), [(1, token.as_str())]); assert!(quiet_reports(&effects).is_empty()); assert_eq!(feed.counts.quiet, 1); assert_eq!(feed.symbols[&token].state, State::Verified); // The book the resubscribe brings back is not asked for: ignored. let book = feed.symbols[&token].book.clone(); let back = fresh_book(&token, &feed.symbols[&token], &book); let effects = feed.on_frame_from(1, minute(2) + TimeDelta::seconds(1), &back); assert!(snapshot_symbols(&effects).is_empty()); // Still quiet the minute after: reported for a move, not resubscribed. changes(&mut feed, &token, &[0], minute(2), free, 3); let effects = feed.idle(minute(3)); assert!(resyncs_of(&effects, &token).is_empty()); assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]); assert_eq!( (feed.counts.losses, feed.counts.subscription_gaps), (losses, gaps) ); } /// Both tokens of a market go quiet together: one report for the market. #[test] fn a_market_quiet_on_both_its_tokens_is_reported_once() { let (mut feed, token, _, free, _, at) = two_connection_symbol(); let market = feed.symbols[&token].market.clone(); let tokens = feed.markets[&market].tokens.clone(); assert!( tokens .iter() .all(|token| feed.symbols[token].state == State::Verified) ); let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n); begin_minute(&mut feed, at); for n in 0..2 { for token in &tokens { changes(&mut feed, token, &[0], minute(n), free, 3); } let effects = feed.idle(minute(n + 1)); if n == 0 { assert_eq!(feed.counts.quiet, 2); } else { assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]); } } } #[test] fn a_subscription_delivering_or_not_there_the_whole_minute_is_not_quiet() { let (mut feed, token, _, free, _, at) = two_connection_symbol(); let markets = fixture_markets(); let market = markets .iter() .find(|market| market.tokens.contains(&token)) .unwrap(); // Connection 5 is up from before the minutes below. feed.on_received(Received::Up(5)); let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n); begin_minute(&mut feed, at); // Both deliver: neither is quiet; the symbols that didn't change // aren't either. changes(&mut feed, &token, &[0, 1], minute(0), free, 3); let effects = feed.idle(minute(1)); assert!(resyncs(&effects).iter().all(|&(_, theirs)| theirs != token)); // Connection 1 dropped and came back within the minute: not judged // for the changes it missed. changes(&mut feed, &token, &[0], minute(1), free, 3); feed.on_received(Received::Down(1)); feed.on_received(Received::Up(1)); let effects = feed.idle(minute(2)); assert!(resyncs_of(&effects, &token).is_empty()); // Connection 5, subscribed in the middle of the minute and delivering // nothing yet: not judged either. feed.move_market(market, 1, 5, minute(2) + TimeDelta::seconds(5)); changes(&mut feed, &token, &[0, 1], minute(2), free, 3); let effects = feed.idle(minute(3)); assert!(resyncs_of(&effects, &token).is_empty()); assert_eq!(feed.counts.quiet, 0); } /// A verified symbol on connections 0 and 1, changed once and then still /// for 30 s: the feed, its token, its book before the change and now, a /// free bid price, and the time 30 s after. fn still_symbol() -> (Feed, String, Book, Book, i64, DateTime) { let (mut feed, token, before, free, millis, at) = two_connection_symbol(); let x = change(&feed, &token, millis, free, 1_000_000); feed.on_frame_from(0, at, &x); feed.on_frame_from(1, at, &x); let ours = feed.symbols[&token].book.clone(); let later = at + TimeDelta::seconds(30); // The minute's Snapshots, out of the way. feed.idle(later); (feed, token, before, ours, free, later) } /// Connection 1 drops and reconnects; its opening book of `token` holds /// `book`. fn reconnect_with(feed: &mut Feed, token: &str, book: &Book, at: DateTime) -> Vec { feed.on_received(Received::Down(1)); feed.on_received(Received::Up(1)); let opening = fresh_book(token, &feed.symbols[token], book); feed.on_frame_from(1, at, &opening) } #[test] fn a_reconnect_book_of_a_still_symbol_equal_to_ours_or_behind_changes_nothing() { let (mut feed, token, before, ours, _, later) = still_symbol(); let losses = feed.counts.losses; let effects = reconnect_with(&mut feed, &token, &ours, later); assert!(snapshot_symbols(&effects).is_empty()); assert_eq!(feed.counts.reconnect_books, 1); // The server's opening books lag: a state we passed is no loss. let behind = feed.counts.books_behind; let effects = reconnect_with(&mut feed, &token, &before, later + TimeDelta::seconds(1)); assert!(snapshot_symbols(&effects).is_empty()); assert_eq!(feed.counts.reconnect_books, 2); assert_eq!(feed.counts.books_behind, behind + 1); assert_eq!(feed.symbols[&token].book, ours); // It lags further back than the changes we remember: no newer than // our last change, it still holds nothing we missed. let symbol = feed.symbols.get_mut(&token).unwrap(); symbol.recent_changes.clear(); let old = symbol.exchange_timestamp - TimeDelta::minutes(10); feed.on_received(Received::Down(1)); feed.on_received(Received::Up(1)); let lagging = book_frame(&token, &feed.symbols[&token], &before, old); let effects = feed.on_frame_from(1, later + TimeDelta::seconds(2), &lagging); assert!(snapshot_symbols(&effects).is_empty()); assert_eq!(feed.counts.books_behind, behind + 2); assert_eq!(feed.symbols[&token].book, ours); assert_eq!(feed.counts.losses, losses); assert_eq!(feed.symbols[&token].state, State::Verified); } /// A reconnect's book that fails its own hash proves nothing: the symbol /// stays verified, with its book and its hash inputs. #[test] fn a_reconnect_book_failing_its_own_hash_changes_nothing() { let (mut feed, token, _, ours, free, later) = still_symbol(); let mut other = ours.clone(); other.set(Side::Buy, free, 7_000_000); let frame = fresh_book(&token, &feed.symbols[&token], &other); let mut frame: serde_json::Value = serde_json::from_str(&frame).unwrap(); frame["hash"] = "0".repeat(40).into(); frame["last_trade_price"] = "0.123".into(); let last_trade_price = feed.symbols[&token].last_trade_price.clone(); feed.on_received(Received::Down(1)); feed.on_received(Received::Up(1)); let effects = feed.on_frame_from(1, later, &frame.to_string()); assert!(effects.is_empty()); let symbol = &feed.symbols[&token]; assert_eq!(symbol.state, State::Verified); assert_eq!(symbol.book, ours); assert_eq!(symbol.last_trade_price, last_trade_price); } /// Only a reconnect's book of a symbol with nothing asked is judged: not /// the opening book of a subscription made since the connection came up /// (a move's), nor any while a fresh book is asked. #[test] fn a_book_that_is_no_reconnects_or_comes_while_one_is_asked_is_not_judged() { let (mut feed, token, _, ours, free, later) = still_symbol(); let markets = fixture_markets(); let market = markets .iter() .find(|market| market.tokens.contains(&token)) .unwrap(); let mut other = ours.clone(); other.set(Side::Buy, free, 7_000_000); feed.on_received(Received::Up(5)); feed.move_market(market, 0, 5, later); let opening = fresh_book(&token, &feed.symbols[&token], &other); let effects = feed.on_frame_from(5, later + TimeDelta::seconds(1), &opening); assert!(snapshot_symbols(&effects).is_empty()); assert_eq!(feed.counts.reconnect_books, 0); feed.symbols.get_mut(&token).unwrap().asked = Some((5, false)); let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(2)); assert!(snapshot_symbols(&effects).is_empty()); assert_eq!(feed.counts.reconnect_books, 0); assert_eq!(feed.symbols[&token].book, ours); } #[test] fn a_reconnect_book_of_a_still_symbol_that_differs_is_a_loss_and_resubscribes_the_other() { let (mut feed, token, _, ours, free, later) = still_symbol(); let (losses, gaps) = (feed.counts.losses, feed.counts.subscription_gaps); // Even before any change of it was verified: a loss, not the gap of // a subscribe, which is long past. feed.symbols.get_mut(&token).unwrap().verified_since_book = false; let mut other = ours.clone(); other.set(Side::Buy, free, 7_000_000); let effects = reconnect_with(&mut feed, &token, &other, later); assert_eq!(snapshot_symbols(&effects), [token.as_str()]); assert_eq!(resyncs_of(&effects, &token), [(0, token.as_str())]); assert_eq!(feed.counts.losses, losses + 1); assert_eq!(feed.counts.subscription_gaps, gaps); assert_eq!(feed.counts.reconnect_losses, 1); assert_eq!(feed.symbols[&token].book, other); assert_eq!(feed.symbols[&token].state, State::Verified); } #[test] fn a_reconnect_book_of_a_symbol_changed_lately_is_ignored() { let (mut feed, token, _, ours, free, later) = still_symbol(); let millis = feed.symbols[&token].exchange_timestamp.timestamp_millis() + 1_000; let x = change(&feed, &token, millis, free, 2_000_000); feed.on_frame_from(0, later, &x); let mut other = ours.clone(); other.set(Side::Buy, free, 7_000_000); let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(1)); assert!(snapshot_symbols(&effects).is_empty()); assert_eq!(feed.counts.reconnect_books, 0); // And the next book of it from that connection is no reconnect's. feed.idle(later + TimeDelta::seconds(30)); let opening = fresh_book(&token, &feed.symbols[&token], &other); let effects = feed.on_frame_from(1, later + TimeDelta::seconds(31), &opening); assert!(snapshot_symbols(&effects).is_empty()); assert_eq!(feed.counts.reconnect_books, 0); } #[test] fn the_followed_connection_down_switches_with_a_snapshot() { let (mut feed, token, book, _, _, at) = two_connection_symbol(); feed.symbols.get_mut(&token).unwrap().state = State::Following(0); let effects = feed.on_received(Received::Down(0)); assert_eq!(feed.symbols[&token].state, State::Following(1)); assert!(resyncs(&effects).contains(&(1, token.as_str()))); // Its fresh book is written even if it equals ours. let fresh = fresh_book(&token, &feed.symbols[&token], &book); let effects = feed.on_frame_from(1, at, &fresh); assert_eq!(snapshot_symbols(&effects), [token.as_str()]); assert!(feed.counts.switches >= 1); assert_eq!(feed.symbols[&token].state, State::Verified); } #[test] fn a_move_subscribes_first_and_leaves_once_the_new_connection_delivers() { let (mut feed, token, book, free, millis, at) = two_connection_symbol(); let markets = fixture_markets(); let market = markets .iter() .find(|market| market.tokens.contains(&token)) .unwrap(); let effects = feed.move_market(market, 0, 5, at); assert!(matches!( effects.as_slice(), [Effect::Send(5, Command::Subscribe(tokens))] if *tokens == market.tokens )); // Both still deliver until connection 5's book comes. let symbol = &feed.symbols[&token]; let (x, after_x) = changes_frame( &token, symbol, &book, &[(Side::Buy, free, 1_000_000)], millis, ); assert_eq!(feed.on_frame_from(0, at, &x).len(), 1); let opening = fresh_book(&token, &feed.symbols[&token], &after_x); let effects = feed.on_frame_from(5, at, &opening); assert!(matches!( effects.as_slice(), [Effect::Send(0, Command::Unsubscribe(tokens))] if *tokens == market.tokens )); assert!(snapshot_symbols(&effects).is_empty()); let mut on = feed.symbols[&token].connections.clone(); on.sort(); assert_eq!(on, [1, 5]); } /// The connection left can't answer a fresh book any more: it is asked /// from another. #[test] fn a_move_ending_while_a_fresh_book_is_asked_from_the_left_one_asks_again() { let (mut feed, token, book, _, _, at) = two_connection_symbol(); let markets = fixture_markets(); let market = markets .iter() .find(|market| market.tokens.contains(&token)) .unwrap(); let symbol = feed.symbols.get_mut(&token).unwrap(); symbol.state = State::Following(1); symbol.asked = Some((0, false)); feed.move_market(market, 0, 5, at); let opening = fresh_book(&token, &feed.symbols[&token], &book); let effects = feed.on_frame_from(5, at, &opening); assert!( resyncs(&effects).contains(&(1, token.as_str())), "{:?}", resyncs(&effects) ); assert_eq!(feed.symbols[&token].asked, Some((1, false))); } /// Replaying the output by time keeps its order: a change held across the /// minute is not stamped before the minute's Snapshot. #[test] fn a_change_held_across_the_minute_is_stamped_after_its_snapshot() { let (mut feed, token, book, free, millis, _) = two_connection_symbol(); let minute = next_minute(feed.last_received_at.unwrap()); let symbol = &feed.symbols[&token]; let changes = [ (Side::Buy, free, 1_000_000), (Side::Buy, free + 1_000, 1_000_000), ]; let (whole, _) = changes_frame(&token, symbol, &book, &changes, millis); let value: serde_json::Value = serde_json::from_str(&whole).unwrap(); let part = |index: usize| { let mut value = value.clone(); value["price_changes"] = serde_json::json!([value["price_changes"][index]]); value.to_string() }; // A batch's first change waits for its second across the minute; // another event comes after it, so the minute's Snapshot is stamped // later than that first change. feed.on_frame_from(0, minute - TimeDelta::milliseconds(2), &part(0)); let other = serde_json::json!({"event_type": "price_change", "market": "0xother", "timestamp": "1", "price_changes": []}); feed.on_frame_from(0, minute - TimeDelta::milliseconds(1), &other.to_string()); let effects = feed.on_frame_from(0, minute + TimeDelta::milliseconds(1), &part(1)); let snapshot_at = effects .iter() .find_map(|effect| match effect { Effect::Write(Output::Snapshot(snapshot)) if snapshot.symbol == token => { Some(snapshot.received_at) } _ => None, }) .expect("the minute's Snapshot"); let stamps: Vec<_> = delta_stamps(&effects) .into_iter() .filter(|(symbol, _)| *symbol == token) .collect(); assert_eq!(stamps.len(), 2); for (_, at) in stamps { assert!(at >= snapshot_at, "{at} before {snapshot_at}"); } } #[test] fn a_market_on_two_connections_reports_its_load_on_one() { let minute = "2026-11-10T16:00:00Z".parse::>().unwrap(); let market = Market { condition_id: "0xm".to_string(), tokens: vec!["0xma".to_string(), "0xmb".to_string()], min_order_size: "5".to_string(), neg_risk: false, }; let mut feed = Feed::new(0); feed.add_market(&market, &[0, 1], minute); let change = serde_json::json!({"event_type": "price_change", "market": "0xm", "timestamp": "1", "price_changes": []}) .to_string(); feed.on_frame_from(0, minute + TimeDelta::seconds(10), &change); feed.on_frame_from(1, minute + TimeDelta::seconds(10), &change); let effects = feed.tick(minute + TimeDelta::seconds(60)); let Some(Effect::Report(Report::Loads(loads))) = effects.first() else { panic!("no report"); }; assert_eq!(loads["0xm"], change.len() as u64 / 60); } // --- markets, loads, the output --- /// A market's opening books, as a subscribe brings them. fn opening_books(frames: &[(DateTime, String)], market: &Market) -> String { let opening: Vec = serde_json::from_str(&frames[0].1).unwrap(); let books: Vec<_> = opening .iter() .filter(|book| { market .tokens .iter() .any(|token| book["asset_id"] == **token) }) .collect(); serde_json::to_string(&books).unwrap() } #[test] fn a_market_added_mid_run_is_subscribed_and_its_first_book_written() { let frames = fixture_frames(); let markets = fixture_markets(); let mut feed = feed_of(&markets[1..]); replay(&mut feed, &frames[1..100]); let effects = feed.add_market(&markets[0], &[0], frames[100].0); assert!(matches!( effects.as_slice(), [Effect::Send(0, Command::Subscribe(tokens))] if *tokens == markets[0].tokens )); let effects = feed.on_frame(frames[100].0, &opening_books(&frames, &markets[0])); let mut written = snapshot_symbols(&effects); written.sort(); let mut tokens: Vec<&str> = markets[0].tokens.iter().map(String::as_str).collect(); tokens.sort(); assert_eq!(written, tokens); assert!( markets[0] .tokens .iter() .all(|token| feed.symbols[token].state == State::Verified) ); } #[test] fn a_removed_market_is_unsubscribed_and_written_no_more() { let frames = fixture_frames(); let markets = fixture_markets(); let mut feed = feed_of(&markets); replay(&mut feed, &frames[..1]); let removed = &markets[0].tokens; let effects = feed.remove_market(&markets[0]); assert!(matches!( effects.as_slice(), [Effect::Send(0, Command::Unsubscribe(tokens))] if tokens == removed )); // The rest of the recording crosses two minutes: no Delta, Snapshot or // resync names a removed token. let effects = replay(&mut feed, &frames[1..]); let mentions = |effect: &Effect| match effect { Effect::Write(Output::Snapshot(snapshot)) => removed.contains(&snapshot.symbol), Effect::Write(Output::Delta(delta)) => removed.contains(&delta.symbol), Effect::Send(_, Command::Resubscribe(token)) => removed.contains(token), Effect::Send(..) => false, Effect::Report(Report::Loads(loads)) => loads.contains_key(&markets[0].condition_id), Effect::Report(Report::Quiet { market, .. }) => *market == markets[0].condition_id, }; assert!(!effects.iter().any(mentions)); // The minute Snapshots of the 38 others, twice. assert_eq!(snapshot_symbols(&effects).len(), 38 * 2); } #[test] fn each_minute_reports_the_bytes_per_second_of_every_market() { let minute = "2026-11-10T16:00:00Z".parse::>().unwrap(); let at = |seconds| minute + TimeDelta::seconds(seconds); let market = |id: &str| Market { condition_id: id.to_string(), tokens: vec![format!("{id}a"), format!("{id}b")], min_order_size: "5".to_string(), neg_risk: false, }; let (busy, quiet, late) = (market("0xbusy"), market("0xquiet"), market("0xlate")); let mut feed = Feed::new(0); feed.add_market(&busy, &[0], at(0)); // Added halfway through the minute: measured over its 30 s. feed.add_market(&quiet, &[0], at(30)); // Added 15 s before it: kept for the next report. feed.add_market(&late, &[0], at(45)); let change = |id: &str| { serde_json::json!({"event_type": "price_change", "market": id, "timestamp": "1", "price_changes": []}) .to_string() }; let trade = serde_json::json!({"event_type": "last_trade_price", "market": "0xbusy", "asset_id": "0xbusya", "price": "0.5", "size": "1", "side": "BUY", "timestamp": "1"}) .to_string(); let tick_size = serde_json::json!({"event_type": "tick_size_change", "market": "0xquiet", "asset_id": "0xquieta", "old_tick_size": "0.01", "new_tick_size": "0.001", "timestamp": "1"}) .to_string(); // One frame, two markets: each takes half. let shared = format!("[{},{}]", change("0xbusy"), change("0xquiet")); let mut reports = Vec::new(); let mut on_frame = |feed: &mut Feed, seconds, text: &str| { for effect in feed.on_frame(at(seconds), text) { if let Effect::Report(Report::Loads(loads)) = effect { reports.push(loads); } } }; on_frame(&mut feed, 10, &change("0xbusy")); on_frame(&mut feed, 20, &trade); on_frame(&mut feed, 40, &tick_size); on_frame(&mut feed, 50, &change("0xlate")); on_frame(&mut feed, 55, &change("0xunknown")); // The next minute: the first report, then nothing more until the one // after, which starts again from zero. on_frame(&mut feed, 60, &shared); on_frame(&mut feed, 120, &change("0xunknown")); let bytes = |text: &String| text.len() as u64; let half = bytes(&shared) / 2; assert_eq!( reports, [ Loads::from([ ( "0xbusy".to_string(), (bytes(&change("0xbusy")) + bytes(&trade)) / 60 ), ("0xquiet".to_string(), bytes(&tick_size) / 30), ]), Loads::from([ ("0xbusy".to_string(), half / 60), ("0xquiet".to_string(), half / 60), ("0xlate".to_string(), bytes(&change("0xlate")) / 75), ]), ] ); } #[test] fn the_recording_reports_every_market_once_measured_long_enough() { let frames = fixture_frames(); let markets = fixture_markets(); let mut feed = feed_of(&markets); let reports: Vec = replay(&mut feed, &frames) .into_iter() .filter_map(|effect| match effect { Effect::Report(Report::Loads(loads)) => Some(loads), _ => None, }) .collect(); // The recording crosses two minutes, the first too soon after its // start for any market to be measured. let start = frames[0].0; assert!(next_minute(start) - start < MIN_MEASURED); assert_eq!(reports.len(), 2); assert!(reports[0].is_empty()); let loads = &reports[1]; assert_eq!(loads.len(), markets.len()); assert!( markets .iter() .all(|market| loads.contains_key(&market.condition_id)) ); assert!(loads.values().sum::() > 0); } #[test] fn the_output_reconciles() { // The book and writer threads as main runs them, fed the two // connections' recording. let frames = two_connections(); let (events_tx, events_rx) = crossbeam::channel::unbounded(); let (outputs_tx, outputs_rx) = crossbeam::channel::unbounded(); let (connection_tx, _connection_rx) = crossbeam::channel::unbounded(); let (_commands_tx, commands_rx) = crossbeam::channel::unbounded(); let (reports_tx, _reports_rx) = crossbeam::channel::unbounded(); for (received_at, connection, text) in &frames { for arrival in arrivals(*received_at, *connection, text) { events_tx.send(Received::Event(arrival)).unwrap(); } } drop(events_tx); let feed = feed_on(&fixture_markets(), &[0, 1], frames[0].0); let book = std::thread::spawn(move || { serve( feed, vec![connection_tx.clone(), connection_tx], events_rx, commands_rx, outputs_tx, reports_tx, ) }); let path = std::env::temp_dir().join(format!("reconcile-{}.sbe", std::process::id())); let written = crate::write(path.to_str().unwrap(), outputs_rx); book.join().unwrap(); written.unwrap(); let report = engineering_challenge::reconcile::reconcile(&path, None); std::fs::remove_file(&path).unwrap(); let report = report.unwrap(); // The opening Snapshots, then one per symbol at each of the two // minutes crossed (20 of the 40 tokens had an order book then). assert_eq!(report.snapshots, 20 * 3, "{report:?}"); assert!(report.deltas > 100, "{report:?}"); assert_eq!(report.orphan_deltas, 0); assert_eq!(report.unexplained, []); } }