This repository has been archived on 2026-11-16. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
orderbook-collector/src/polymarket/feed.rs
T
pascal 480ee3ad2d feat(polymarket): subscription monitoring and connection recycling
The exchange can fail without telling us, and first one wins hides a
subscription gone quiet on one of a symbol's two connections.

- A connection that receives nothing for 30 s, not even the keepalive's
  answer, reconnects.
- Each book thread counts a symbol's changes per connection every minute:
  a subscription with none against three or more on the other is
  resubscribed, and moved to another connection if it stays quiet.
- The orchestrator recycles one connection at a time, each about hourly,
  and never while another connection is down or just back.
- A reconnect's opening book of a symbol unchanged for 10 s is judged like
  a resync's answer: one newer than ours that differs shows a symbol quiet
  on both connections, written as a Snapshot and logged as a loss.
2026-11-11 17:42:27 +00:00

3287 lines
129 KiB
Rust

//! A book thread: folds the events its markets' connections send into one book
//! per symbol, first one wins: a change is taken from whichever connection
//! delivers it first, Polymarket's book hash saying where it belongs
//! (ADR 0003). It resyncs when a change fits on no connection in time, and
//! emits Deltas and Snapshots. It also looks for what first one wins hides:
//! a subscription gone quiet on one connection, each minute, and symbols
//! quiet on both, in a reconnect's opening books. The rules are in
//! `roadmap/04. redundant connections.md` and
//! `roadmap/05. subscription monitoring.md`, the protocol facts and the hash
//! formula in `docs/notes.md`, the terms in `GLOSSARY.md`.
use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
use std::time::Duration;
use chrono::{DateTime, DurationRound, TimeDelta, Utc};
use crossbeam::channel::{Receiver, Sender, never, select};
use engineering_challenge::Side;
use engineering_challenge::cause::{self, Cause};
use engineering_challenge::util::dtos::challenge_orderbook::PriceLevelUpdateDomain;
use serde::Serialize;
use sha1::{Digest, Sha1};
use super::Market;
use super::socket::Command;
use super::wire::{Arrival, BookMessage, Event, LevelMessage, PriceChangeMessage, Received};
use engineering_challenge::book::{Book, Output, format_fixed, parse_fixed};
/// A market measured for less than this waits for the next load report: over
/// a few seconds, its opening books alone would read as a burst.
const MIN_MEASURED: TimeDelta = TimeDelta::seconds(30);
/// How many of a symbol's last changes are remembered: a slower connection's
/// copy is recognised among them (it ran up to 82 behind, docs/notes.md), and
/// so is a fresh book behind our stream.
const RECENT_CHANGES: usize = 128;
/// How long a change may be held while another connection might still deliver
/// the one before it: the second delivery lagged up to 0.7 s.
const HOLD: TimeDelta = TimeDelta::seconds(1);
/// How long once no other connection can: only the rest of a batch is still to
/// come, from the same connection (40 ms at worst).
const HOLD_ALONE: TimeDelta = TimeDelta::milliseconds(50);
/// How often a book thread with held changes wakes to check their ceilings.
const EXPIRE_EVERY: Duration = Duration::from_millis(10);
/// The most changes a connection holds for a symbol: past it, its ceiling is
/// reached at once. A burst (thousands a second at a crypto market's
/// 5-minute mark) would otherwise pile up until the ceiling, each one tried
/// against the book.
const MAX_HELD: usize = 512;
/// How many tails of a connection's held changes are tried against the book:
/// the whole, and the last few that start on a batch.
const TAILS_TRIED: usize = 4;
/// When a change's copies are counted: no second copy came later than 0.7 s.
const COPIES_BY: TimeDelta = TimeDelta::seconds(2);
/// A connection that delivered none of a symbol's changes in a minute while
/// another delivered at least this many is quiet for it (sprint 05: crossed
/// once in 21 live minutes; at 1, copies landing in the next minute would
/// count).
const QUIET_AGAINST: u32 = 3;
/// A reconnect's opening book is judged only for a symbol unchanged this
/// long: a moving symbol's books run ahead of or behind our stream.
const STILL_FOR: TimeDelta = TimeDelta::seconds(10);
/// What the book thread does with an event or a command: write output, ask a
/// connection (by number) for something, or report to the orchestrator.
#[derive(Debug)]
pub enum Effect {
Write(Output),
Send(usize, Command),
Report(Report),
}
/// What a book thread tells the orchestrator.
#[derive(Debug, PartialEq, Eq)]
pub enum Report {
Loads(Loads),
/// The market stayed quiet on the connection after being resubscribed
/// there: its subscription should move.
Quiet {
market: String,
connection: usize,
},
}
/// Each market's load (`conditionId` → bytes per second on one connection),
/// every market of the book thread measured for at least [`MIN_MEASURED`].
pub type Loads = HashMap<String, u64>;
/// What the orchestrator tells a book thread. A market's commands all go to
/// its one book thread, in order.
#[derive(Debug)]
pub enum BookCommand {
/// A connection just opened, and how to reach it: sent to every book
/// thread before any command that names it.
Connected(usize, Sender<Command>),
/// Track the market, on these connections.
Add(Market, Vec<usize>),
/// The market's subscription on `from` goes to `to`.
Move {
market: Market,
from: usize,
to: usize,
},
Remove(Market),
}
/// Polymarket's book hash, as bytes: 20 instead of 40 as text, for 128 of
/// them per symbol.
type Hash = [u8; 20];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum State {
/// No book yet since subscribing, or since every connection carrying it
/// went down: changes are dropped until one comes.
AwaitingBook,
/// Its book gives the exchange's hash after every change: first one wins.
Verified,
/// A change fitted on no connection in time: this connection's changes
/// are applied unchecked, the others' dropped, until a fresh book.
Following(usize),
/// A complete book didn't match its own hash, so our formula or metadata
/// is wrong for it: this connection's changes are applied unchecked, for
/// good.
Unverifiable(usize),
}
/// A change that doesn't fit the book yet, kept unwritten.
#[derive(Debug, Clone)]
struct Held {
received_at: DateTime<Utc>,
exchange_timestamp: DateTime<Utc>,
/// As the exchange wrote it: the hash takes it so.
timestamp: String,
side: Side,
price: i64,
quantity: i64,
hash: Hash,
}
impl Held {
fn same(&self, other: &Held) -> bool {
self.key() == other.key()
}
/// What tells a change from another: two copies have the same.
fn key(&self) -> (Hash, Side, i64, i64) {
(self.hash, self.side, self.price, self.quantity)
}
}
impl RecentChange {
fn is_copy(&self, change: &Held) -> bool {
(self.hash, self.side, self.price, self.quantity) == change.key()
}
}
/// A symbol's subscription on one of its connections, as watched for going
/// quiet.
#[derive(Debug, Clone, Copy)]
struct Watch {
connection: usize,
/// Its changes received from this connection this minute: every one,
/// whatever first one wins then made of it.
received: u32,
/// When it was last subscribed there: a minute it didn't cover whole is
/// not judged.
since: DateTime<Utc>,
/// When the connection last delivered a complete book of it: its first
/// since the connection came up is the reconnect's.
book_at: DateTime<Utc>,
}
impl Watch {
fn new(connection: usize, since: DateTime<Utc>) -> Self {
Self {
connection,
received: 0,
since,
book_at: DateTime::UNIX_EPOCH,
}
}
}
/// A change as applied, with the exchange's hash of the book after it.
struct RecentChange {
side: Side,
price: i64,
quantity: i64,
/// The level's quantity before it (0 if absent): what undoes it.
previous: i64,
hash: Hash,
/// When its first copy came: a later copy's lag is measured from it.
received_at: DateTime<Utc>,
/// How many copies came: one, of a symbol on two connections, is a change
/// the other missed.
copies: u8,
/// Its copies are counted, or not to be: applied unchecked, the others'
/// copies were dropped unseen.
counted: bool,
}
struct Symbol {
state: State,
book: Book,
/// Of the last change applied, or of the book that replaced ours: a change
/// older than this is a late copy.
exchange_timestamp: DateTime<Utc>,
/// Its last [`RECENT_CHANGES`] changes.
recent_changes: VecDeque<RecentChange>,
/// By connection: the changes it delivered that don't fit yet.
held: BTreeMap<usize, Vec<Held>>,
/// When the book last moved: held changes older than this, on a
/// connection whose changes still don't fit, are that connection's lag.
advanced_at: DateTime<Utc>,
/// The connection a fresh book was asked from, and whether it is written
/// whatever it holds: the connection followed changed.
asked: Option<(usize, bool)>,
/// Whether a change was verified since its first book: a fresh book that
/// differs before then shows a subscription gap, not a loss.
verified_since_book: bool,
/// It had a book when every connection carrying it went down: the
/// reconnect's book is compared with it.
had_book: bool,
/// The connections it is subscribed on (three while a move overlaps).
connections: Vec<usize>,
/// One per connection in `connections`.
watches: Vec<Watch>,
/// The connection it was quiet on last minute, and resubscribed.
quiet_on: Option<usize>,
/// The receive time of its last Delta: a held change is stamped no earlier,
/// so replaying the output by time keeps its order.
last_written: DateTime<Utc>,
// The other inputs of the book hash, as the exchange writes them.
market: String,
tick_size: String,
last_trade_price: String,
min_order_size: String,
neg_risk: bool,
}
impl Symbol {
fn new(market: &Market, connections: &[usize], now: DateTime<Utc>) -> Self {
Self {
state: State::AwaitingBook,
book: Book::default(),
exchange_timestamp: DateTime::UNIX_EPOCH,
recent_changes: VecDeque::new(),
held: BTreeMap::new(),
advanced_at: DateTime::UNIX_EPOCH,
asked: None,
verified_since_book: false,
had_book: false,
connections: connections.to_vec(),
watches: connections
.iter()
.map(|&connection| Watch::new(connection, now))
.collect(),
quiet_on: None,
last_written: DateTime::UNIX_EPOCH,
market: market.condition_id.clone(),
tick_size: String::new(),
last_trade_price: String::new(),
min_order_size: market.min_order_size.clone(),
neg_risk: market.neg_risk,
}
}
/// Asks `connection` for a fresh book: unsubscribe and subscribe again.
fn ask_fresh_book(
&mut self,
token: &str,
connection: usize,
switch: bool,
effects: &mut Vec<Effect>,
) {
// A switch already asked stays one until its book comes.
let switch = switch || self.asked.is_some_and(|(_, switch)| switch);
self.asked = Some((connection, switch));
effects.push(Effect::Send(
connection,
Command::Resubscribe(token.to_string()),
));
}
fn watch_mut(&mut self, connection: usize) -> Option<&mut Watch> {
self.watches
.iter_mut()
.find(|watch| watch.connection == connection)
}
/// The connection quiet for it over the minute begun at `started`: one
/// that delivered none of its changes while another delivered at least
/// [`QUIET_AGAINST`], both up and subscribed for the whole minute.
fn quiet(&self, started: DateTime<Utc>, up_since: &[Option<DateTime<Utc>>]) -> Option<usize> {
let judged: Vec<&Watch> = self
.watches
.iter()
.filter(|watch| {
watch.since <= started
&& up_since
.get(watch.connection)
.copied()
.flatten()
.is_some_and(|up| up <= started)
})
.collect();
let busiest = judged.iter().map(|watch| watch.received).max()?;
(busiest >= QUIET_AGAINST)
.then(|| judged.iter().find(|watch| watch.received == 0))
.flatten()
.map(|watch| watch.connection)
}
/// One of its connections that is up, other than `except`.
fn other_up(&self, up: &[bool], except: usize) -> Option<usize> {
self.connections
.iter()
.copied()
.find(|&connection| connection != except && is_up(up, connection))
}
/// Whether `book` is ours as it was after one of our recent changes: ours
/// rewound, each change undone. Comparing `book` plus our later changes
/// with ours instead would be unsound: a lost change at a price we had
/// changed before would be overwritten by that earlier change.
fn was_ours(&self, book: &Book) -> bool {
let mut rewound = self.book.clone();
self.recent_changes.iter().rev().any(|change| {
rewound.set(change.side, change.price, change.previous);
rewound == *book
})
}
/// `book` with our recent changes, from the `from`-th on, applied.
fn brought_forward(&self, book: &Book, from: usize) -> Book {
let mut book = book.clone();
for change in self.recent_changes.iter().skip(from) {
book.set(change.side, change.price, change.quantity);
}
book
}
/// Applies a change, `checked` against its hash or not, and remembers it;
/// its Delta, unless it leaves the book as it was.
fn apply(
&mut self,
token: &str,
change: &Held,
now: DateTime<Utc>,
checked: bool,
counts: &mut Counts,
) -> Option<Output> {
let previous = self.book.set(change.side, change.price, change.quantity);
if self.recent_changes.len() == RECENT_CHANGES
&& let Some(mut oldest) = self.recent_changes.pop_front()
{
Self::count_copies(&mut oldest, self.connections.len(), counts);
}
self.recent_changes.push_back(RecentChange {
side: change.side,
price: change.price,
quantity: change.quantity,
previous,
hash: change.hash,
received_at: change.received_at,
copies: 1,
counted: !checked,
});
self.exchange_timestamp = change.exchange_timestamp;
self.advanced_at = now;
if previous == change.quantity {
return None;
}
self.last_written = self.last_written.max(change.received_at);
Some(Output::Delta(PriceLevelUpdateDomain {
backtest_timestamp: self.last_written,
exchange_timestamp: change.exchange_timestamp,
price_level: change.price,
quantity: change.quantity,
side: change.side,
symbol: token.to_string(),
}))
}
/// A change delivered by one connection only, of a symbol on several:
/// rescued.
fn count_copies(change: &mut RecentChange, connections: usize, counts: &mut Counts) {
if !change.counted {
change.counted = true;
if change.copies == 1 && connections > 1 {
counts.rescued += 1;
}
}
}
/// Counts the copies of its changes old enough for every copy to have
/// come.
fn count_old_copies(&mut self, now: DateTime<Utc>, counts: &mut Counts) {
let connections = self.connections.len();
for change in &mut self.recent_changes {
if now - change.received_at > COPIES_BY {
Self::count_copies(change, connections, counts);
}
}
}
/// Where the shortest tail of `connection`'s held changes that gives the
/// hash of its last one starts: that tail fits; what comes before it is
/// already in the book. A tail is whole batches, so only those starting a
/// batch are tried: the whole list, and the last [`TAILS_TRIED`].
fn fit(&self, token: &str, connection: usize) -> Option<usize> {
let held = self.held.get(&connection)?;
let last = held.last()?;
let batch_starts = (1..held.len())
.rev()
.filter(|&start| held[start].hash != held[start - 1].hash)
.take(TAILS_TRIED);
batch_starts.chain([0]).find(|&start| {
let mut book = self.book.clone();
for change in &held[start..] {
book.set(change.side, change.price, change.quantity);
}
hash(token, self, &book, &last.timestamp) == last.hash
})
}
/// Drops the held changes that the book now holds: copies of applied
/// changes, and changes older than its last.
fn prune(&mut self) {
let (recent, newest) = (&self.recent_changes, self.exchange_timestamp);
for held in self.held.values_mut() {
held.retain(|change| {
change.exchange_timestamp >= newest
&& !recent.iter().any(|applied| applied.is_copy(change))
});
}
self.held.retain(|_, held| !held.is_empty());
}
}
fn is_up(up: &[bool], connection: usize) -> bool {
up.get(connection).copied().unwrap_or(false)
}
/// A market's frame bytes since `since`, from all of its connections.
struct Measure {
bytes: u64,
since: DateTime<Utc>,
}
impl Measure {
/// Bytes per second since `since` on one of `connections`, then starts
/// again from `now`; none before [`MIN_MEASURED`], the bytes kept for the
/// next report.
fn take(&mut self, now: DateTime<Utc>, connections: usize) -> Option<u64> {
let measured = now - self.since;
if measured < MIN_MEASURED {
return None;
}
let load =
self.bytes * 1_000 / measured.num_milliseconds() as u64 / connections.max(1) as u64;
*self = Measure {
bytes: 0,
since: now,
};
Some(load)
}
}
/// A market of the book thread.
struct Tracked {
tokens: Vec<String>,
measure: Measure,
/// A move under way: the connection left once the new one, the second
/// here, delivers a book of the market. Until then both carry it, so the
/// move costs no change.
leaving: Option<(usize, usize)>,
}
/// What the minute line counts, since the start.
#[derive(Debug, Default)]
struct Counts {
written: u64,
verified: u64,
duplicates: u64,
/// Changes of a symbol on several connections that only one delivered:
/// the others missed them.
rescued: u64,
/// Held changes dropped at their ceiling, the book having moved on
/// without them: their connection was behind.
held_dropped: u64,
chains_broken: u64,
resyncs: u64,
/// Fresh books that were a state we had already passed.
books_behind: u64,
subscription_gaps: u64,
losses: u64,
switches: u64,
/// Subscriptions found quiet, each resubscribed.
quiet: u64,
/// Markets quiet again after their resubscribe: reported for a move.
quiet_reported: u64,
/// A reconnect's opening books judged, those of still symbols.
reconnect_books: u64,
/// Of them, books that differed from every state of ours: both
/// subscriptions were quiet.
reconnect_losses: u64,
}
pub struct Feed {
/// Its book thread's number, for the logs.
number: usize,
symbols: BTreeMap<String, Symbol>,
/// By `conditionId`.
markets: HashMap<String, Tracked>,
/// By connection number: whether it is up.
up: Vec<bool>,
/// By connection number: since when it is up, if it is.
up_since: Vec<Option<DateTime<Utc>>>,
/// When the minute being counted began: its quiet subscriptions are
/// judged at its end. None before the first.
minute_started: Option<DateTime<Utc>>,
/// Changes received this minute, from all connections.
received_this_minute: u64,
/// The symbols with held changes: their ceilings are checked.
waiting: BTreeSet<String>,
/// The receive time of the last event applied: the time a Snapshot's book
/// is exact as of.
last_received_at: Option<DateTime<Utc>>,
/// The next wall-clock minute: its Snapshots, load report and log line.
next_minute_at: Option<DateTime<Utc>>,
counts: Counts,
/// How long after its first copy each duplicate came this minute, µs.
lags: Vec<i64>,
/// The most events waiting for the book thread since the last minute log:
/// how far behind it fell.
peak_waiting: usize,
}
impl Feed {
pub fn new(number: usize) -> Self {
Self {
number,
symbols: BTreeMap::new(),
markets: HashMap::new(),
up: Vec::new(),
up_since: Vec::new(),
minute_started: None,
received_this_minute: 0,
waiting: BTreeSet::new(),
last_received_at: None,
next_minute_at: None,
counts: Counts::default(),
lags: Vec::new(),
peak_waiting: 0,
}
}
/// Tracks a market's symbols, then asks its connections to subscribe
/// them: a book can't arrive for a symbol not yet known. Its load is
/// measured from `now`.
pub fn add_market(
&mut self,
market: &Market,
connections: &[usize],
now: DateTime<Utc>,
) -> Vec<Effect> {
for token in &market.tokens {
self.symbols
.insert(token.clone(), Symbol::new(market, connections, now));
}
let tracked = Tracked {
tokens: market.tokens.clone(),
measure: Measure {
bytes: 0,
since: now,
},
leaving: None,
};
self.markets.insert(market.condition_id.clone(), tracked);
connections
.iter()
.map(|&connection| Effect::Send(connection, Command::Subscribe(market.tokens.clone())))
.collect()
}
/// Moves a market's subscription from one connection to another: it
/// subscribes on `to` now, and leaves `from` once `to` delivers a book of
/// the market. The book stays as it is: no Snapshot, no subscription gap.
pub fn move_market(
&mut self,
market: &Market,
from: usize,
to: usize,
now: DateTime<Utc>,
) -> Vec<Effect> {
let Some(tracked) = self.markets.get_mut(&market.condition_id) else {
return Vec::new();
};
// A move still under way when the next comes: it ends now.
let mut effects = match tracked.leaving.take() {
Some((left, _)) => self.leave(&market.condition_id, left),
None => Vec::new(),
};
if let Some(tracked) = self.markets.get_mut(&market.condition_id) {
tracked.leaving = Some((from, to));
}
for token in &market.tokens {
if let Some(symbol) = self.symbols.get_mut(token)
&& !symbol.connections.contains(&to)
{
symbol.connections.push(to);
symbol.watches.push(Watch::new(to, now));
}
}
effects.push(Effect::Send(to, Command::Subscribe(market.tokens.clone())));
effects
}
/// The market's symbols stop being carried by `connection`: it is
/// unsubscribed, and a symbol following it follows another.
fn leave(&mut self, condition_id: &str, connection: usize) -> Vec<Effect> {
let Some(tracked) = self.markets.get(condition_id) else {
return Vec::new();
};
let tokens = tracked.tokens.clone();
let mut effects = vec![Effect::Send(
connection,
Command::Unsubscribe(tokens.clone()),
)];
for token in &tokens {
if let Some(symbol) = self.symbols.get_mut(token) {
symbol.connections.retain(|&on| on != connection);
symbol
.watches
.retain(|watch| watch.connection != connection);
symbol.held.remove(&connection);
Self::lose(
token,
symbol,
connection,
&self.up,
&mut self.counts,
&mut effects,
);
// A fresh book asked from it would never come: ask another.
if let Some((asked, switch)) = symbol.asked
&& asked == connection
&& let Some(other) = symbol.other_up(&self.up, connection)
{
symbol.ask_fresh_book(token, other, switch, &mut effects);
self.counts.resyncs += 1;
}
}
}
effects
}
/// Forgets a market's symbols and unsubscribes them; nothing is written
/// for them. Their events still on the way are ignored, as any unknown
/// token's.
pub fn remove_market(&mut self, market: &Market) -> Vec<Effect> {
let mut connections = Vec::new();
for token in &market.tokens {
if let Some(symbol) = self.symbols.remove(token) {
connections = symbol.connections;
}
self.waiting.remove(token);
}
self.markets.remove(&market.condition_id);
connections
.into_iter()
.map(|connection| Effect::Send(connection, Command::Unsubscribe(market.tokens.clone())))
.collect()
}
/// What a connection sent: an event, or itself coming up or going down.
pub fn on_received(&mut self, received: Received) -> Vec<Effect> {
match received {
Received::Event(arrival) => self.on_event(arrival),
Received::Up(connection) => {
// Its first events come after this marker; any other
// connection's before it were received earlier.
let at = self.last_received_at.unwrap_or(DateTime::UNIX_EPOCH);
self.set_up(connection, true, at);
Vec::new()
}
Received::Down(connection) => self.on_down(connection),
}
}
fn set_up(&mut self, connection: usize, up: bool, at: DateTime<Utc>) {
if self.up.len() <= connection {
self.up.resize(connection + 1, false);
self.up_since.resize(connection + 1, None);
}
if up && !self.up[connection] {
self.up_since[connection] = Some(at);
} else if !up {
self.up_since[connection] = None;
}
self.up[connection] = up;
}
pub fn on_event(&mut self, arrival: Arrival) -> Vec<Effect> {
let (received_at, connection) = (arrival.received_at, arrival.connection);
// A connection that delivers is up, its marker possibly still behind.
if !is_up(&self.up, connection) {
self.set_up(connection, true, received_at);
}
// Minute Snapshots due before this event hold the books as they were.
let mut effects = self.tick(received_at);
self.last_received_at = Some(received_at);
if let Some(tracked) = arrival
.event
.market()
.and_then(|market| self.markets.get_mut(market))
{
tracked.measure.bytes += arrival.bytes;
}
match arrival.event {
Event::Book(book) => self.on_book(received_at, connection, book, &mut effects),
Event::PriceChange(change) => {
self.on_price_change(received_at, connection, change, &mut effects)
}
Event::TickSizeChange { asset_id, .. } => {
if let Some(symbol) = self.symbols.get_mut(&asset_id)
&& symbol.state == State::Verified
{
log!("polymarket: {asset_id}: tick size changed, resync");
Self::break_chain(
&asset_id,
symbol,
connection,
received_at,
&self.up,
&mut self.counts,
&mut effects,
);
}
}
// The hash takes a new last trade price before this event, and
// not always its value, so a fresh book brings it instead
// (docs/notes.md).
Event::LastTradePrice { .. } | Event::Other => {}
}
effects.extend(self.expire(received_at));
effects
}
/// While no event comes: the minute, and the held changes' ceilings.
pub fn idle(&mut self, now: DateTime<Utc>) -> Vec<Effect> {
let mut effects = self.tick(now);
effects.extend(self.expire(now));
effects
}
/// Whether the book thread must wake soon for held changes.
fn holds(&self) -> bool {
!self.waiting.is_empty()
}
fn on_down(&mut self, connection: usize) -> Vec<Effect> {
self.set_up(connection, false, DateTime::UNIX_EPOCH);
let mut effects = Vec::new();
for (token, symbol) in &mut self.symbols {
if !symbol.connections.contains(&connection) {
continue;
}
symbol.held.remove(&connection);
Self::lose(
token,
symbol,
connection,
&self.up,
&mut self.counts,
&mut effects,
);
}
self.waiting.retain(|token| {
self.symbols
.get(token)
.is_some_and(|symbol| !symbol.held.is_empty())
});
effects
}
/// `connection` no longer delivers for the symbol (it went down, or the
/// symbol left it): a symbol following it follows another, whose fresh
/// book is written whatever it holds; one with no connection left up
/// awaits a book.
fn lose(
token: &str,
symbol: &mut Symbol,
connection: usize,
up: &[bool],
counts: &mut Counts,
effects: &mut Vec<Effect>,
) {
let other = symbol.other_up(up, connection);
match (symbol.state, other) {
(State::AwaitingBook, _) => {}
(State::Following(followed) | State::Unverifiable(followed), Some(other))
if followed == connection =>
{
symbol.state = match symbol.state {
State::Unverifiable(_) => State::Unverifiable(other),
_ => State::Following(other),
};
symbol.ask_fresh_book(token, other, true, effects);
counts.resyncs += 1;
}
(_, None) => {
// Nothing carries it: changes are being missed.
symbol.state = State::AwaitingBook;
symbol.had_book = true;
symbol.held.clear();
symbol.asked = None;
}
_ => {}
}
}
/// Once `now` reaches the next wall-clock minute: reports the loads,
/// emits the minute Snapshots, asks again for the fresh books still not
/// come, and resubscribes (or reports) the subscriptions quiet over the
/// minute that ends.
pub fn tick(&mut self, now: DateTime<Utc>) -> Vec<Effect> {
let next = *self.next_minute_at.get_or_insert_with(|| next_minute(now));
if now < next {
return Vec::new();
}
self.next_minute_at = Some(next_minute(now));
let started = self.minute_started.replace(now);
let mut loads = Loads::new();
for (market, tracked) in &mut self.markets {
let connections = tracked
.tokens
.first()
.and_then(|token| self.symbols.get(token))
.map_or(1, |symbol| symbol.connections.len());
if let Some(load) = tracked.measure.take(now, connections) {
loads.insert(market.clone(), load);
}
}
let mut effects = vec![Effect::Report(Report::Loads(loads))];
let Some(received_at) = self.last_received_at else {
return effects;
};
self.log_minute();
let mut reported = BTreeSet::new();
for (token, symbol) in &mut self.symbols {
symbol.count_old_copies(now, &mut self.counts);
let quiet = started
.filter(|_| symbol.state == State::Verified && symbol.asked.is_none())
.and_then(|started| symbol.quiet(started, &self.up_since));
for watch in &mut symbol.watches {
watch.received = 0;
}
match quiet {
// Resubscribed already, and still quiet: it should move.
Some(connection) if symbol.quiet_on == Some(connection) => {
symbol.quiet_on = None;
if reported.insert((symbol.market.clone(), connection)) {
self.counts.quiet_reported += 1;
effects.push(Effect::Report(Report::Quiet {
market: symbol.market.clone(),
connection,
}));
}
}
// The other connection carries it meanwhile: the book that
// comes back is ignored, no Snapshot, no subscription gap.
Some(connection) => {
log!("polymarket: {token}: quiet on connection {connection}, resubscribed");
self.counts.quiet += 1;
symbol.quiet_on = Some(connection);
if let Some(watch) = symbol.watch_mut(connection) {
watch.since = now;
}
effects.push(Effect::Send(
connection,
Command::Resubscribe(token.clone()),
));
}
None => symbol.quiet_on = None,
}
if symbol.state != State::AwaitingBook {
effects.push(Effect::Write(symbol.book.snapshot(
token,
received_at,
symbol.exchange_timestamp,
)));
// A change held across the minute is written after it.
symbol.last_written = symbol.last_written.max(received_at);
}
// The answer may have been lost or unreadable: ask again.
let again = match (symbol.state, symbol.asked) {
(_, Some((connection, _))) => Some(connection),
(State::AwaitingBook, None) => symbol
.connections
.iter()
.copied()
.find(|&connection| is_up(&self.up, connection)),
_ => None,
};
if let Some(connection) = again {
let switch = symbol.asked.is_some_and(|(_, switch)| switch);
symbol.ask_fresh_book(token, connection, switch, &mut effects);
self.counts.resyncs += 1;
}
}
effects
}
fn log_minute(&mut self) {
self.lags.sort_unstable();
let lag = |quantile: f64| {
let index = ((self.lags.len() as f64 * quantile) as usize)
.min(self.lags.len().saturating_sub(1));
self.lags
.get(index)
.map_or(0.0, |&micros| micros as f64 / 1_000.0)
};
let counts = &self.counts;
log!(
"polymarket: book thread {}: {} symbols; so far {} changes written ({} verified), {} duplicates, \
{} rescued, {} held dropped, {} chains broken, {} resyncs, {} full books behind ours, \
{} subscription gaps, {} losses, {} switches, {} quiet subscriptions resubscribed, \
{} markets reported quiet again, {} reconnect books judged ({} losses); second copies \
{:.1} ms late at the median, {:.1} ms at p99; this minute {} changes received, at most \
{} events waited",
self.number,
self.symbols.len(),
counts.written,
counts.verified,
counts.duplicates,
counts.rescued,
counts.held_dropped,
counts.chains_broken,
counts.resyncs,
counts.books_behind,
counts.subscription_gaps,
counts.losses,
counts.switches,
counts.quiet,
counts.quiet_reported,
counts.reconnect_books,
counts.reconnect_losses,
lag(0.5),
lag(0.99),
self.received_this_minute,
self.peak_waiting
);
self.lags.clear();
self.peak_waiting = 0;
self.received_this_minute = 0;
}
/// Held changes past their ceiling: dropped if the book moved on without
/// them (their connection is behind), else the chain is broken and their
/// connection followed.
fn expire(&mut self, now: DateTime<Utc>) -> Vec<Effect> {
let mut effects = Vec::new();
let mut done = Vec::new();
for token in &self.waiting {
let Some(symbol) = self.symbols.get_mut(token) else {
done.push(token.clone());
continue;
};
let due = symbol.held.iter().find_map(|(&connection, held)| {
let first = held.first()?;
// Another connection that is up and doesn't hold the same
// change could still deliver the one before it.
let others_can = symbol.connections.iter().any(|&other| {
other != connection
&& is_up(&self.up, other)
&& symbol
.held
.get(&other)
.and_then(|held| held.first())
.is_none_or(|theirs| !theirs.same(first))
});
let ceiling = if others_can { HOLD } else { HOLD_ALONE };
let due = now - first.received_at > ceiling || held.len() > MAX_HELD;
due.then_some((connection, first.received_at))
});
if let Some((connection, since)) = due {
if symbol.advanced_at > since {
let dropped = symbol.held.remove(&connection).map_or(0, |held| held.len());
self.counts.held_dropped += dropped as u64;
} else {
Self::break_chain(
token,
symbol,
connection,
now,
&self.up,
&mut self.counts,
&mut effects,
);
}
}
if symbol.held.is_empty() {
done.push(token.clone());
}
}
for token in done {
self.waiting.remove(&token);
}
effects
}
/// No connection's changes fit: `connection`'s are applied unchecked from
/// now, and a fresh book is asked from another, so the followed stream is
/// not interrupted.
fn break_chain(
token: &str,
symbol: &mut Symbol,
connection: usize,
now: DateTime<Utc>,
up: &[bool],
counts: &mut Counts,
effects: &mut Vec<Effect>,
) {
counts.chains_broken += 1;
symbol.state = State::Following(connection);
for change in symbol.held.remove(&connection).unwrap_or_default() {
if let Some(delta) = symbol.apply(token, &change, now, false, counts) {
counts.written += 1;
effects.push(Effect::Write(delta));
}
}
symbol.held.clear();
let asked = symbol.other_up(up, connection).unwrap_or(connection);
symbol.ask_fresh_book(token, asked, false, effects);
counts.resyncs += 1;
}
/// A full book counts when asked for (a symbol's first, from any of its
/// connections, or the answer to a resync), or when it is a reconnect's
/// opening book of a symbol unchanged for [`STILL_FOR`]. It replaces
/// ours, brought forward first if it is behind our stream, and is written
/// as a Snapshot when it is the symbol's first, when it differs from
/// ours, or when the connection followed changed. Then it is checked
/// against its own hash. A reconnect's book that is ours, or behind
/// ours, changes nothing at all.
fn on_book(
&mut self,
received_at: DateTime<Utc>,
connection: usize,
message: BookMessage,
effects: &mut Vec<Effect>,
) {
let token = message.asset_id;
// A move ends once the new connection delivers a book of the market.
let leaving = self
.markets
.get(&message.market)
.and_then(|tracked| tracked.leaving)
.filter(|&(_, to)| to == connection);
if let Some((from, _)) = leaving {
if let Some(tracked) = self.markets.get_mut(&message.market) {
tracked.leaving = None;
}
effects.extend(self.leave(&message.market, from));
}
let Some(symbol) = self.symbols.get_mut(&token) else {
return;
};
// Books on subscribe carry every field; those sent mid-stream, after
// a trade, omit these two and are never an answer.
let complete = message.last_trade_price.is_some();
let asked = symbol.asked.filter(|&(asked, _)| asked == connection);
let first = symbol.state == State::AwaitingBook && symbol.connections.contains(&connection);
// The connection's first complete book since it came up, for a
// symbol subscribed there before: the reconnect's opening book.
let up_since = self.up_since.get(connection).copied().flatten();
let reconnect = complete
&& symbol.watch_mut(connection).is_some_and(|watch| {
// A book at the very instant it came up came before it.
let opening = up_since.is_some_and(|up| watch.since < up && watch.book_at <= up);
watch.book_at = received_at;
opening
});
// Judged only for a still, verified symbol, nothing held or asked.
let reconnect = reconnect
&& asked.is_none()
&& symbol.asked.is_none()
&& symbol.state == State::Verified
&& symbol.held.is_empty()
&& received_at - symbol.advanced_at >= STILL_FOR;
if !complete || (asked.is_none() && !first && !reconnect) {
return;
}
let (Some(book), Some(exchange_timestamp), Some(book_hash)) = (
parse_book(&message.bids, &message.asks),
parse_millis(&message.timestamp),
parse_hash(&message.hash),
) else {
// Nobody asked for a reconnect's book: unreadable, it is ignored.
if reconnect {
return;
}
// A protocol error: ask again. A book that stays unreadable is asked
// for once per answer, and its symbol stays unchecked meanwhile.
log!("polymarket: {token}: unreadable book, asking for a fresh one");
symbol.ask_fresh_book(&token, connection, false, effects);
self.counts.resyncs += 1;
return;
};
if reconnect {
self.counts.reconnect_books += 1;
// The exchange's opening books lag by minutes (docs/notes.md): one
// that is ours, or a state we passed, leaves ours, and the hash
// inputs, as they are.
let brought = symbol
.recent_changes
.iter()
.rposition(|change| change.hash == book_hash)
.map(|position| symbol.brought_forward(&book, position + 1));
if book == symbol.book || brought.as_ref() == Some(&symbol.book) {
return;
}
// Our chain holds up to our last change, so a book no newer than
// it holds nothing we missed, however far back it lags: further
// than the changes we remember, too.
if exchange_timestamp <= symbol.exchange_timestamp || symbol.was_ours(&book) {
self.counts.books_behind += 1;
return;
}
}
let inputs = reconnect.then(|| (symbol.tick_size.clone(), symbol.last_trade_price.clone()));
if let Some(tick_size) = message.tick_size {
symbol.tick_size = tick_size;
}
if let Some(last_trade_price) = message.last_trade_price {
symbol.last_trade_price = last_trade_price;
}
// Checked as received, before being brought forward below.
let verified = hash(&token, symbol, &book, &message.timestamp) == book_hash;
if let Some(inputs) = inputs {
// A book failing its own hash proves nothing against a verified
// one: ours stands, with its hash inputs.
if !verified {
(symbol.tick_size, symbol.last_trade_price) = inputs;
return;
}
// Newer than ours and no state of ours: both subscriptions were
// quiet. The other one is renewed too; its book is not waited for.
self.counts.reconnect_losses += 1;
for other in symbol.connections.clone() {
if other != connection {
if let Some(watch) = symbol.watch_mut(other) {
watch.since = received_at;
}
effects.push(Effect::Send(other, Command::Resubscribe(token.clone())));
}
}
}
let switch = asked.is_some_and(|(_, switch)| switch);
let compared = !first || symbol.had_book;
let mut book = book;
let book_time = exchange_timestamp;
let mut exchange_timestamp = exchange_timestamp;
// The book a resync brings can lag the changes already applied, with a
// newer timestamp or last trade price than its levels (docs/notes.md).
// It is behind if it is ours as it was a few changes ago: that past
// state was the exchange's own, and the changes since are what it
// sent us, so ours stands.
if compared && book != symbol.book && symbol.was_ours(&book) {
self.counts.books_behind += 1;
book = symbol.book.clone();
exchange_timestamp = symbol.exchange_timestamp;
} else if let Some(position) = symbol
.recent_changes
.iter()
.rposition(|change| change.hash == book_hash)
.filter(|_| compared)
{
// Its hash is that of one of our changes: the exchange's state
// after it, which the changes we got since bring to now.
book = symbol.brought_forward(&book, position + 1);
exchange_timestamp = symbol.exchange_timestamp;
}
let differs = compared && book != symbol.book;
let cause = if switch {
Some(Cause::Switch)
} else if differs && !reconnect && !symbol.verified_since_book && !symbol.had_book {
// A subscribe hides a few ms of changes (docs/notes.md): until a
// change is verified, a differing book is that gap.
Some(Cause::SubscriptionGap)
} else if differs {
Some(Cause::Loss)
} else {
None
};
match cause {
Some(Cause::Loss) => self.counts.losses += 1,
Some(Cause::SubscriptionGap) => self.counts.subscription_gaps += 1,
Some(Cause::Switch) => self.counts.switches += 1,
None => {}
}
symbol.book = book;
symbol.exchange_timestamp = exchange_timestamp;
symbol.advanced_at = received_at;
symbol.held.clear();
symbol.asked = None;
symbol.had_book = false;
if !compared || cause.is_some() {
// Our changes before this book no longer lead to ours: rewinding
// through them would make up states that never existed.
symbol.recent_changes.clear();
let snapshot = symbol
.book
.snapshot(&token, received_at, exchange_timestamp);
effects.push(Effect::Write(snapshot));
symbol.last_written = symbol.last_written.max(received_at);
if let Some(cause) = cause {
let found = if reconnect {
", found by a reconnect"
} else {
""
};
let detail = format!(
"{found} (its time {}, ours {})",
book_time.format("%H:%M:%S%.3f"),
exchange_timestamp.format("%H:%M:%S%.3f")
);
log!(
"polymarket: {}",
cause::line(&token, received_at, cause, &detail)
);
}
}
symbol.state = if verified {
State::Verified
} else {
if !matches!(symbol.state, State::Unverifiable(_)) {
log!(
"polymarket: {token}: a complete book fails its own hash, changes go unchecked"
);
}
State::Unverifiable(connection)
};
self.waiting.remove(&token);
}
fn on_price_change(
&mut self,
received_at: DateTime<Utc>,
connection: usize,
message: PriceChangeMessage,
effects: &mut Vec<Effect>,
) {
let exchange_timestamp = parse_millis(&message.timestamp);
for change in message.price_changes {
let token = change.asset_id;
let Some(symbol) = self.symbols.get_mut(&token) else {
continue;
};
if let Some(watch) = symbol.watch_mut(connection) {
watch.received += 1;
}
self.received_this_minute += 1;
let side = match change.side.as_str() {
"BUY" => Some(Side::Buy),
"SELL" => Some(Side::Sell),
_ => None,
};
let (Some(side), Some(price), Some(quantity), Some(exchange_timestamp), Some(hash)) = (
side,
parse_fixed(&change.price),
parse_fixed(&change.size),
exchange_timestamp,
parse_hash(&change.hash),
) else {
// Another connection's copy can still bring it; if none does,
// the next change of this one waits for it in vain.
log!(
"polymarket: {token}: unreadable change from connection {connection}, dropped"
);
continue;
};
let change = Held {
received_at,
exchange_timestamp,
timestamp: message.timestamp.clone(),
side,
price,
quantity,
hash,
};
match symbol.state {
State::AwaitingBook => {}
State::Following(followed) | State::Unverifiable(followed) => {
if followed == connection
&& let Some(delta) =
symbol.apply(&token, &change, received_at, false, &mut self.counts)
{
self.counts.written += 1;
effects.push(Effect::Write(delta));
}
}
State::Verified => {
Self::first_one_wins(
&token,
symbol,
connection,
change,
received_at,
&mut self.counts,
&mut self.lags,
effects,
);
if symbol.held.is_empty() {
self.waiting.remove(&token);
} else {
self.waiting.insert(token);
}
}
}
}
}
/// A verified symbol's change: dropped if the book has it, else held on
/// its connection until a tail of its held changes fits; each fit may let
/// another connection's held changes fit in turn.
// The Feed's fields it needs, borrowed apart from the symbol it holds:
// a method on Feed couldn't take `symbol` from `self.symbols` too.
#[allow(clippy::too_many_arguments)]
fn first_one_wins(
token: &str,
symbol: &mut Symbol,
connection: usize,
change: Held,
now: DateTime<Utc>,
counts: &mut Counts,
lags: &mut Vec<i64>,
effects: &mut Vec<Effect>,
) {
if let Some(first) = symbol
.recent_changes
.iter_mut()
.find(|applied| applied.is_copy(&change))
{
counts.duplicates += 1;
first.copies = first.copies.saturating_add(1);
if let Some(lag) = (change.received_at - first.received_at).num_microseconds() {
lags.push(lag);
}
return;
}
// While the chain holds, everything up to the book's state was applied.
if change.exchange_timestamp < symbol.exchange_timestamp {
counts.duplicates += 1;
return;
}
symbol.held.entry(connection).or_default().push(change);
let mut fitted = true;
while fitted {
fitted = false;
let connections: Vec<usize> = symbol.held.keys().copied().collect();
for held_on in connections {
let Some(start) = symbol.fit(token, held_on) else {
continue;
};
let held = symbol.held.remove(&held_on).unwrap_or_default();
for change in &held[start..] {
counts.verified += 1;
if let Some(delta) = symbol.apply(token, change, now, true, counts) {
counts.written += 1;
effects.push(Effect::Write(delta));
}
}
symbol.verified_since_book = true;
symbol.prune();
fitted = true;
break;
}
}
}
}
/// A book thread: runs until every connection and the orchestrator have
/// dropped its channels, then drops its senders so the writer thread finishes
/// once every book thread has.
// ponytail: unbounded channels never block a socket read nor drop an event;
// the ceiling is memory if a consumer stays behind. The minute log's peak of
// events waiting makes that visible; bounded channels are a constructor change
// if it ever matters.
pub fn run(
feed: Feed,
events: Receiver<Received>,
commands: Receiver<BookCommand>,
outputs: Sender<Output>,
reports: Sender<Report>,
) {
serve(feed, Vec::new(), events, commands, outputs, reports)
}
/// [`run`], knowing `connections` already (by number); the orchestrator
/// announces the others as it opens them.
fn serve(
mut feed: Feed,
mut connections: Vec<Sender<Command>>,
events: Receiver<Received>,
commands: Receiver<BookCommand>,
outputs: Sender<Output>,
reports: Sender<Report>,
) {
let no_commands = never();
let mut orchestrator_gone = false;
loop {
let mut wake = (next_minute(Utc::now()) - Utc::now())
.to_std()
.unwrap_or(Duration::ZERO);
if feed.holds() {
wake = wake.min(EXPIRE_EVERY);
}
let commands = if orchestrator_gone {
&no_commands
} else {
&commands
};
let effects = select! {
recv(events) -> received => match received {
Ok(received) => {
feed.peak_waiting = feed.peak_waiting.max(events.len());
feed.on_received(received)
}
Err(_) => return,
},
recv(commands) -> command => match command {
Ok(BookCommand::Connected(number, connection)) => {
assert_eq!(number, connections.len(), "connections open in order");
connections.push(connection);
Vec::new()
}
Ok(BookCommand::Add(market, on)) => feed.add_market(&market, &on, Utc::now()),
Ok(BookCommand::Move { market, from, to }) => {
feed.move_market(&market, from, to, Utc::now())
}
Ok(BookCommand::Remove(market)) => feed.remove_market(&market),
// It is stopping: the connections' end still decides ours.
Err(_) => {
orchestrator_gone = true;
Vec::new()
}
},
default(wake) => feed.idle(Utc::now()),
};
for effect in effects {
match effect {
Effect::Write(output) => {
if outputs.send(output).is_err() {
return log!("polymarket: the writer thread is gone, stopping");
}
}
// The connection may be gone already, while stopping, and the
// orchestrator too.
Effect::Send(connection, command) => _ = connections[connection].send(command),
Effect::Report(report) => _ = reports.send(report),
}
}
}
}
/// Polymarket's book hash: SHA-1 of the book as its REST `/book` returns it,
/// `hash` emptied, keys in this order (docs/notes.md): `book` with the
/// symbol's other inputs.
fn hash(token: &str, symbol: &Symbol, book: &Book, timestamp: &str) -> Hash {
#[derive(Serialize)]
struct Level {
price: String,
size: String,
}
#[derive(Serialize)]
struct HashedBook<'a> {
market: &'a str,
asset_id: &'a str,
timestamp: &'a str,
hash: &'a str,
bids: Vec<Level>,
asks: Vec<Level>,
min_order_size: &'a str,
tick_size: &'a str,
neg_risk: bool,
last_trade_price: &'a str,
}
let level = |(&price, &size): (&i64, &i64)| Level {
price: format_fixed(price),
size: format_fixed(size),
};
let hashed = HashedBook {
market: &symbol.market,
asset_id: token,
timestamp,
hash: "",
// Bids ascending, asks descending: the best level of each side last.
bids: book.bids.iter().map(level).collect(),
asks: book.asks.iter().rev().map(level).collect(),
min_order_size: &symbol.min_order_size,
tick_size: &symbol.tick_size,
neg_risk: symbol.neg_risk,
last_trade_price: &symbol.last_trade_price,
};
let json = serde_json::to_string(&hashed).expect("strings and bools always serialize");
Sha1::digest(json.as_bytes()).into()
}
/// The exchange's 40 lowercase hex characters.
fn parse_hash(text: &str) -> Option<Hash> {
if text.len() != 40 {
return None;
}
let mut hash = [0; 20];
for (byte, pair) in hash.iter_mut().zip(text.as_bytes().chunks(2)) {
*byte = u8::from_str_radix(std::str::from_utf8(pair).ok()?, 16).ok()?;
}
Some(hash)
}
fn parse_book(bids: &[LevelMessage], asks: &[LevelMessage]) -> Option<Book> {
let side = |levels: &[LevelMessage]| {
levels
.iter()
.map(|level| Some((parse_fixed(&level.price)?, parse_fixed(&level.size)?)))
.collect::<Option<BTreeMap<_, _>>>()
};
Some(Book {
bids: side(bids)?,
asks: side(asks)?,
})
}
/// Polymarket timestamps: milliseconds since the epoch, as a string.
fn parse_millis(text: &str) -> Option<DateTime<Utc>> {
DateTime::from_timestamp_millis(text.parse().ok()?)
}
fn next_minute(now: DateTime<Utc>) -> DateTime<Utc> {
now.duration_trunc(TimeDelta::minutes(1))
.expect("a minute divides any UTC time")
+ TimeDelta::minutes(1)
}
#[cfg(test)]
mod tests {
use serde::Deserialize;
use super::*;
use crate::polymarket::{fixture_markets, wire};
/// A frame's events as `connection` would send them.
fn arrivals(received_at: DateTime<Utc>, connection: usize, text: &str) -> Vec<Arrival> {
// The connection logs an unreadable frame and sends nothing.
let events = wire::decode(text).unwrap_or_default();
let bytes = (text.len() / events.len().max(1)) as u64;
events
.into_iter()
.filter(|event| event.market().is_some())
.map(|event| Arrival {
received_at,
connection,
bytes,
event,
})
.collect()
}
impl Feed {
fn on_frame_from(
&mut self,
connection: usize,
received_at: DateTime<Utc>,
text: &str,
) -> Vec<Effect> {
arrivals(received_at, connection, text)
.into_iter()
.flat_map(|arrival| self.on_event(arrival))
.collect()
}
fn on_frame(&mut self, received_at: DateTime<Utc>, text: &str) -> Vec<Effect> {
self.on_frame_from(0, received_at, text)
}
}
/// A frame of the recordings: receive time, connection, text.
type Frame = (DateTime<Utc>, usize, String);
fn read_frames(file: &str) -> Vec<Frame> {
#[derive(Deserialize)]
struct Line {
received_at_us: i64,
#[serde(default)]
connection: usize,
text: String,
}
let path = format!("{}/tests/fixtures/{file}", env!("CARGO_MANIFEST_DIR"));
let mut frames: Vec<Frame> = std::fs::read_to_string(path)
.unwrap()
.lines()
.map(|line| {
let line: Line = serde_json::from_str(line).unwrap();
let received_at = DateTime::from_timestamp_micros(line.received_at_us).unwrap();
(received_at, line.connection, line.text)
})
.collect();
frames.sort_by_key(|(received_at, _, _)| *received_at);
frames
}
/// The recorded live feed for the fixture's markets, on one connection
/// (tests/fixtures/README.md).
fn fixture_frames() -> Vec<(DateTime<Utc>, String)> {
read_frames("polymarket-frames.jsonl")
.into_iter()
.map(|(received_at, _, text)| (received_at, text))
.collect()
}
/// The same markets recorded on two connections at once, in receive order.
fn two_connections() -> Vec<Frame> {
read_frames("polymarket-frames-two.jsonl")
}
/// A book thread's feed holding `markets` on `connections`, as the
/// orchestrator's Adds leave it at `start`.
fn feed_on(markets: &[Market], connections: &[usize], start: DateTime<Utc>) -> Feed {
let mut feed = Feed::new(0);
for market in markets {
feed.add_market(market, connections, start);
}
feed
}
/// The one-connection recording's feed: its markets on connection 0.
fn feed_of(markets: &[Market]) -> Feed {
feed_on(markets, &[0], fixture_frames()[0].0)
}
fn replay<'a>(
feed: &mut Feed,
frames: impl IntoIterator<Item = &'a (DateTime<Utc>, String)>,
) -> Vec<Effect> {
frames
.into_iter()
.flat_map(|(received_at, text)| feed.on_frame(*received_at, text))
.collect()
}
fn replay_both<'a>(
feed: &mut Feed,
frames: impl IntoIterator<Item = &'a Frame>,
) -> Vec<Effect> {
frames
.into_iter()
.flat_map(|(received_at, connection, text)| {
feed.on_frame_from(*connection, *received_at, text)
})
.collect()
}
fn resyncs(effects: &[Effect]) -> Vec<(usize, &str)> {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Send(connection, Command::Resubscribe(token)) => {
Some((*connection, token.as_str()))
}
_ => None,
})
.collect()
}
fn snapshot_symbols(effects: &[Effect]) -> Vec<&str> {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Write(Output::Snapshot(snapshot)) => Some(snapshot.symbol.as_str()),
_ => None,
})
.collect()
}
/// A Delta's content: exchange time, side, price, quantity.
type Content = (DateTime<Utc>, Side, i64, i64);
/// Per symbol, its Deltas' content, in order, from `since` (exchange
/// time) on.
fn deltas(effects: &[Effect], since: DateTime<Utc>) -> BTreeMap<String, Vec<Content>> {
let mut deltas: BTreeMap<_, Vec<_>> = BTreeMap::new();
for effect in effects {
if let Effect::Write(Output::Delta(delta)) = effect
&& delta.exchange_timestamp >= since
{
deltas.entry(delta.symbol.clone()).or_default().push((
delta.exchange_timestamp,
delta.side,
delta.price_level,
delta.quantity,
));
}
}
deltas
}
fn delta_stamps(effects: &[Effect]) -> Vec<(String, DateTime<Utc>)> {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Write(Output::Delta(delta)) => {
Some((delta.symbol.clone(), delta.backtest_timestamp))
}
_ => None,
})
.collect()
}
/// Whether the changes of frame `index` are lost for good if dropped: no
/// later frame touches their levels, and their symbols change again
/// later, so a missing change is seen.
fn telling_loss<T>(frames: &[T], index: usize, text: impl Fn(&T) -> &str) -> bool {
let levels = |text: &str| -> Vec<(String, String, String)> {
wire::decode(text)
.unwrap_or_default()
.into_iter()
.flat_map(|event| match event {
Event::PriceChange(change) => change.price_changes,
_ => Vec::new(),
})
.map(|change| (change.asset_id, change.side, change.price))
.collect()
};
let lost = levels(text(&frames[index]));
let later: Vec<_> = frames[index + 1..]
.iter()
.flat_map(|frame| levels(text(frame)))
.collect();
!lost.is_empty()
&& lost.iter().all(|level| !later.contains(level))
&& lost.iter().all(|(token, _, _)| {
later
.iter()
.filter(|(theirs, _, _)| theirs == token)
.count()
>= 3
})
}
/// The tokens a frame's price changes touch.
fn changed_tokens(text: &str) -> Vec<String> {
wire::decode(text)
.unwrap_or_default()
.into_iter()
.flat_map(|event| match event {
Event::PriceChange(change) => change.price_changes,
_ => Vec::new(),
})
.map(|change| change.asset_id)
.collect()
}
fn hex(hash: Hash) -> String {
hash.iter().map(|byte| format!("{byte:02x}")).collect()
}
/// A frame of `changes` to `token`'s book `base`, all at `timestamp` (ms),
/// carrying the hash of the book after all of them (a batch, if several),
/// and that book.
fn changes_frame(
token: &str,
symbol: &Symbol,
base: &Book,
changes: &[(Side, i64, i64)],
timestamp: i64,
) -> (String, Book) {
let mut book = base.clone();
for &(side, price, quantity) in changes {
book.set(side, price, quantity);
}
let timestamp = timestamp.to_string();
let hash = hex(hash(token, symbol, &book, &timestamp));
let price_changes: Vec<_> = changes
.iter()
.map(|&(side, price, quantity)| {
serde_json::json!({"asset_id": token, "price": format_fixed(price),
"size": format_fixed(quantity),
"side": if side == Side::Buy { "BUY" } else { "SELL" }, "hash": hash})
})
.collect();
let frame = serde_json::json!({"event_type": "price_change", "market": symbol.market,
"timestamp": timestamp, "price_changes": price_changes});
(frame.to_string(), book)
}
/// A complete book frame for `book`, as the exchange would send it in
/// answer to a resync, 5 ms after the symbol's last change.
fn fresh_book(token: &str, symbol: &Symbol, book: &Book) -> String {
let at = symbol.exchange_timestamp + TimeDelta::milliseconds(5);
book_frame(token, symbol, book, at)
}
/// A complete book frame for `book`, last changed at `at` (exchange time).
fn book_frame(token: &str, symbol: &Symbol, book: &Book, at: DateTime<Utc>) -> String {
let timestamp = at.timestamp_millis().to_string();
let levels = |levels: &BTreeMap<i64, i64>| {
levels
.iter()
.map(|(&price, &size)| serde_json::json!({"price": format_fixed(price), "size": format_fixed(size)}))
.collect::<Vec<_>>()
};
serde_json::json!({
"event_type": "book",
"market": symbol.market,
"asset_id": token,
"timestamp": timestamp,
"hash": hex(hash(token, symbol, book, &timestamp)),
"bids": levels(&book.bids),
"asks": levels(&book.asks),
"tick_size": symbol.tick_size,
"last_trade_price": symbol.last_trade_price,
})
.to_string()
}
/// The first verified symbol with a bid side, and a free bid price below
/// 1 for synthetic changes; the next free one is `free + 1_000` or above.
fn verified_symbol(feed: &Feed) -> (String, i64) {
let (token, symbol) = feed
.symbols
.iter()
.find(|(_, symbol)| symbol.state == State::Verified && symbol.book.bids.len() > 1)
.expect("a verified symbol");
let free = (1_000..)
.step_by(1_000)
.find(|price| {
!symbol.book.bids.contains_key(price)
&& !symbol.book.bids.contains_key(&(price + 1_000))
})
.unwrap();
(token.clone(), free)
}
// --- one connection: sprint 02's rules still hold ---
/// Split batches and trades make checks wait or fail with nothing lost:
/// the recording must replay without a single loss.
#[test]
fn replay_has_no_loss() {
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &fixture_frames());
assert_eq!(feed.counts.losses, 0);
assert!(
feed.symbols
.values()
.all(|symbol| !matches!(symbol.state, State::Unverifiable(_)))
);
assert!(
feed.counts.verified > 1_000,
"only {} changes verified",
feed.counts.verified
);
// Alone, a connection never has a duplicate but the server's repeats.
assert!(feed.counts.duplicates < 10, "{:?}", feed.counts);
}
#[test]
fn a_lost_frame_breaks_the_chain_and_its_fresh_book_counts_a_loss() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut control = feed_of(&markets);
let mut lost = None;
for (index, (received_at, text)) in frames.iter().enumerate() {
let tokens = changed_tokens(text);
if lost.is_none()
&& index > frames.len() / 4
&& !tokens.is_empty()
&& telling_loss(&frames, index, |(_, text)| text)
&& tokens.iter().all(|token| {
control.symbols[token].state == State::Verified
&& control.symbols[token].held.is_empty()
})
{
lost = Some((index, tokens[0].clone()));
}
control.on_frame(*received_at, text);
}
let (lost, token) = lost.unwrap();
let mut feed = feed_of(&markets);
replay(&mut feed, &frames[..lost]);
let chains = feed.counts.chains_broken;
let effects = replay(&mut feed, &frames[lost + 1..]);
assert!(
resyncs(&effects).contains(&(0, token.as_str())),
"the loss is never caught"
);
assert!(feed.counts.chains_broken > chains);
assert_eq!(feed.symbols[&token].state, State::Following(0));
// The fresh book: the exchange's, which our control holds.
let book = fresh_book(&token, &feed.symbols[&token], &control.symbols[&token].book);
let losses = feed.counts.losses;
let effects = feed.on_frame(frames.last().unwrap().0, &book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!(feed.counts.losses, losses + 1);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
/// A feed half replayed, a verified symbol of it that holds nothing, and a
/// complete book of that symbol as it was before its last change: a book
/// behind our stream, as a resync's fresh book can be. Its fresh book is
/// asked from connection 0.
fn feed_past_a_stale_book() -> (Feed, String, DateTime<Utc>, String) {
let frames = fixture_frames();
let mut feed = feed_of(&fixture_markets());
let half = frames.len() / 2;
replay(&mut feed, &frames[..half]);
let (token, symbol) = feed
.symbols
.iter_mut()
.find(|(_, symbol)| {
symbol.state == State::Verified
&& symbol.held.is_empty()
&& symbol.recent_changes.len() >= 2
})
.expect("a verified symbol with changes");
let token = token.clone();
let last = symbol.recent_changes.back().unwrap();
let mut before = symbol.book.clone();
before.set(last.side, last.price, last.previous);
let before_hash = symbol.recent_changes[symbol.recent_changes.len() - 2].hash;
let mut stale: serde_json::Value =
serde_json::from_str(&fresh_book(&token, symbol, &before)).unwrap();
stale["hash"] = serde_json::json!(hex(before_hash));
symbol.asked = Some((0, false));
(feed, token, frames[half].0, stale.to_string())
}
#[test]
fn a_book_behind_our_stream_is_not_a_loss() {
let (mut feed, token, now, stale_book) = feed_past_a_stale_book();
let ours = feed.symbols[&token].book.clone();
let effects = feed.on_frame(now, &stale_book);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.books_behind, 1);
assert_eq!(feed.counts.losses, 0);
assert_eq!(feed.symbols[&token].book, ours);
assert_eq!(feed.symbols[&token].asked, None);
}
/// Being behind says nothing of our own book: if it was the one lost, the
/// book brought forward differs from it.
#[test]
fn a_book_behind_our_stream_still_catches_our_loss() {
let (mut feed, token, now, stale_book) = feed_past_a_stale_book();
let right = feed.symbols[&token].book.clone();
feed.symbols
.get_mut(&token)
.unwrap()
.book
.set(Side::Buy, 1, 1);
let effects = feed.on_frame(now, &stale_book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!(feed.counts.losses, 1);
assert_eq!(feed.symbols[&token].book, right);
}
/// Seen live: a resync's fresh book behind by its levels but carrying a
/// newer timestamp, so its hash is none of our changes'. Ours stands, and
/// nothing stale is written.
#[test]
fn a_book_behind_by_levels_with_a_newer_timestamp_is_not_a_loss() {
let (mut feed, token, now, stale_book) = feed_past_a_stale_book();
let message: BookMessage = serde_json::from_str(&stale_book).unwrap();
let levels = parse_book(&message.bids, &message.asks).unwrap();
let symbol = &feed.symbols[&token];
let book = fresh_book(&token, symbol, &levels);
let ours = symbol.book.clone();
let effects = feed.on_frame(now, &book);
assert!(snapshot_symbols(&effects).is_empty());
let counts = &feed.counts;
assert_eq!(
(counts.books_behind, counts.subscription_gaps, counts.losses),
(1, 0, 0)
);
assert_eq!(feed.symbols[&token].book, ours);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
/// The level test must not mistake our wrong book for a newer one: we
/// changed a level, then lost the exchange's next change to it. The fresh
/// book plus our earlier change would equal ours; ours rewound never does.
#[test]
fn a_lost_change_at_a_level_we_had_changed_is_not_taken_for_behind() {
let (mut feed, token, now, _) = feed_past_a_stale_book();
let symbol = &feed.symbols[&token];
let last = symbol.recent_changes.back().unwrap();
let mut exchange = symbol.book.clone();
let lost_quantity = last.quantity + 1_000_000;
assert_ne!(last.previous, lost_quantity);
exchange.set(last.side, last.price, lost_quantity);
let book = fresh_book(&token, symbol, &exchange);
let effects = feed.on_frame(now, &book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!(feed.counts.books_behind, 0);
assert_eq!(feed.symbols[&token].book, exchange);
}
/// Seen live: our subscribe hides changes made just before the opening
/// book reaches us, so the first fresh book differs at a level none of our
/// changes touched, before any change was verified.
#[test]
fn a_differing_book_before_any_verified_change_is_a_subscription_gap() {
let frames = fixture_frames();
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &frames[..1]);
let (token, symbol) = feed
.symbols
.iter_mut()
.find(|(_, symbol)| symbol.state == State::Verified)
.unwrap();
let token = token.clone();
assert!(!symbol.verified_since_book);
symbol.asked = Some((0, false));
let mut exchange = symbol.book.clone();
exchange.set(Side::Buy, 1_000, 123_000_000);
let book = fresh_book(&token, symbol, &exchange);
let effects = feed.on_frame(frames[1].0, &book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!((feed.counts.subscription_gaps, feed.counts.losses), (1, 0));
assert_eq!(feed.symbols[&token].book, exchange);
}
/// Once a change has been verified since the opening book, our book was
/// right then: a fresh book that differs is a loss.
#[test]
fn a_differing_book_after_a_verified_change_is_a_loss() {
let frames = fixture_frames();
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &frames[..frames.len() / 2]);
let (token, symbol) = feed
.symbols
.iter_mut()
.find(|(_, symbol)| symbol.state == State::Verified && symbol.verified_since_book)
.unwrap();
let token = token.clone();
symbol.asked = Some((0, false));
let mut exchange = symbol.book.clone();
exchange.set(Side::Buy, 1_000, 123_000_000);
let book = fresh_book(&token, symbol, &exchange);
let effects = feed.on_frame(frames[frames.len() / 2].0, &book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!((feed.counts.subscription_gaps, feed.counts.losses), (0, 1));
assert_eq!(feed.symbols[&token].book, exchange);
}
#[test]
fn a_book_with_an_empty_side_parses() {
let ask = LevelMessage {
price: "0.999".to_string(),
size: "5".to_string(),
};
let book = parse_book(&[], &[ask]).unwrap();
assert!(book.bids.is_empty());
assert_eq!(book.asks, BTreeMap::from([(999_000, 5_000_000)]));
}
#[test]
fn a_book_failing_its_own_hash_goes_unverifiable_without_resync() {
let mut markets = fixture_markets();
for market in &mut markets {
market.min_order_size = "6".to_string();
}
let mut feed = feed_of(&markets);
let frames = fixture_frames();
let opening = feed.on_frame(frames[0].0, &frames[0].1);
assert_eq!(snapshot_symbols(&opening).len(), 40);
let effects = replay(&mut feed, &frames[1..]);
assert_eq!(resyncs(&effects), []);
assert!(
feed.symbols
.values()
.all(|symbol| symbol.state == State::Unverifiable(0))
);
assert!(
effects
.iter()
.any(|effect| matches!(effect, Effect::Write(Output::Delta(_))))
);
assert_eq!(feed.counts.verified, 0);
}
#[test]
fn frames_that_are_not_events_are_skipped() {
let mut feed = feed_of(&fixture_markets());
let at = fixture_frames()[0].0;
for text in [
"PONG",
"NO NEW ASSETS\n",
"not json",
"{\"event_type\":\"best_bid_ask\"}",
"[]",
] {
assert!(feed.on_frame(at, text).is_empty(), "{text:?}");
}
}
#[test]
fn a_hash_parses_from_its_hex_and_nothing_else() {
let text = "b75cdbc9bc53ef345ed7bc6fe7872140992cb182";
assert_eq!(parse_hash(text).map(hex).as_deref(), Some(text));
assert_eq!(parse_hash("b75c"), None);
assert_eq!(parse_hash(&"zz".repeat(20)), None);
}
// --- two connections: first one wins ---
/// The exchange time from which both connections had delivered their
/// opening books, with a margin: before it, two replays start from books
/// taken at different moments.
fn settled(frames: &[Frame]) -> DateTime<Utc> {
frames[0].0 + TimeDelta::seconds(5)
}
#[test]
fn two_connections_write_what_one_alone_writes_once() {
let frames = two_connections();
let markets = fixture_markets();
let start = frames[0].0;
let one_only: Vec<Frame> = frames
.iter()
.filter(|(_, connection, _)| *connection == 0)
.cloned()
.collect();
let mut alone = feed_on(&markets, &[0], start);
let alone_effects = replay_both(&mut alone, &one_only);
let mut both = feed_on(&markets, &[0, 1], start);
let both_effects = replay_both(&mut both, &frames);
// One opening Snapshot per symbol, not one per connection.
let opening =
|effects: &[Effect]| snapshot_symbols(&effects[..effects.len().min(200)]).len();
assert_eq!(opening(&both_effects), 40);
assert_eq!(opening(&alone_effects), 40);
let since = settled(&frames);
let (alone_deltas, both_deltas) =
(deltas(&alone_effects, since), deltas(&both_effects, since));
assert!(alone_deltas.values().map(Vec::len).sum::<usize>() > 1_000);
assert_eq!(alone_deltas, both_deltas);
// Every change came twice: the second copy is a duplicate.
assert!(
both.counts.duplicates > both.counts.verified / 2,
"{:?}",
both.counts
);
assert_eq!(both.counts.losses, 0);
assert!(!both.lags.is_empty() || both.counts.duplicates == 0);
}
/// The first copy's receive time is the one written.
#[test]
fn each_delta_takes_the_first_receive_time() {
let frames = two_connections();
let markets = fixture_markets();
let start = frames[0].0;
let mut both = feed_on(&markets, &[0, 1], start);
let both_effects = replay_both(&mut both, &frames);
for connection in [0, 1] {
let one: Vec<Frame> = frames
.iter()
.filter(|(_, on, _)| *on == connection)
.cloned()
.collect();
let mut alone = feed_on(&markets, &[connection], start);
let alone_effects = replay_both(&mut alone, &one);
// Delta by delta, in order per symbol: never later than one alone.
let mut alone_stamps: BTreeMap<String, VecDeque<DateTime<Utc>>> = BTreeMap::new();
for (symbol, at) in delta_stamps(&alone_effects) {
alone_stamps.entry(symbol).or_default().push_back(at);
}
let mut earlier = 0;
for (symbol, at) in delta_stamps(&both_effects) {
if let Some(theirs) = alone_stamps.get_mut(&symbol).and_then(VecDeque::pop_front) {
earlier += usize::from(at < theirs);
assert!(
at <= theirs + TimeDelta::milliseconds(60),
"{symbol}: {at} after {theirs}"
);
}
}
assert!(earlier > 0, "connection {connection} was never slower");
}
}
/// One change frame of a single verified symbol, past the opening, as both
/// connections delivered it: its index on each.
fn a_change_on_both(
frames: &[Frame],
feed_at: impl Fn(usize) -> Feed,
) -> (usize, usize, String) {
let quarter = frames.len() / 4;
for (index, (_, connection, text)) in frames.iter().enumerate().skip(quarter) {
let tokens = changed_tokens(text);
if *connection != 0
|| tokens.is_empty()
|| !telling_loss(frames, index, |(_, _, text)| text)
{
continue;
}
let feed = feed_at(index);
let settled = |token: &String| {
feed.symbols[token].state == State::Verified && feed.symbols[token].held.is_empty()
};
if !tokens.iter().all(settled) {
continue;
}
if let Some(other) = frames
.iter()
.enumerate()
.skip(quarter)
.position(|(_, (_, on, theirs))| *on == 1 && theirs == text)
{
return (index, other + quarter, tokens[0].clone());
}
}
panic!("no change on both connections");
}
#[test]
fn a_change_lost_on_one_connection_is_rescued() {
let frames = two_connections();
let markets = fixture_markets();
let start = frames[0].0;
let feed_at = |index: usize| {
let mut feed = feed_on(&markets, &[0, 1], start);
replay_both(&mut feed, &frames[..index]);
feed
};
let (on_0, _, token) = a_change_on_both(&frames, feed_at);
let mut full = feed_on(&markets, &[0, 1], start);
let full_effects = replay_both(&mut full, &frames);
let mut lossy = feed_on(&markets, &[0, 1], start);
let mut lossy_effects = replay_both(&mut lossy, &frames[..on_0]);
lossy_effects.extend(replay_both(&mut lossy, &frames[on_0 + 1..]));
let since = settled(&frames);
assert_eq!(
deltas(&full_effects, since),
deltas(&lossy_effects, since),
"{token}"
);
// Every copy has come a minute after the recording: count them.
let after = frames.last().unwrap().0 + TimeDelta::minutes(1);
full.tick(after);
lossy.tick(after);
assert_eq!(lossy.counts.losses, 0);
assert_eq!(lossy.counts.chains_broken, full.counts.chains_broken);
assert!(
lossy.counts.rescued > full.counts.rescued,
"{:?}",
lossy.counts
);
}
#[test]
fn a_change_lost_on_both_breaks_the_chain_and_resyncs_on_the_other() {
let frames = two_connections();
let markets = fixture_markets();
let start = frames[0].0;
let feed_at = |index: usize| {
let mut feed = feed_on(&markets, &[0, 1], start);
replay_both(&mut feed, &frames[..index]);
feed
};
let (on_0, on_1, token) = a_change_on_both(&frames, feed_at);
let mut control = feed_on(&markets, &[0, 1], start);
replay_both(&mut control, &frames);
let mut feed = feed_on(&markets, &[0, 1], start);
let kept: Vec<Frame> = frames
.iter()
.enumerate()
.filter(|(index, _)| *index != on_0 && *index != on_1)
.map(|(_, frame)| frame.clone())
.collect();
let chains = control.counts.chains_broken;
let effects = replay_both(&mut feed, &kept);
assert!(feed.counts.chains_broken > chains, "{:?}", feed.counts);
let followed = match feed.symbols[&token].state {
State::Following(followed) => followed,
state => panic!("{token} is {state:?}"),
};
// The fresh book is asked from the connection not followed.
assert!(
resyncs(&effects).contains(&(1 - followed, token.as_str())),
"{:?}",
resyncs(&effects)
);
assert_eq!(feed.symbols[&token].asked, Some((1 - followed, false)));
let book = fresh_book(&token, &feed.symbols[&token], &control.symbols[&token].book);
let effects = feed.on_frame_from(1 - followed, frames.last().unwrap().0, &book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!(feed.counts.losses, 1);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
#[test]
fn a_slow_connection_changes_nothing_but_the_duplicates() {
let frames = two_connections();
let markets = fixture_markets();
let start = frames[0].0;
let mut delayed: Vec<Frame> = frames
.iter()
.map(|(at, connection, text)| {
let at = if *connection == 1 {
*at + TimeDelta::seconds(2)
} else {
*at
};
(at, *connection, text.clone())
})
.collect();
delayed.sort_by_key(|(at, _, _)| *at);
let mut slow = feed_on(&markets, &[0, 1], start);
let slow_effects = replay_both(&mut slow, &delayed);
// Connection 0 alone sets the pace: compare with its own replay.
let one_only: Vec<Frame> = frames
.iter()
.filter(|(_, connection, _)| *connection == 0)
.cloned()
.collect();
let mut alone = feed_on(&markets, &[0], start);
let alone_effects = replay_both(&mut alone, &one_only);
let since = settled(&frames);
assert_eq!(deltas(&slow_effects, since), deltas(&alone_effects, since));
assert_eq!(
slow.counts.chains_broken, alone.counts.chains_broken,
"{:?}",
slow.counts
);
assert_eq!(slow.counts.losses, 0);
assert!(slow.counts.duplicates > 1_000);
// Its receive times are connection 0's, symbol by symbol (a held
// change waits longer while connection 1 might still bring the one
// before it, so symbols interleave differently).
let by_symbol = |effects: &[Effect]| {
let mut stamps: BTreeMap<String, Vec<DateTime<Utc>>> = BTreeMap::new();
for (symbol, at) in delta_stamps(effects) {
stamps.entry(symbol).or_default().push(at);
}
stamps
};
assert_eq!(by_symbol(&slow_effects), by_symbol(&alone_effects));
}
/// A verified symbol on connections 0 and 1, its book, a free bid price,
/// and an exchange time past its last change.
fn two_connection_symbol() -> (Feed, String, Book, i64, i64, DateTime<Utc>) {
let frames = two_connections();
let mut feed = feed_on(&fixture_markets(), &[0, 1], frames[0].0);
replay_both(&mut feed, &frames);
let (token, free) = verified_symbol(&feed);
let symbol = &feed.symbols[&token];
let at = frames.last().unwrap().0 + TimeDelta::seconds(1);
let millis = symbol.exchange_timestamp.timestamp_millis() + 1_000;
let book = symbol.book.clone();
(feed, token, book, free, millis, at)
}
#[test]
fn a_change_held_for_the_one_before_it_is_stamped_after_it() {
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
let symbol = &feed.symbols[&token];
let (x, after_x) = changes_frame(
&token,
symbol,
&book,
&[(Side::Buy, free, 1_000_000)],
millis,
);
let (y, after_y) = changes_frame(
&token,
symbol,
&after_x,
&[(Side::Buy, free + 1_000, 2_000_000)],
millis + 1,
);
// Connection 0 missed X: its Y waits.
assert!(feed.on_frame_from(0, at, &y).is_empty());
assert!(feed.waiting.contains(&token));
// Connection 1 brings X, 10 ms later: X, then Y.
let later = at + TimeDelta::milliseconds(10);
let effects = feed.on_frame_from(1, later, &x);
assert_eq!(
delta_stamps(&effects),
[(token.clone(), later), (token.clone(), later)]
);
assert_eq!(feed.symbols[&token].book, after_y);
// Connection 1's own Y comes after: a duplicate.
let duplicates = feed.counts.duplicates;
assert!(
feed.on_frame_from(1, later + TimeDelta::milliseconds(1), &y)
.is_empty()
);
assert_eq!(feed.counts.duplicates, duplicates + 1);
// Once every copy could have come: X came once, rescued; Y twice.
let rescued = feed.counts.rescued;
feed.tick(at + TimeDelta::minutes(2));
assert_eq!(feed.counts.rescued, rescued + 1);
}
/// A burst on a connection that missed the change before it: the held
/// changes stop at the cap, the chain breaks, and it is all quick.
#[test]
fn a_burst_behind_a_missing_change_is_capped() {
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
let symbol = &feed.symbols[&token];
let (_, mut after) = changes_frame(
&token,
symbol,
&book,
&[(Side::Buy, free, 1_000_000)],
millis,
);
let mut burst = Vec::new();
for step in 1..=5_000 {
let (frame, next) = changes_frame(
&token,
symbol,
&after,
&[(Side::Buy, free + 1_000, step * 1_000)],
millis + step,
);
burst.push(frame);
after = next;
}
let started = std::time::Instant::now();
let mut most = 0;
for (step, frame) in burst.iter().enumerate() {
feed.on_frame_from(0, at + TimeDelta::microseconds(step as i64), frame);
let held = feed.symbols[&token].held.get(&0).map_or(0, Vec::len);
most = most.max(held);
}
assert!(most <= MAX_HELD + 1, "{most} held");
assert_eq!(feed.symbols[&token].state, State::Following(0));
assert!(
started.elapsed() < std::time::Duration::from_secs(10),
"{:?}",
started.elapsed()
);
}
#[test]
fn a_batch_is_written_once_it_fits() {
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
let symbol = &feed.symbols[&token];
let changes = [
(Side::Buy, free, 1_000_000),
(Side::Buy, free + 1_000, 1_000_000),
];
let (whole, after) = changes_frame(&token, symbol, &book, &changes, millis);
// The exchange sends a batch one change per frame, each with the
// batch's hash.
let value: serde_json::Value = serde_json::from_str(&whole).unwrap();
let part = |index: usize| {
let mut value = value.clone();
value["price_changes"] = serde_json::json!([value["price_changes"][index]]);
value.to_string()
};
assert!(feed.on_frame_from(0, at, &part(0)).is_empty());
let effects = feed.on_frame_from(0, at + TimeDelta::milliseconds(1), &part(1));
assert_eq!(delta_stamps(&effects).len(), 2);
assert_eq!(feed.symbols[&token].book, after);
assert!(!feed.waiting.contains(&token));
}
#[test]
fn a_change_older_than_the_book_is_dropped() {
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
let symbol = &feed.symbols[&token];
let (old, _) = changes_frame(
&token,
symbol,
&book,
&[(Side::Buy, free, 1_000_000)],
millis - 60_000,
);
let duplicates = feed.counts.duplicates;
assert!(feed.on_frame_from(1, at, &old).is_empty());
assert_eq!(feed.counts.duplicates, duplicates + 1);
assert!(!feed.waiting.contains(&token));
}
#[test]
fn held_changes_past_their_ceiling_break_the_chain_when_the_book_stood_still() {
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
let symbol = &feed.symbols[&token];
let (_, after_x) = changes_frame(
&token,
symbol,
&book,
&[(Side::Buy, free, 1_000_000)],
millis,
);
let (y, after_y) = changes_frame(
&token,
symbol,
&after_x,
&[(Side::Buy, free + 1_000, 2_000_000)],
millis + 1,
);
// Both missed X and hold the same Y: nobody can bring X, 50 ms.
feed.on_frame_from(0, at, &y);
feed.on_frame_from(1, at, &y);
assert!(feed.idle(at + TimeDelta::milliseconds(40)).is_empty());
let effects = feed.idle(at + TimeDelta::milliseconds(60));
assert_eq!(feed.symbols[&token].state, State::Following(0));
assert_eq!(resyncs(&effects), [(1, token.as_str())]);
// Y is written unchecked, on the book that lacks X.
assert_eq!(delta_stamps(&effects), [(token.clone(), at)]);
assert_ne!(feed.symbols[&token].book, after_y);
// Connection 1's changes are dropped while 0 is followed.
let (z, _) = changes_frame(
&token,
&feed.symbols[&token],
&after_y,
&[(Side::Buy, free, 3_000_000)],
millis + 2,
);
assert!(
feed.on_frame_from(1, at + TimeDelta::milliseconds(70), &z)
.is_empty()
);
assert_eq!(
feed.on_frame_from(0, at + TimeDelta::milliseconds(71), &z)
.len(),
1
);
}
#[test]
fn a_connection_behind_has_its_held_changes_dropped_at_the_ceiling() {
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
let symbol = &feed.symbols[&token];
let (x, after_x) = changes_frame(
&token,
symbol,
&book,
&[(Side::Buy, free, 1_000_000)],
millis,
);
// Connection 0 holds a change that will never fit (made on another
// book), while connection 1 moves the book on.
let mut elsewhere = after_x.clone();
elsewhere.set(Side::Buy, free + 1_000, 1);
let (stray, _) = changes_frame(
&token,
symbol,
&elsewhere,
&[(Side::Sell, 999_000, 5_000_000)],
millis,
);
feed.on_frame_from(0, at, &stray);
let effects = feed.on_frame_from(1, at + TimeDelta::milliseconds(5), &x);
assert_eq!(delta_stamps(&effects).len(), 1);
// Connection 1 could still deliver: a second, then dropped.
assert!(feed.idle(at + TimeDelta::milliseconds(900)).is_empty());
let effects = feed.idle(at + TimeDelta::milliseconds(1_100));
assert!(effects.is_empty());
assert_eq!(feed.symbols[&token].state, State::Verified);
assert_eq!(feed.counts.held_dropped, 1);
assert!(!feed.waiting.contains(&token));
}
#[test]
fn a_book_not_asked_for_changes_nothing() {
let (mut feed, token, book, _, _, at) = two_connection_symbol();
let symbol = &feed.symbols[&token];
let mut other = book.clone();
other.set(Side::Buy, 1_000, 5_000_000);
let unasked = fresh_book(&token, symbol, &other);
assert!(feed.on_frame_from(1, at, &unasked).is_empty());
assert_eq!(feed.symbols[&token].book, book);
}
#[test]
fn one_connection_down_changes_nothing_and_both_down_await_a_book() {
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
assert!(feed.on_received(Received::Down(1)).is_empty());
assert_eq!(feed.symbols[&token].state, State::Verified);
let symbol = &feed.symbols[&token];
let (x, after_x) = changes_frame(
&token,
symbol,
&book,
&[(Side::Buy, free, 1_000_000)],
millis,
);
// Alone, a change that doesn't fit waits 50 ms, not a second.
assert_eq!(feed.on_frame_from(0, at, &x).len(), 1);
assert_eq!(feed.symbols[&token].book, after_x);
feed.on_received(Received::Down(0));
assert_eq!(feed.symbols[&token].state, State::AwaitingBook);
// The reconnect's opening book is compared with ours: equal, nothing.
feed.on_received(Received::Up(0));
let same = fresh_book(&token, &feed.symbols[&token], &after_x);
assert!(snapshot_symbols(&feed.on_frame_from(0, at, &same)).is_empty());
assert_eq!(feed.symbols[&token].state, State::Verified);
// Differing, a loss.
feed.on_received(Received::Down(0));
feed.on_received(Received::Down(1));
let mut other = after_x.clone();
other.set(Side::Buy, free, 7_000_000);
let differing = fresh_book(&token, &feed.symbols[&token], &other);
let losses = feed.counts.losses;
assert_eq!(
snapshot_symbols(&feed.on_frame_from(1, at, &differing)),
[token.as_str()]
);
assert_eq!(feed.counts.losses, losses + 1);
}
/// A frame setting `token`'s bid at `price` to `quantity`, `millis` its
/// exchange time, from the book as it is now.
fn change(feed: &Feed, token: &str, millis: i64, price: i64, quantity: i64) -> String {
let symbol = &feed.symbols[token];
let changes = [(Side::Buy, price, quantity)];
changes_frame(token, symbol, &symbol.book, &changes, millis).0
}
fn resyncs_of<'a>(effects: &'a [Effect], token: &str) -> Vec<(usize, &'a str)> {
resyncs(effects)
.into_iter()
.filter(|&(_, theirs)| theirs == token)
.collect()
}
/// From the next wall-clock minute after `at`, which the feed has begun:
/// the following minute is counted whole.
fn begin_minute(feed: &mut Feed, at: DateTime<Utc>) -> DateTime<Utc> {
let minute = next_minute(at);
feed.idle(minute);
minute
}
/// `count` changes of `token`, from 10 s after `start`, on the
/// connections `on` only.
fn changes(
feed: &mut Feed,
token: &str,
on: &[usize],
start: DateTime<Utc>,
price: i64,
count: i64,
) {
let millis = feed.symbols[token].exchange_timestamp.timestamp_millis() + 1_000;
for step in 0..count {
let at = start + TimeDelta::seconds(10 + step);
let frame = change(feed, token, millis + step, price, (step + 1) * 1_000_000);
for &connection in on {
feed.on_frame_from(connection, at, &frame);
}
}
}
fn quiet_reports(effects: &[Effect]) -> Vec<(&str, usize)> {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Report(Report::Quiet { market, connection }) => {
Some((market.as_str(), *connection))
}
_ => None,
})
.collect()
}
#[test]
fn a_subscription_quiet_on_one_connection_is_resubscribed_then_reported() {
let (mut feed, token, _, free, _, at) = two_connection_symbol();
let market = feed.symbols[&token].market.clone();
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
begin_minute(&mut feed, at);
let (losses, gaps) = (feed.counts.losses, feed.counts.subscription_gaps);
// Two changes against none: too few to say.
changes(&mut feed, &token, &[0], minute(0), free, 2);
let effects = feed.idle(minute(1));
assert!(resyncs_of(&effects, &token).is_empty());
assert_eq!(feed.counts.quiet, 0);
// Three: quiet on connection 1, resubscribed there.
changes(&mut feed, &token, &[0], minute(1), free, 3);
let effects = feed.idle(minute(2));
assert_eq!(resyncs_of(&effects, &token), [(1, token.as_str())]);
assert!(quiet_reports(&effects).is_empty());
assert_eq!(feed.counts.quiet, 1);
assert_eq!(feed.symbols[&token].state, State::Verified);
// The book the resubscribe brings back is not asked for: ignored.
let book = feed.symbols[&token].book.clone();
let back = fresh_book(&token, &feed.symbols[&token], &book);
let effects = feed.on_frame_from(1, minute(2) + TimeDelta::seconds(1), &back);
assert!(snapshot_symbols(&effects).is_empty());
// Still quiet the minute after: reported for a move, not resubscribed.
changes(&mut feed, &token, &[0], minute(2), free, 3);
let effects = feed.idle(minute(3));
assert!(resyncs_of(&effects, &token).is_empty());
assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]);
assert_eq!(
(feed.counts.losses, feed.counts.subscription_gaps),
(losses, gaps)
);
}
/// Both tokens of a market go quiet together: one report for the market.
#[test]
fn a_market_quiet_on_both_its_tokens_is_reported_once() {
let (mut feed, token, _, free, _, at) = two_connection_symbol();
let market = feed.symbols[&token].market.clone();
let tokens = feed.markets[&market].tokens.clone();
assert!(
tokens
.iter()
.all(|token| feed.symbols[token].state == State::Verified)
);
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
begin_minute(&mut feed, at);
for n in 0..2 {
for token in &tokens {
changes(&mut feed, token, &[0], minute(n), free, 3);
}
let effects = feed.idle(minute(n + 1));
if n == 0 {
assert_eq!(feed.counts.quiet, 2);
} else {
assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]);
}
}
}
#[test]
fn a_subscription_delivering_or_not_there_the_whole_minute_is_not_quiet() {
let (mut feed, token, _, free, _, at) = two_connection_symbol();
let markets = fixture_markets();
let market = markets
.iter()
.find(|market| market.tokens.contains(&token))
.unwrap();
// Connection 5 is up from before the minutes below.
feed.on_received(Received::Up(5));
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
begin_minute(&mut feed, at);
// Both deliver: neither is quiet; the symbols that didn't change
// aren't either.
changes(&mut feed, &token, &[0, 1], minute(0), free, 3);
let effects = feed.idle(minute(1));
assert!(resyncs(&effects).iter().all(|&(_, theirs)| theirs != token));
// Connection 1 dropped and came back within the minute: not judged
// for the changes it missed.
changes(&mut feed, &token, &[0], minute(1), free, 3);
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let effects = feed.idle(minute(2));
assert!(resyncs_of(&effects, &token).is_empty());
// Connection 5, subscribed in the middle of the minute and delivering
// nothing yet: not judged either.
feed.move_market(market, 1, 5, minute(2) + TimeDelta::seconds(5));
changes(&mut feed, &token, &[0, 1], minute(2), free, 3);
let effects = feed.idle(minute(3));
assert!(resyncs_of(&effects, &token).is_empty());
assert_eq!(feed.counts.quiet, 0);
}
/// A verified symbol on connections 0 and 1, changed once and then still
/// for 30 s: the feed, its token, its book before the change and now, a
/// free bid price, and the time 30 s after.
fn still_symbol() -> (Feed, String, Book, Book, i64, DateTime<Utc>) {
let (mut feed, token, before, free, millis, at) = two_connection_symbol();
let x = change(&feed, &token, millis, free, 1_000_000);
feed.on_frame_from(0, at, &x);
feed.on_frame_from(1, at, &x);
let ours = feed.symbols[&token].book.clone();
let later = at + TimeDelta::seconds(30);
// The minute's Snapshots, out of the way.
feed.idle(later);
(feed, token, before, ours, free, later)
}
/// Connection 1 drops and reconnects; its opening book of `token` holds
/// `book`.
fn reconnect_with(feed: &mut Feed, token: &str, book: &Book, at: DateTime<Utc>) -> Vec<Effect> {
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let opening = fresh_book(token, &feed.symbols[token], book);
feed.on_frame_from(1, at, &opening)
}
#[test]
fn a_reconnect_book_of_a_still_symbol_equal_to_ours_or_behind_changes_nothing() {
let (mut feed, token, before, ours, _, later) = still_symbol();
let losses = feed.counts.losses;
let effects = reconnect_with(&mut feed, &token, &ours, later);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 1);
// The server's opening books lag: a state we passed is no loss.
let behind = feed.counts.books_behind;
let effects = reconnect_with(&mut feed, &token, &before, later + TimeDelta::seconds(1));
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 2);
assert_eq!(feed.counts.books_behind, behind + 1);
assert_eq!(feed.symbols[&token].book, ours);
// It lags further back than the changes we remember: no newer than
// our last change, it still holds nothing we missed.
let symbol = feed.symbols.get_mut(&token).unwrap();
symbol.recent_changes.clear();
let old = symbol.exchange_timestamp - TimeDelta::minutes(10);
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let lagging = book_frame(&token, &feed.symbols[&token], &before, old);
let effects = feed.on_frame_from(1, later + TimeDelta::seconds(2), &lagging);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.books_behind, behind + 2);
assert_eq!(feed.symbols[&token].book, ours);
assert_eq!(feed.counts.losses, losses);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
/// A reconnect's book that fails its own hash proves nothing: the symbol
/// stays verified, with its book and its hash inputs.
#[test]
fn a_reconnect_book_failing_its_own_hash_changes_nothing() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
let frame = fresh_book(&token, &feed.symbols[&token], &other);
let mut frame: serde_json::Value = serde_json::from_str(&frame).unwrap();
frame["hash"] = "0".repeat(40).into();
frame["last_trade_price"] = "0.123".into();
let last_trade_price = feed.symbols[&token].last_trade_price.clone();
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let effects = feed.on_frame_from(1, later, &frame.to_string());
assert!(effects.is_empty());
let symbol = &feed.symbols[&token];
assert_eq!(symbol.state, State::Verified);
assert_eq!(symbol.book, ours);
assert_eq!(symbol.last_trade_price, last_trade_price);
}
/// Only a reconnect's book of a symbol with nothing asked is judged: not
/// the opening book of a subscription made since the connection came up
/// (a move's), nor any while a fresh book is asked.
#[test]
fn a_book_that_is_no_reconnects_or_comes_while_one_is_asked_is_not_judged() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let markets = fixture_markets();
let market = markets
.iter()
.find(|market| market.tokens.contains(&token))
.unwrap();
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
feed.on_received(Received::Up(5));
feed.move_market(market, 0, 5, later);
let opening = fresh_book(&token, &feed.symbols[&token], &other);
let effects = feed.on_frame_from(5, later + TimeDelta::seconds(1), &opening);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
feed.symbols.get_mut(&token).unwrap().asked = Some((5, false));
let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(2));
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
assert_eq!(feed.symbols[&token].book, ours);
}
#[test]
fn a_reconnect_book_of_a_still_symbol_that_differs_is_a_loss_and_resubscribes_the_other() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let (losses, gaps) = (feed.counts.losses, feed.counts.subscription_gaps);
// Even before any change of it was verified: a loss, not the gap of
// a subscribe, which is long past.
feed.symbols.get_mut(&token).unwrap().verified_since_book = false;
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
let effects = reconnect_with(&mut feed, &token, &other, later);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!(resyncs_of(&effects, &token), [(0, token.as_str())]);
assert_eq!(feed.counts.losses, losses + 1);
assert_eq!(feed.counts.subscription_gaps, gaps);
assert_eq!(feed.counts.reconnect_losses, 1);
assert_eq!(feed.symbols[&token].book, other);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
#[test]
fn a_reconnect_book_of_a_symbol_changed_lately_is_ignored() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let millis = feed.symbols[&token].exchange_timestamp.timestamp_millis() + 1_000;
let x = change(&feed, &token, millis, free, 2_000_000);
feed.on_frame_from(0, later, &x);
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(1));
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
// And the next book of it from that connection is no reconnect's.
feed.idle(later + TimeDelta::seconds(30));
let opening = fresh_book(&token, &feed.symbols[&token], &other);
let effects = feed.on_frame_from(1, later + TimeDelta::seconds(31), &opening);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
}
#[test]
fn the_followed_connection_down_switches_with_a_snapshot() {
let (mut feed, token, book, _, _, at) = two_connection_symbol();
feed.symbols.get_mut(&token).unwrap().state = State::Following(0);
let effects = feed.on_received(Received::Down(0));
assert_eq!(feed.symbols[&token].state, State::Following(1));
assert!(resyncs(&effects).contains(&(1, token.as_str())));
// Its fresh book is written even if it equals ours.
let fresh = fresh_book(&token, &feed.symbols[&token], &book);
let effects = feed.on_frame_from(1, at, &fresh);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert!(feed.counts.switches >= 1);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
#[test]
fn a_move_subscribes_first_and_leaves_once_the_new_connection_delivers() {
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
let markets = fixture_markets();
let market = markets
.iter()
.find(|market| market.tokens.contains(&token))
.unwrap();
let effects = feed.move_market(market, 0, 5, at);
assert!(matches!(
effects.as_slice(),
[Effect::Send(5, Command::Subscribe(tokens))] if *tokens == market.tokens
));
// Both still deliver until connection 5's book comes.
let symbol = &feed.symbols[&token];
let (x, after_x) = changes_frame(
&token,
symbol,
&book,
&[(Side::Buy, free, 1_000_000)],
millis,
);
assert_eq!(feed.on_frame_from(0, at, &x).len(), 1);
let opening = fresh_book(&token, &feed.symbols[&token], &after_x);
let effects = feed.on_frame_from(5, at, &opening);
assert!(matches!(
effects.as_slice(),
[Effect::Send(0, Command::Unsubscribe(tokens))] if *tokens == market.tokens
));
assert!(snapshot_symbols(&effects).is_empty());
let mut on = feed.symbols[&token].connections.clone();
on.sort();
assert_eq!(on, [1, 5]);
}
/// The connection left can't answer a fresh book any more: it is asked
/// from another.
#[test]
fn a_move_ending_while_a_fresh_book_is_asked_from_the_left_one_asks_again() {
let (mut feed, token, book, _, _, at) = two_connection_symbol();
let markets = fixture_markets();
let market = markets
.iter()
.find(|market| market.tokens.contains(&token))
.unwrap();
let symbol = feed.symbols.get_mut(&token).unwrap();
symbol.state = State::Following(1);
symbol.asked = Some((0, false));
feed.move_market(market, 0, 5, at);
let opening = fresh_book(&token, &feed.symbols[&token], &book);
let effects = feed.on_frame_from(5, at, &opening);
assert!(
resyncs(&effects).contains(&(1, token.as_str())),
"{:?}",
resyncs(&effects)
);
assert_eq!(feed.symbols[&token].asked, Some((1, false)));
}
/// Replaying the output by time keeps its order: a change held across the
/// minute is not stamped before the minute's Snapshot.
#[test]
fn a_change_held_across_the_minute_is_stamped_after_its_snapshot() {
let (mut feed, token, book, free, millis, _) = two_connection_symbol();
let minute = next_minute(feed.last_received_at.unwrap());
let symbol = &feed.symbols[&token];
let changes = [
(Side::Buy, free, 1_000_000),
(Side::Buy, free + 1_000, 1_000_000),
];
let (whole, _) = changes_frame(&token, symbol, &book, &changes, millis);
let value: serde_json::Value = serde_json::from_str(&whole).unwrap();
let part = |index: usize| {
let mut value = value.clone();
value["price_changes"] = serde_json::json!([value["price_changes"][index]]);
value.to_string()
};
// A batch's first change waits for its second across the minute;
// another event comes after it, so the minute's Snapshot is stamped
// later than that first change.
feed.on_frame_from(0, minute - TimeDelta::milliseconds(2), &part(0));
let other = serde_json::json!({"event_type": "price_change", "market": "0xother",
"timestamp": "1", "price_changes": []});
feed.on_frame_from(0, minute - TimeDelta::milliseconds(1), &other.to_string());
let effects = feed.on_frame_from(0, minute + TimeDelta::milliseconds(1), &part(1));
let snapshot_at = effects
.iter()
.find_map(|effect| match effect {
Effect::Write(Output::Snapshot(snapshot)) if snapshot.symbol == token => {
Some(snapshot.received_at)
}
_ => None,
})
.expect("the minute's Snapshot");
let stamps: Vec<_> = delta_stamps(&effects)
.into_iter()
.filter(|(symbol, _)| *symbol == token)
.collect();
assert_eq!(stamps.len(), 2);
for (_, at) in stamps {
assert!(at >= snapshot_at, "{at} before {snapshot_at}");
}
}
#[test]
fn a_market_on_two_connections_reports_its_load_on_one() {
let minute = "2026-11-10T16:00:00Z".parse::<DateTime<Utc>>().unwrap();
let market = Market {
condition_id: "0xm".to_string(),
tokens: vec!["0xma".to_string(), "0xmb".to_string()],
min_order_size: "5".to_string(),
neg_risk: false,
};
let mut feed = Feed::new(0);
feed.add_market(&market, &[0, 1], minute);
let change = serde_json::json!({"event_type": "price_change", "market": "0xm",
"timestamp": "1", "price_changes": []})
.to_string();
feed.on_frame_from(0, minute + TimeDelta::seconds(10), &change);
feed.on_frame_from(1, minute + TimeDelta::seconds(10), &change);
let effects = feed.tick(minute + TimeDelta::seconds(60));
let Some(Effect::Report(Report::Loads(loads))) = effects.first() else {
panic!("no report");
};
assert_eq!(loads["0xm"], change.len() as u64 / 60);
}
// --- markets, loads, the output ---
/// A market's opening books, as a subscribe brings them.
fn opening_books(frames: &[(DateTime<Utc>, String)], market: &Market) -> String {
let opening: Vec<serde_json::Value> = serde_json::from_str(&frames[0].1).unwrap();
let books: Vec<_> = opening
.iter()
.filter(|book| {
market
.tokens
.iter()
.any(|token| book["asset_id"] == **token)
})
.collect();
serde_json::to_string(&books).unwrap()
}
#[test]
fn a_market_added_mid_run_is_subscribed_and_its_first_book_written() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut feed = feed_of(&markets[1..]);
replay(&mut feed, &frames[1..100]);
let effects = feed.add_market(&markets[0], &[0], frames[100].0);
assert!(matches!(
effects.as_slice(),
[Effect::Send(0, Command::Subscribe(tokens))] if *tokens == markets[0].tokens
));
let effects = feed.on_frame(frames[100].0, &opening_books(&frames, &markets[0]));
let mut written = snapshot_symbols(&effects);
written.sort();
let mut tokens: Vec<&str> = markets[0].tokens.iter().map(String::as_str).collect();
tokens.sort();
assert_eq!(written, tokens);
assert!(
markets[0]
.tokens
.iter()
.all(|token| feed.symbols[token].state == State::Verified)
);
}
#[test]
fn a_removed_market_is_unsubscribed_and_written_no_more() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut feed = feed_of(&markets);
replay(&mut feed, &frames[..1]);
let removed = &markets[0].tokens;
let effects = feed.remove_market(&markets[0]);
assert!(matches!(
effects.as_slice(),
[Effect::Send(0, Command::Unsubscribe(tokens))] if tokens == removed
));
// The rest of the recording crosses two minutes: no Delta, Snapshot or
// resync names a removed token.
let effects = replay(&mut feed, &frames[1..]);
let mentions = |effect: &Effect| match effect {
Effect::Write(Output::Snapshot(snapshot)) => removed.contains(&snapshot.symbol),
Effect::Write(Output::Delta(delta)) => removed.contains(&delta.symbol),
Effect::Send(_, Command::Resubscribe(token)) => removed.contains(token),
Effect::Send(..) => false,
Effect::Report(Report::Loads(loads)) => loads.contains_key(&markets[0].condition_id),
Effect::Report(Report::Quiet { market, .. }) => *market == markets[0].condition_id,
};
assert!(!effects.iter().any(mentions));
// The minute Snapshots of the 38 others, twice.
assert_eq!(snapshot_symbols(&effects).len(), 38 * 2);
}
#[test]
fn each_minute_reports_the_bytes_per_second_of_every_market() {
let minute = "2026-11-10T16:00:00Z".parse::<DateTime<Utc>>().unwrap();
let at = |seconds| minute + TimeDelta::seconds(seconds);
let market = |id: &str| Market {
condition_id: id.to_string(),
tokens: vec![format!("{id}a"), format!("{id}b")],
min_order_size: "5".to_string(),
neg_risk: false,
};
let (busy, quiet, late) = (market("0xbusy"), market("0xquiet"), market("0xlate"));
let mut feed = Feed::new(0);
feed.add_market(&busy, &[0], at(0));
// Added halfway through the minute: measured over its 30 s.
feed.add_market(&quiet, &[0], at(30));
// Added 15 s before it: kept for the next report.
feed.add_market(&late, &[0], at(45));
let change = |id: &str| {
serde_json::json!({"event_type": "price_change", "market": id, "timestamp": "1",
"price_changes": []})
.to_string()
};
let trade = serde_json::json!({"event_type": "last_trade_price", "market": "0xbusy",
"asset_id": "0xbusya", "price": "0.5", "size": "1", "side": "BUY",
"timestamp": "1"})
.to_string();
let tick_size = serde_json::json!({"event_type": "tick_size_change", "market": "0xquiet",
"asset_id": "0xquieta", "old_tick_size": "0.01", "new_tick_size": "0.001",
"timestamp": "1"})
.to_string();
// One frame, two markets: each takes half.
let shared = format!("[{},{}]", change("0xbusy"), change("0xquiet"));
let mut reports = Vec::new();
let mut on_frame = |feed: &mut Feed, seconds, text: &str| {
for effect in feed.on_frame(at(seconds), text) {
if let Effect::Report(Report::Loads(loads)) = effect {
reports.push(loads);
}
}
};
on_frame(&mut feed, 10, &change("0xbusy"));
on_frame(&mut feed, 20, &trade);
on_frame(&mut feed, 40, &tick_size);
on_frame(&mut feed, 50, &change("0xlate"));
on_frame(&mut feed, 55, &change("0xunknown"));
// The next minute: the first report, then nothing more until the one
// after, which starts again from zero.
on_frame(&mut feed, 60, &shared);
on_frame(&mut feed, 120, &change("0xunknown"));
let bytes = |text: &String| text.len() as u64;
let half = bytes(&shared) / 2;
assert_eq!(
reports,
[
Loads::from([
(
"0xbusy".to_string(),
(bytes(&change("0xbusy")) + bytes(&trade)) / 60
),
("0xquiet".to_string(), bytes(&tick_size) / 30),
]),
Loads::from([
("0xbusy".to_string(), half / 60),
("0xquiet".to_string(), half / 60),
("0xlate".to_string(), bytes(&change("0xlate")) / 75),
]),
]
);
}
#[test]
fn the_recording_reports_every_market_once_measured_long_enough() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut feed = feed_of(&markets);
let reports: Vec<Loads> = replay(&mut feed, &frames)
.into_iter()
.filter_map(|effect| match effect {
Effect::Report(Report::Loads(loads)) => Some(loads),
_ => None,
})
.collect();
// The recording crosses two minutes, the first too soon after its
// start for any market to be measured.
let start = frames[0].0;
assert!(next_minute(start) - start < MIN_MEASURED);
assert_eq!(reports.len(), 2);
assert!(reports[0].is_empty());
let loads = &reports[1];
assert_eq!(loads.len(), markets.len());
assert!(
markets
.iter()
.all(|market| loads.contains_key(&market.condition_id))
);
assert!(loads.values().sum::<u64>() > 0);
}
#[test]
fn the_output_reconciles() {
// The book and writer threads as main runs them, fed the two
// connections' recording.
let frames = two_connections();
let (events_tx, events_rx) = crossbeam::channel::unbounded();
let (outputs_tx, outputs_rx) = crossbeam::channel::unbounded();
let (connection_tx, _connection_rx) = crossbeam::channel::unbounded();
let (_commands_tx, commands_rx) = crossbeam::channel::unbounded();
let (reports_tx, _reports_rx) = crossbeam::channel::unbounded();
for (received_at, connection, text) in &frames {
for arrival in arrivals(*received_at, *connection, text) {
events_tx.send(Received::Event(arrival)).unwrap();
}
}
drop(events_tx);
let feed = feed_on(&fixture_markets(), &[0, 1], frames[0].0);
let book = std::thread::spawn(move || {
serve(
feed,
vec![connection_tx.clone(), connection_tx],
events_rx,
commands_rx,
outputs_tx,
reports_tx,
)
});
let path = std::env::temp_dir().join(format!("reconcile-{}.sbe", std::process::id()));
let written = crate::write(path.to_str().unwrap(), outputs_rx);
book.join().unwrap();
written.unwrap();
let report = engineering_challenge::reconcile::reconcile(&path, None);
std::fs::remove_file(&path).unwrap();
let report = report.unwrap();
// The opening Snapshots, then one per symbol at each of the two
// minutes crossed (20 of the 40 tokens had an order book then).
assert_eq!(report.snapshots, 20 * 3, "{report:?}");
assert!(report.deltas > 100, "{report:?}");
assert_eq!(report.orphan_deltas, 0);
assert_eq!(report.unexplained, []);
}
}