Archived
The orchestrator, the connection loop and the book thread's shell move to src/core/, generic over an Exchange and its loss rule (ADR 0004). Polymarket runs through it unchanged. Kalshi joins: a signed handshake (KALSHI_KEY_ID_FILE, KALSHI_PRIVATE_KEY_FILE), a reconnect on a skipped sequence number, first one wins by clock and rank, and its market listing with markets tracked from three hours before they open. Breaking: the exchange is now the first argument, `engineering-challenge <polymarket|kalshi|kalshi-demo> [OUTPUT]`.
2862 lines
112 KiB
Rust
2862 lines
112 KiB
Rust
//! Polymarket's rule for a book thread: folds the events of a symbol's
|
|
//! connections into its book, first one wins: a change is taken from
|
|
//! whichever connection delivers it first, Polymarket's book hash saying
|
|
//! where it belongs (ADR 0003). It resyncs when a change fits on no
|
|
//! connection in time. It also looks for what first one wins hides: symbols
|
|
//! quiet on both connections, in a reconnect's opening books. The rules are in
|
|
//! `roadmap/04. redundant connections.md` and
|
|
//! `roadmap/05. subscription monitoring.md`, the protocol facts and the hash
|
|
//! formula in `docs/notes.md`, the terms in `GLOSSARY.md`. The shell around it
|
|
//! (Snapshots, moves, the quiet watch) is `src/core/feed.rs`.
|
|
|
|
use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
|
|
|
use chrono::{DateTime, TimeDelta, Utc};
|
|
use engineering_challenge::Side;
|
|
use engineering_challenge::cause::{self, Cause};
|
|
use serde::Serialize;
|
|
use sha1::{Digest, Sha1};
|
|
|
|
use super::Market;
|
|
use super::wire::{BookMessage, Event, LevelMessage, PriceChangeMessage};
|
|
use crate::core::feed::{self, Effect, Minute, Rule, Shell, is_up};
|
|
use crate::core::socket::Command;
|
|
use engineering_challenge::book::{Book, Output, format_fixed, parse_fixed};
|
|
|
|
/// How many of a symbol's last changes are remembered: a slower connection's
|
|
/// copy is recognised among them (it ran up to 82 behind, docs/notes.md), and
|
|
/// so is a fresh book behind our stream.
|
|
const RECENT_CHANGES: usize = 128;
|
|
/// How long a change may be held while another connection might still deliver
|
|
/// the one before it: the second delivery lagged up to 0.7 s.
|
|
const HOLD: TimeDelta = TimeDelta::seconds(1);
|
|
/// How long once no other connection can: only the rest of a batch is still to
|
|
/// come, from the same connection (40 ms at worst).
|
|
const HOLD_ALONE: TimeDelta = TimeDelta::milliseconds(50);
|
|
/// The most changes a connection holds for a symbol: past it, its ceiling is
|
|
/// reached at once. A burst (thousands a second at a crypto market's
|
|
/// 5-minute mark) would otherwise pile up until the ceiling, each one tried
|
|
/// against the book.
|
|
const MAX_HELD: usize = 512;
|
|
/// How many tails of a connection's held changes are tried against the book:
|
|
/// the whole, and the last few that start on a batch.
|
|
const TAILS_TRIED: usize = 4;
|
|
/// When a change's copies are counted: no second copy came later than 0.7 s.
|
|
const COPIES_BY: TimeDelta = TimeDelta::seconds(2);
|
|
/// A reconnect's opening book is judged only for a symbol unchanged this
|
|
/// long: a moving symbol's books run ahead of or behind our stream.
|
|
const STILL_FOR: TimeDelta = TimeDelta::seconds(10);
|
|
|
|
/// Polymarket's book hash, as bytes: 20 instead of 40 as text, for 128 of
|
|
/// them per symbol.
|
|
type Hash = [u8; 20];
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
enum State {
|
|
/// No book yet since subscribing, or since every connection carrying it
|
|
/// went down: changes are dropped until one comes.
|
|
AwaitingBook,
|
|
/// Its book gives the exchange's hash after every change: first one wins.
|
|
Verified,
|
|
/// A change fitted on no connection in time: this connection's changes
|
|
/// are applied unchecked, the others' dropped, until a fresh book.
|
|
Following(usize),
|
|
/// A complete book didn't match its own hash, so our formula or metadata
|
|
/// is wrong for it: this connection's changes are applied unchecked, for
|
|
/// good.
|
|
Unverifiable(usize),
|
|
}
|
|
|
|
/// A change that doesn't fit the book yet, kept unwritten.
|
|
#[derive(Debug, Clone)]
|
|
struct Held {
|
|
received_at: DateTime<Utc>,
|
|
exchange_timestamp: DateTime<Utc>,
|
|
/// As the exchange wrote it: the hash takes it so.
|
|
timestamp: String,
|
|
side: Side,
|
|
price: i64,
|
|
quantity: i64,
|
|
hash: Hash,
|
|
}
|
|
|
|
impl Held {
|
|
fn same(&self, other: &Held) -> bool {
|
|
self.key() == other.key()
|
|
}
|
|
|
|
/// What tells a change from another: two copies have the same.
|
|
fn key(&self) -> (Hash, Side, i64, i64) {
|
|
(self.hash, self.side, self.price, self.quantity)
|
|
}
|
|
}
|
|
|
|
impl RecentChange {
|
|
fn is_copy(&self, change: &Held) -> bool {
|
|
(self.hash, self.side, self.price, self.quantity) == change.key()
|
|
}
|
|
}
|
|
|
|
/// A change as applied, with the exchange's hash of the book after it.
|
|
struct RecentChange {
|
|
side: Side,
|
|
price: i64,
|
|
quantity: i64,
|
|
/// The level's quantity before it (0 if absent): what undoes it.
|
|
previous: i64,
|
|
hash: Hash,
|
|
/// When its first copy came: a later copy's lag is measured from it.
|
|
received_at: DateTime<Utc>,
|
|
/// How many copies came: one, of a symbol on two connections, is a change
|
|
/// the other missed.
|
|
copies: u8,
|
|
/// Its copies are counted, or not to be: applied unchecked, the others'
|
|
/// copies were dropped unseen.
|
|
counted: bool,
|
|
}
|
|
|
|
/// A symbol's state for the rule.
|
|
pub struct Chained {
|
|
state: State,
|
|
/// Its last [`RECENT_CHANGES`] changes.
|
|
recent_changes: VecDeque<RecentChange>,
|
|
/// By connection: the changes it delivered that don't fit yet.
|
|
held: BTreeMap<usize, Vec<Held>>,
|
|
/// When the book last moved: held changes older than this, on a
|
|
/// connection whose changes still don't fit, are that connection's lag.
|
|
advanced_at: DateTime<Utc>,
|
|
/// The connection a fresh book was asked from, and whether it is written
|
|
/// whatever it holds: the connection followed changed.
|
|
asked: Option<(usize, bool)>,
|
|
/// Whether a change was verified since its first book: a fresh book that
|
|
/// differs before then shows a subscription gap, not a loss.
|
|
verified_since_book: bool,
|
|
/// It had a book when every connection carrying it went down: the
|
|
/// reconnect's book is compared with it.
|
|
had_book: bool,
|
|
// The other inputs of the book hash, as the exchange writes them (the
|
|
// market's id is the shell's).
|
|
tick_size: String,
|
|
last_trade_price: String,
|
|
min_order_size: String,
|
|
neg_risk: bool,
|
|
}
|
|
|
|
impl Chained {
|
|
fn new(market: &Market) -> Self {
|
|
Self {
|
|
state: State::AwaitingBook,
|
|
recent_changes: VecDeque::new(),
|
|
held: BTreeMap::new(),
|
|
advanced_at: DateTime::UNIX_EPOCH,
|
|
asked: None,
|
|
verified_since_book: false,
|
|
had_book: false,
|
|
tick_size: String::new(),
|
|
last_trade_price: String::new(),
|
|
min_order_size: market.min_order_size.clone(),
|
|
neg_risk: market.neg_risk,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A symbol: the shell's, with the rule's state.
|
|
type Symbol = feed::Symbol<Chained>;
|
|
|
|
impl Symbol {
|
|
/// Asks `connection` for a fresh book: unsubscribe and subscribe again.
|
|
fn ask_fresh_book(
|
|
&mut self,
|
|
token: &str,
|
|
connection: usize,
|
|
switch: bool,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
// A switch already asked stays one until its book comes.
|
|
let switch = switch || self.rule.asked.is_some_and(|(_, switch)| switch);
|
|
self.rule.asked = Some((connection, switch));
|
|
effects.push(Effect::Send(
|
|
connection,
|
|
Command::Resubscribe(token.to_string()),
|
|
));
|
|
}
|
|
|
|
/// Whether `book` is ours as it was after one of our recent changes: ours
|
|
/// rewound, each change undone. Comparing `book` plus our later changes
|
|
/// with ours instead would be unsound: a lost change at a price we had
|
|
/// changed before would be overwritten by that earlier change.
|
|
fn was_ours(&self, book: &Book) -> bool {
|
|
let mut rewound = self.book.clone();
|
|
self.rule.recent_changes.iter().rev().any(|change| {
|
|
rewound.set(change.side, change.price, change.previous);
|
|
rewound == *book
|
|
})
|
|
}
|
|
|
|
/// `book` with our recent changes, from the `from`-th on, applied.
|
|
fn brought_forward(&self, book: &Book, from: usize) -> Book {
|
|
let mut book = book.clone();
|
|
for change in self.rule.recent_changes.iter().skip(from) {
|
|
book.set(change.side, change.price, change.quantity);
|
|
}
|
|
book
|
|
}
|
|
|
|
/// Applies a change, `checked` against its hash or not, and remembers it;
|
|
/// its Delta, unless it leaves the book as it was.
|
|
fn apply(
|
|
&mut self,
|
|
token: &str,
|
|
change: &Held,
|
|
now: DateTime<Utc>,
|
|
checked: bool,
|
|
counts: &mut Counts,
|
|
) -> Option<Output> {
|
|
let previous = self.book.set(change.side, change.price, change.quantity);
|
|
if self.rule.recent_changes.len() == RECENT_CHANGES
|
|
&& let Some(mut oldest) = self.rule.recent_changes.pop_front()
|
|
{
|
|
Self::count_copies(&mut oldest, self.connections.len(), counts);
|
|
}
|
|
self.rule.recent_changes.push_back(RecentChange {
|
|
side: change.side,
|
|
price: change.price,
|
|
quantity: change.quantity,
|
|
previous,
|
|
hash: change.hash,
|
|
received_at: change.received_at,
|
|
copies: 1,
|
|
counted: !checked,
|
|
});
|
|
self.exchange_timestamp = change.exchange_timestamp;
|
|
self.rule.advanced_at = now;
|
|
if previous == change.quantity {
|
|
return None;
|
|
}
|
|
Some(self.delta(
|
|
token,
|
|
change.side,
|
|
change.price,
|
|
change.quantity,
|
|
change.received_at,
|
|
change.exchange_timestamp,
|
|
))
|
|
}
|
|
|
|
/// A change delivered by one connection only, of a symbol on several:
|
|
/// rescued.
|
|
fn count_copies(change: &mut RecentChange, connections: usize, counts: &mut Counts) {
|
|
if !change.counted {
|
|
change.counted = true;
|
|
if change.copies == 1 && connections > 1 {
|
|
counts.rescued += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Counts the copies of its changes old enough for every copy to have
|
|
/// come.
|
|
fn count_old_copies(&mut self, now: DateTime<Utc>, counts: &mut Counts) {
|
|
let connections = self.connections.len();
|
|
for change in &mut self.rule.recent_changes {
|
|
if now - change.received_at > COPIES_BY {
|
|
Self::count_copies(change, connections, counts);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Where the shortest tail of `connection`'s held changes that gives the
|
|
/// hash of its last one starts: that tail fits; what comes before it is
|
|
/// already in the book. A tail is whole batches, so only those starting a
|
|
/// batch are tried: the whole list, and the last [`TAILS_TRIED`].
|
|
fn fit(&self, token: &str, connection: usize) -> Option<usize> {
|
|
let held = self.rule.held.get(&connection)?;
|
|
let last = held.last()?;
|
|
let batch_starts = (1..held.len())
|
|
.rev()
|
|
.filter(|&start| held[start].hash != held[start - 1].hash)
|
|
.take(TAILS_TRIED);
|
|
batch_starts.chain([0]).find(|&start| {
|
|
let mut book = self.book.clone();
|
|
for change in &held[start..] {
|
|
book.set(change.side, change.price, change.quantity);
|
|
}
|
|
hash(token, self, &book, &last.timestamp) == last.hash
|
|
})
|
|
}
|
|
|
|
/// Drops the held changes that the book now holds: copies of applied
|
|
/// changes, and changes older than its last.
|
|
fn prune(&mut self) {
|
|
let (recent, newest) = (&self.rule.recent_changes, self.exchange_timestamp);
|
|
for held in self.rule.held.values_mut() {
|
|
held.retain(|change| {
|
|
change.exchange_timestamp >= newest
|
|
&& !recent.iter().any(|applied| applied.is_copy(change))
|
|
});
|
|
}
|
|
self.rule.held.retain(|_, held| !held.is_empty());
|
|
}
|
|
}
|
|
|
|
/// What the minute line counts, since the start.
|
|
#[derive(Debug, Default)]
|
|
struct Counts {
|
|
written: u64,
|
|
verified: u64,
|
|
duplicates: u64,
|
|
/// Changes of a symbol on several connections that only one delivered:
|
|
/// the others missed them.
|
|
rescued: u64,
|
|
/// Held changes dropped at their ceiling, the book having moved on
|
|
/// without them: their connection was behind.
|
|
held_dropped: u64,
|
|
chains_broken: u64,
|
|
resyncs: u64,
|
|
/// Fresh books that were a state we had already passed.
|
|
books_behind: u64,
|
|
subscription_gaps: u64,
|
|
losses: u64,
|
|
switches: u64,
|
|
/// A reconnect's opening books judged, those of still symbols.
|
|
reconnect_books: u64,
|
|
/// Of them, books that differed from every state of ours: both
|
|
/// subscriptions were quiet.
|
|
reconnect_losses: u64,
|
|
}
|
|
|
|
/// Polymarket's rule, for one book thread.
|
|
#[derive(Default)]
|
|
pub struct Chain {
|
|
counts: Counts,
|
|
/// The symbols with held changes: their ceilings are checked.
|
|
waiting: BTreeSet<String>,
|
|
/// How long after its first copy each duplicate came this minute, µs.
|
|
lags: Vec<i64>,
|
|
}
|
|
|
|
impl Chain {
|
|
/// `connection` no longer delivers for the symbol (it went down, or the
|
|
/// symbol left it): a symbol following it follows another, whose fresh
|
|
/// book is written whatever it holds; one with no connection left up
|
|
/// awaits a book.
|
|
fn lose(
|
|
token: &str,
|
|
symbol: &mut Symbol,
|
|
connection: usize,
|
|
up: &[bool],
|
|
counts: &mut Counts,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
let other = symbol.other_up(up, connection);
|
|
match (symbol.rule.state, other) {
|
|
(State::AwaitingBook, _) => {}
|
|
(State::Following(followed) | State::Unverifiable(followed), Some(other))
|
|
if followed == connection =>
|
|
{
|
|
symbol.rule.state = match symbol.rule.state {
|
|
State::Unverifiable(_) => State::Unverifiable(other),
|
|
_ => State::Following(other),
|
|
};
|
|
symbol.ask_fresh_book(token, other, true, effects);
|
|
counts.resyncs += 1;
|
|
}
|
|
(_, None) => {
|
|
// Nothing carries it: changes are being missed.
|
|
symbol.rule.state = State::AwaitingBook;
|
|
symbol.rule.had_book = true;
|
|
symbol.rule.held.clear();
|
|
symbol.rule.asked = None;
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
/// Held changes past their ceiling: dropped if the book moved on without
|
|
/// them (their connection is behind), else the chain is broken and their
|
|
/// connection followed.
|
|
fn expire(
|
|
&mut self,
|
|
shell: &mut Shell<Chained>,
|
|
now: DateTime<Utc>,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
let mut done = Vec::new();
|
|
for token in &self.waiting {
|
|
let Some(symbol) = shell.symbols.get_mut(token) else {
|
|
done.push(token.clone());
|
|
continue;
|
|
};
|
|
let due = symbol.rule.held.iter().find_map(|(&connection, held)| {
|
|
let first = held.first()?;
|
|
// Another connection that is up and doesn't hold the same
|
|
// change could still deliver the one before it.
|
|
let others_can = symbol.connections.iter().any(|&other| {
|
|
other != connection
|
|
&& is_up(&shell.up, other)
|
|
&& symbol
|
|
.rule
|
|
.held
|
|
.get(&other)
|
|
.and_then(|held| held.first())
|
|
.is_none_or(|theirs| !theirs.same(first))
|
|
});
|
|
let ceiling = if others_can { HOLD } else { HOLD_ALONE };
|
|
let due = now - first.received_at > ceiling || held.len() > MAX_HELD;
|
|
due.then_some((connection, first.received_at))
|
|
});
|
|
if let Some((connection, since)) = due {
|
|
if symbol.rule.advanced_at > since {
|
|
let dropped = symbol
|
|
.rule
|
|
.held
|
|
.remove(&connection)
|
|
.map_or(0, |held| held.len());
|
|
self.counts.held_dropped += dropped as u64;
|
|
} else {
|
|
Self::break_chain(
|
|
token,
|
|
symbol,
|
|
connection,
|
|
now,
|
|
&shell.up,
|
|
&mut self.counts,
|
|
effects,
|
|
);
|
|
}
|
|
}
|
|
if symbol.rule.held.is_empty() {
|
|
done.push(token.clone());
|
|
}
|
|
}
|
|
for token in done {
|
|
self.waiting.remove(&token);
|
|
}
|
|
}
|
|
|
|
/// No connection's changes fit: `connection`'s are applied unchecked from
|
|
/// now, and a fresh book is asked from another, so the followed stream is
|
|
/// not interrupted.
|
|
fn break_chain(
|
|
token: &str,
|
|
symbol: &mut Symbol,
|
|
connection: usize,
|
|
now: DateTime<Utc>,
|
|
up: &[bool],
|
|
counts: &mut Counts,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
counts.chains_broken += 1;
|
|
symbol.rule.state = State::Following(connection);
|
|
for change in symbol.rule.held.remove(&connection).unwrap_or_default() {
|
|
if let Some(delta) = symbol.apply(token, &change, now, false, counts) {
|
|
counts.written += 1;
|
|
effects.push(Effect::Write(delta));
|
|
}
|
|
}
|
|
symbol.rule.held.clear();
|
|
let asked = symbol.other_up(up, connection).unwrap_or(connection);
|
|
symbol.ask_fresh_book(token, asked, false, effects);
|
|
counts.resyncs += 1;
|
|
}
|
|
|
|
/// A full book counts when asked for (a symbol's first, from any of its
|
|
/// connections, or the answer to a resync), or when it is a reconnect's
|
|
/// opening book of a symbol unchanged for [`STILL_FOR`]. It replaces
|
|
/// ours, brought forward first if it is behind our stream, and is written
|
|
/// as a Snapshot when it is the symbol's first, when it differs from
|
|
/// ours, or when the connection followed changed. Then it is checked
|
|
/// against its own hash. A reconnect's book that is ours, or behind
|
|
/// ours, changes nothing at all.
|
|
fn on_book(
|
|
&mut self,
|
|
shell: &mut Shell<Chained>,
|
|
received_at: DateTime<Utc>,
|
|
connection: usize,
|
|
message: BookMessage,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
let token = message.asset_id;
|
|
let Some(symbol) = shell.symbols.get_mut(&token) else {
|
|
return;
|
|
};
|
|
// Books on subscribe carry every field; those sent mid-stream, after
|
|
// a trade, omit these two and are never an answer.
|
|
let complete = message.last_trade_price.is_some();
|
|
let asked = symbol.rule.asked.filter(|&(asked, _)| asked == connection);
|
|
let first =
|
|
symbol.rule.state == State::AwaitingBook && symbol.connections.contains(&connection);
|
|
// The connection's first complete book since it came up, for a
|
|
// symbol subscribed there before: the reconnect's opening book.
|
|
let up_since = shell.up_since.get(connection).copied().flatten();
|
|
let reconnect = complete
|
|
&& symbol.watch_mut(connection).is_some_and(|watch| {
|
|
// A book at the very instant it came up came before it.
|
|
let opening = up_since.is_some_and(|up| watch.since < up && watch.book_at <= up);
|
|
watch.book_at = received_at;
|
|
opening
|
|
});
|
|
// Judged only for a still, verified symbol, nothing held or asked.
|
|
let reconnect = reconnect
|
|
&& asked.is_none()
|
|
&& symbol.rule.asked.is_none()
|
|
&& symbol.rule.state == State::Verified
|
|
&& symbol.rule.held.is_empty()
|
|
&& received_at - symbol.rule.advanced_at >= STILL_FOR;
|
|
if !complete || (asked.is_none() && !first && !reconnect) {
|
|
return;
|
|
}
|
|
let (Some(book), Some(exchange_timestamp), Some(book_hash)) = (
|
|
parse_book(&message.bids, &message.asks),
|
|
parse_millis(&message.timestamp),
|
|
parse_hash(&message.hash),
|
|
) else {
|
|
// Nobody asked for a reconnect's book: unreadable, it is ignored.
|
|
if reconnect {
|
|
return;
|
|
}
|
|
// A protocol error: ask again. A book that stays unreadable is asked
|
|
// for once per answer, and its symbol stays unchecked meanwhile.
|
|
log!("polymarket: {token}: unreadable book, asking for a fresh one");
|
|
symbol.ask_fresh_book(&token, connection, false, effects);
|
|
self.counts.resyncs += 1;
|
|
return;
|
|
};
|
|
if reconnect {
|
|
self.counts.reconnect_books += 1;
|
|
// The exchange's opening books lag by minutes (docs/notes.md): one
|
|
// that is ours, or a state we passed, leaves ours, and the hash
|
|
// inputs, as they are.
|
|
let brought = symbol
|
|
.rule
|
|
.recent_changes
|
|
.iter()
|
|
.rposition(|change| change.hash == book_hash)
|
|
.map(|position| symbol.brought_forward(&book, position + 1));
|
|
if book == symbol.book || brought.as_ref() == Some(&symbol.book) {
|
|
return;
|
|
}
|
|
// Our chain holds up to our last change, so a book no newer than
|
|
// it holds nothing we missed, however far back it lags: further
|
|
// than the changes we remember, too.
|
|
if exchange_timestamp <= symbol.exchange_timestamp || symbol.was_ours(&book) {
|
|
self.counts.books_behind += 1;
|
|
return;
|
|
}
|
|
}
|
|
let inputs = reconnect.then(|| {
|
|
(
|
|
symbol.rule.tick_size.clone(),
|
|
symbol.rule.last_trade_price.clone(),
|
|
)
|
|
});
|
|
if let Some(tick_size) = message.tick_size {
|
|
symbol.rule.tick_size = tick_size;
|
|
}
|
|
if let Some(last_trade_price) = message.last_trade_price {
|
|
symbol.rule.last_trade_price = last_trade_price;
|
|
}
|
|
// Checked as received, before being brought forward below.
|
|
let verified = hash(&token, symbol, &book, &message.timestamp) == book_hash;
|
|
if let Some(inputs) = inputs {
|
|
// A book failing its own hash proves nothing against a verified
|
|
// one: ours stands, with its hash inputs.
|
|
if !verified {
|
|
(symbol.rule.tick_size, symbol.rule.last_trade_price) = inputs;
|
|
return;
|
|
}
|
|
// Newer than ours and no state of ours: both subscriptions were
|
|
// quiet. The other one is renewed too; its book is not waited for.
|
|
self.counts.reconnect_losses += 1;
|
|
for other in symbol.connections.clone() {
|
|
if other != connection {
|
|
if let Some(watch) = symbol.watch_mut(other) {
|
|
watch.since = received_at;
|
|
}
|
|
effects.push(Effect::Send(other, Command::Resubscribe(token.clone())));
|
|
}
|
|
}
|
|
}
|
|
let switch = asked.is_some_and(|(_, switch)| switch);
|
|
let compared = !first || symbol.rule.had_book;
|
|
|
|
let mut book = book;
|
|
let book_time = exchange_timestamp;
|
|
let mut exchange_timestamp = exchange_timestamp;
|
|
// The book a resync brings can lag the changes already applied, with a
|
|
// newer timestamp or last trade price than its levels (docs/notes.md).
|
|
// It is behind if it is ours as it was a few changes ago: that past
|
|
// state was the exchange's own, and the changes since are what it
|
|
// sent us, so ours stands.
|
|
if compared && book != symbol.book && symbol.was_ours(&book) {
|
|
self.counts.books_behind += 1;
|
|
book = symbol.book.clone();
|
|
exchange_timestamp = symbol.exchange_timestamp;
|
|
} else if let Some(position) = symbol
|
|
.rule
|
|
.recent_changes
|
|
.iter()
|
|
.rposition(|change| change.hash == book_hash)
|
|
.filter(|_| compared)
|
|
{
|
|
// Its hash is that of one of our changes: the exchange's state
|
|
// after it, which the changes we got since bring to now.
|
|
book = symbol.brought_forward(&book, position + 1);
|
|
exchange_timestamp = symbol.exchange_timestamp;
|
|
}
|
|
let differs = compared && book != symbol.book;
|
|
let cause = if switch {
|
|
Some(Cause::Switch)
|
|
} else if differs && !reconnect && !symbol.rule.verified_since_book && !symbol.rule.had_book
|
|
{
|
|
// A subscribe hides a few ms of changes (docs/notes.md): until a
|
|
// change is verified, a differing book is that gap.
|
|
Some(Cause::SubscriptionGap)
|
|
} else if differs {
|
|
Some(Cause::Loss)
|
|
} else {
|
|
None
|
|
};
|
|
match cause {
|
|
Some(Cause::Loss) => self.counts.losses += 1,
|
|
Some(Cause::SubscriptionGap) => self.counts.subscription_gaps += 1,
|
|
Some(Cause::Switch) => self.counts.switches += 1,
|
|
None => {}
|
|
}
|
|
symbol.book = book;
|
|
symbol.exchange_timestamp = exchange_timestamp;
|
|
symbol.rule.advanced_at = received_at;
|
|
symbol.rule.held.clear();
|
|
symbol.rule.asked = None;
|
|
symbol.rule.had_book = false;
|
|
if !compared || cause.is_some() {
|
|
// Our changes before this book no longer lead to ours: rewinding
|
|
// through them would make up states that never existed.
|
|
symbol.rule.recent_changes.clear();
|
|
let snapshot = symbol
|
|
.book
|
|
.snapshot(&token, received_at, exchange_timestamp);
|
|
effects.push(Effect::Write(snapshot));
|
|
symbol.last_written = symbol.last_written.max(received_at);
|
|
if let Some(cause) = cause {
|
|
let found = if reconnect {
|
|
", found by a reconnect"
|
|
} else {
|
|
""
|
|
};
|
|
let detail = format!(
|
|
"{found} (its time {}, ours {})",
|
|
book_time.format("%H:%M:%S%.3f"),
|
|
exchange_timestamp.format("%H:%M:%S%.3f")
|
|
);
|
|
log!(
|
|
"polymarket: {}",
|
|
cause::line(&token, received_at, cause, &detail)
|
|
);
|
|
}
|
|
}
|
|
symbol.rule.state = if verified {
|
|
State::Verified
|
|
} else {
|
|
if !matches!(symbol.rule.state, State::Unverifiable(_)) {
|
|
log!(
|
|
"polymarket: {token}: a complete book fails its own hash, changes go unchecked"
|
|
);
|
|
}
|
|
State::Unverifiable(connection)
|
|
};
|
|
self.waiting.remove(&token);
|
|
}
|
|
|
|
fn on_price_change(
|
|
&mut self,
|
|
shell: &mut Shell<Chained>,
|
|
received_at: DateTime<Utc>,
|
|
connection: usize,
|
|
message: PriceChangeMessage,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
let exchange_timestamp = parse_millis(&message.timestamp);
|
|
for change in message.price_changes {
|
|
let token = change.asset_id;
|
|
shell.count_received(&token, connection);
|
|
let Some(symbol) = shell.symbols.get_mut(&token) else {
|
|
continue;
|
|
};
|
|
let side = match change.side.as_str() {
|
|
"BUY" => Some(Side::Buy),
|
|
"SELL" => Some(Side::Sell),
|
|
_ => None,
|
|
};
|
|
let (Some(side), Some(price), Some(quantity), Some(exchange_timestamp), Some(hash)) = (
|
|
side,
|
|
parse_fixed(&change.price),
|
|
parse_fixed(&change.size),
|
|
exchange_timestamp,
|
|
parse_hash(&change.hash),
|
|
) else {
|
|
// Another connection's copy can still bring it; if none does,
|
|
// the next change of this one waits for it in vain.
|
|
log!(
|
|
"polymarket: {token}: unreadable change from connection {connection}, dropped"
|
|
);
|
|
continue;
|
|
};
|
|
let change = Held {
|
|
received_at,
|
|
exchange_timestamp,
|
|
timestamp: message.timestamp.clone(),
|
|
side,
|
|
price,
|
|
quantity,
|
|
hash,
|
|
};
|
|
match symbol.rule.state {
|
|
State::AwaitingBook => {}
|
|
State::Following(followed) | State::Unverifiable(followed) => {
|
|
if followed == connection
|
|
&& let Some(delta) =
|
|
symbol.apply(&token, &change, received_at, false, &mut self.counts)
|
|
{
|
|
self.counts.written += 1;
|
|
effects.push(Effect::Write(delta));
|
|
}
|
|
}
|
|
State::Verified => {
|
|
Self::first_one_wins(
|
|
&token,
|
|
symbol,
|
|
connection,
|
|
change,
|
|
received_at,
|
|
&mut self.counts,
|
|
&mut self.lags,
|
|
effects,
|
|
);
|
|
if symbol.rule.held.is_empty() {
|
|
self.waiting.remove(&token);
|
|
} else {
|
|
self.waiting.insert(token);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A verified symbol's change: dropped if the book has it, else held on
|
|
/// its connection until a tail of its held changes fits; each fit may let
|
|
/// another connection's held changes fit in turn.
|
|
// The rule's fields it needs (the counts, the lags), borrowed apart from
|
|
// the symbol, which its callers hold out of the shell.
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn first_one_wins(
|
|
token: &str,
|
|
symbol: &mut Symbol,
|
|
connection: usize,
|
|
change: Held,
|
|
now: DateTime<Utc>,
|
|
counts: &mut Counts,
|
|
lags: &mut Vec<i64>,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
if let Some(first) = symbol
|
|
.rule
|
|
.recent_changes
|
|
.iter_mut()
|
|
.find(|applied| applied.is_copy(&change))
|
|
{
|
|
counts.duplicates += 1;
|
|
first.copies = first.copies.saturating_add(1);
|
|
if let Some(lag) = (change.received_at - first.received_at).num_microseconds() {
|
|
lags.push(lag);
|
|
}
|
|
return;
|
|
}
|
|
// While the chain holds, everything up to the book's state was applied.
|
|
if change.exchange_timestamp < symbol.exchange_timestamp {
|
|
counts.duplicates += 1;
|
|
return;
|
|
}
|
|
symbol.rule.held.entry(connection).or_default().push(change);
|
|
let mut fitted = true;
|
|
while fitted {
|
|
fitted = false;
|
|
let connections: Vec<usize> = symbol.rule.held.keys().copied().collect();
|
|
for held_on in connections {
|
|
let Some(start) = symbol.fit(token, held_on) else {
|
|
continue;
|
|
};
|
|
let held = symbol.rule.held.remove(&held_on).unwrap_or_default();
|
|
for change in &held[start..] {
|
|
counts.verified += 1;
|
|
if let Some(delta) = symbol.apply(token, change, now, true, counts) {
|
|
counts.written += 1;
|
|
effects.push(Effect::Write(delta));
|
|
}
|
|
}
|
|
symbol.rule.verified_since_book = true;
|
|
symbol.prune();
|
|
fitted = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Rule for Chain {
|
|
type Market = Market;
|
|
type Event = Event;
|
|
type Symbol = Chained;
|
|
|
|
fn symbol(&self, market: &Market) -> Chained {
|
|
Chained::new(market)
|
|
}
|
|
|
|
fn on_event(
|
|
&mut self,
|
|
shell: &mut Shell<Chained>,
|
|
connection: usize,
|
|
received_at: DateTime<Utc>,
|
|
event: Event,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
match event {
|
|
Event::Book(book) => self.on_book(shell, received_at, connection, book, effects),
|
|
Event::PriceChange(change) => {
|
|
self.on_price_change(shell, received_at, connection, change, effects)
|
|
}
|
|
Event::TickSizeChange { asset_id, .. } => {
|
|
if let Some(symbol) = shell.symbols.get_mut(&asset_id)
|
|
&& symbol.rule.state == State::Verified
|
|
{
|
|
log!("polymarket: {asset_id}: tick size changed, resync");
|
|
Self::break_chain(
|
|
&asset_id,
|
|
symbol,
|
|
connection,
|
|
received_at,
|
|
&shell.up,
|
|
&mut self.counts,
|
|
effects,
|
|
);
|
|
}
|
|
}
|
|
// The hash takes a new last trade price before this event, and
|
|
// not always its value, so a fresh book brings it instead
|
|
// (docs/notes.md).
|
|
Event::LastTradePrice { .. } | Event::Other => {}
|
|
}
|
|
}
|
|
|
|
/// Its held changes there are dropped, and a symbol following it
|
|
/// follows another.
|
|
fn on_leave(
|
|
&mut self,
|
|
up: &[bool],
|
|
token: &str,
|
|
symbol: &mut Symbol,
|
|
connection: usize,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
symbol.rule.held.remove(&connection);
|
|
Self::lose(token, symbol, connection, up, &mut self.counts, effects);
|
|
// A fresh book asked from it would never come: ask another.
|
|
if let Some((asked, switch)) = symbol.rule.asked
|
|
&& asked == connection
|
|
&& let Some(other) = symbol.other_up(up, connection)
|
|
{
|
|
symbol.ask_fresh_book(token, other, switch, effects);
|
|
self.counts.resyncs += 1;
|
|
}
|
|
}
|
|
|
|
fn on_down(
|
|
&mut self,
|
|
shell: &mut Shell<Chained>,
|
|
connection: usize,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
for (token, symbol) in &mut shell.symbols {
|
|
if !symbol.connections.contains(&connection) {
|
|
continue;
|
|
}
|
|
symbol.rule.held.remove(&connection);
|
|
Self::lose(
|
|
token,
|
|
symbol,
|
|
connection,
|
|
&shell.up,
|
|
&mut self.counts,
|
|
effects,
|
|
);
|
|
}
|
|
self.waiting.retain(|token| {
|
|
shell
|
|
.symbols
|
|
.get(token)
|
|
.is_some_and(|symbol| !symbol.rule.held.is_empty())
|
|
});
|
|
}
|
|
|
|
fn on_timer(
|
|
&mut self,
|
|
shell: &mut Shell<Chained>,
|
|
now: DateTime<Utc>,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
self.expire(shell, now, effects);
|
|
}
|
|
|
|
fn wakes_soon(&self) -> bool {
|
|
!self.waiting.is_empty()
|
|
}
|
|
|
|
fn on_removed(&mut self, token: &str) {
|
|
self.waiting.remove(token);
|
|
}
|
|
|
|
fn has_book(symbol: &Chained) -> bool {
|
|
symbol.state != State::AwaitingBook
|
|
}
|
|
|
|
fn may_be_quiet(symbol: &Chained) -> bool {
|
|
symbol.state == State::Verified && symbol.asked.is_none()
|
|
}
|
|
|
|
/// Counts the copies of its old changes, and asks again for a fresh
|
|
/// book still not come.
|
|
fn on_minute(
|
|
&mut self,
|
|
up: &[bool],
|
|
token: &str,
|
|
symbol: &mut Symbol,
|
|
now: DateTime<Utc>,
|
|
effects: &mut Vec<Effect>,
|
|
) {
|
|
symbol.count_old_copies(now, &mut self.counts);
|
|
// The answer may have been lost or unreadable: ask again.
|
|
let again = match (symbol.rule.state, symbol.rule.asked) {
|
|
(_, Some((connection, _))) => Some(connection),
|
|
(State::AwaitingBook, None) => symbol
|
|
.connections
|
|
.iter()
|
|
.copied()
|
|
.find(|&connection| is_up(up, connection)),
|
|
_ => None,
|
|
};
|
|
if let Some(connection) = again {
|
|
let switch = symbol.rule.asked.is_some_and(|(_, switch)| switch);
|
|
symbol.ask_fresh_book(token, connection, switch, effects);
|
|
self.counts.resyncs += 1;
|
|
}
|
|
}
|
|
|
|
fn log_minute(&mut self, minute: &Minute) {
|
|
self.lags.sort_unstable();
|
|
let lag = |quantile: f64| {
|
|
let index = ((self.lags.len() as f64 * quantile) as usize)
|
|
.min(self.lags.len().saturating_sub(1));
|
|
self.lags
|
|
.get(index)
|
|
.map_or(0.0, |µs| micros as f64 / 1_000.0)
|
|
};
|
|
let counts = &self.counts;
|
|
log!(
|
|
"polymarket: book thread {}: {} symbols; so far {} changes written ({} verified), {} duplicates, \
|
|
{} rescued, {} held dropped, {} chains broken, {} resyncs, {} full books behind ours, \
|
|
{} subscription gaps, {} losses, {} switches, {} quiet subscriptions resubscribed, \
|
|
{} markets reported quiet again, {} reconnect books judged ({} losses); second copies \
|
|
{:.1} ms late at the median, {:.1} ms at p99; this minute {} changes received, at most \
|
|
{} events waited",
|
|
minute.number,
|
|
minute.symbols,
|
|
counts.written,
|
|
counts.verified,
|
|
counts.duplicates,
|
|
counts.rescued,
|
|
counts.held_dropped,
|
|
counts.chains_broken,
|
|
counts.resyncs,
|
|
counts.books_behind,
|
|
counts.subscription_gaps,
|
|
counts.losses,
|
|
counts.switches,
|
|
minute.quiet,
|
|
minute.quiet_reported,
|
|
counts.reconnect_books,
|
|
counts.reconnect_losses,
|
|
lag(0.5),
|
|
lag(0.99),
|
|
minute.received,
|
|
minute.peak_waiting
|
|
);
|
|
self.lags.clear();
|
|
}
|
|
}
|
|
|
|
/// Polymarket's book hash: SHA-1 of the book as its REST `/book` returns it,
|
|
/// `hash` emptied, keys in this order (docs/notes.md): `book` with the
|
|
/// symbol's other inputs.
|
|
fn hash(token: &str, symbol: &Symbol, book: &Book, timestamp: &str) -> Hash {
|
|
#[derive(Serialize)]
|
|
struct Level {
|
|
price: String,
|
|
size: String,
|
|
}
|
|
#[derive(Serialize)]
|
|
struct HashedBook<'a> {
|
|
market: &'a str,
|
|
asset_id: &'a str,
|
|
timestamp: &'a str,
|
|
hash: &'a str,
|
|
bids: Vec<Level>,
|
|
asks: Vec<Level>,
|
|
min_order_size: &'a str,
|
|
tick_size: &'a str,
|
|
neg_risk: bool,
|
|
last_trade_price: &'a str,
|
|
}
|
|
let level = |(&price, &size): (&i64, &i64)| Level {
|
|
price: format_fixed(price),
|
|
size: format_fixed(size),
|
|
};
|
|
let hashed = HashedBook {
|
|
market: &symbol.market,
|
|
asset_id: token,
|
|
timestamp,
|
|
hash: "",
|
|
// Bids ascending, asks descending: the best level of each side last.
|
|
bids: book.bids.iter().map(level).collect(),
|
|
asks: book.asks.iter().rev().map(level).collect(),
|
|
min_order_size: &symbol.rule.min_order_size,
|
|
tick_size: &symbol.rule.tick_size,
|
|
neg_risk: symbol.rule.neg_risk,
|
|
last_trade_price: &symbol.rule.last_trade_price,
|
|
};
|
|
let json = serde_json::to_string(&hashed).expect("strings and bools always serialize");
|
|
Sha1::digest(json.as_bytes()).into()
|
|
}
|
|
|
|
/// The exchange's 40 lowercase hex characters.
|
|
fn parse_hash(text: &str) -> Option<Hash> {
|
|
if text.len() != 40 {
|
|
return None;
|
|
}
|
|
let mut hash = [0; 20];
|
|
for (byte, pair) in hash.iter_mut().zip(text.as_bytes().chunks(2)) {
|
|
*byte = u8::from_str_radix(std::str::from_utf8(pair).ok()?, 16).ok()?;
|
|
}
|
|
Some(hash)
|
|
}
|
|
|
|
fn parse_book(bids: &[LevelMessage], asks: &[LevelMessage]) -> Option<Book> {
|
|
let side = |levels: &[LevelMessage]| {
|
|
levels
|
|
.iter()
|
|
.map(|level| Some((parse_fixed(&level.price)?, parse_fixed(&level.size)?)))
|
|
.collect::<Option<BTreeMap<_, _>>>()
|
|
};
|
|
Some(Book {
|
|
bids: side(bids)?,
|
|
asks: side(asks)?,
|
|
})
|
|
}
|
|
|
|
/// Polymarket timestamps: milliseconds since the epoch, as a string.
|
|
fn parse_millis(text: &str) -> Option<DateTime<Utc>> {
|
|
DateTime::from_timestamp_millis(text.parse().ok()?)
|
|
}
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use serde::Deserialize;
|
|
|
|
use super::*;
|
|
use crate::core::feed::{Loads, MIN_MEASURED, Report, next_minute, serve};
|
|
use crate::core::{Arrival, Event as _, Received};
|
|
use crate::polymarket::{fixture_markets, wire};
|
|
|
|
type Feed = feed::Feed<Chain>;
|
|
|
|
/// A frame's events as `connection` would send them.
|
|
fn arrivals(received_at: DateTime<Utc>, connection: usize, text: &str) -> Vec<Arrival<Event>> {
|
|
// The connection logs an unreadable frame and sends nothing.
|
|
let events = wire::decode(text).unwrap_or_default();
|
|
let bytes = (text.len() / events.len().max(1)) as u64;
|
|
events
|
|
.into_iter()
|
|
.filter(|event| event.market().is_some())
|
|
.map(|event| Arrival {
|
|
received_at,
|
|
connection,
|
|
bytes,
|
|
event,
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
impl Feed {
|
|
fn on_frame_from(
|
|
&mut self,
|
|
connection: usize,
|
|
received_at: DateTime<Utc>,
|
|
text: &str,
|
|
) -> Vec<Effect> {
|
|
arrivals(received_at, connection, text)
|
|
.into_iter()
|
|
.flat_map(|arrival| self.on_event(arrival))
|
|
.collect()
|
|
}
|
|
|
|
fn on_frame(&mut self, received_at: DateTime<Utc>, text: &str) -> Vec<Effect> {
|
|
self.on_frame_from(0, received_at, text)
|
|
}
|
|
}
|
|
|
|
/// A frame of the recordings: receive time, connection, text.
|
|
type Frame = (DateTime<Utc>, usize, String);
|
|
|
|
fn read_frames(file: &str) -> Vec<Frame> {
|
|
#[derive(Deserialize)]
|
|
struct Line {
|
|
received_at_us: i64,
|
|
#[serde(default)]
|
|
connection: usize,
|
|
text: String,
|
|
}
|
|
let path = format!("{}/tests/fixtures/{file}", env!("CARGO_MANIFEST_DIR"));
|
|
let mut frames: Vec<Frame> = std::fs::read_to_string(path)
|
|
.unwrap()
|
|
.lines()
|
|
.map(|line| {
|
|
let line: Line = serde_json::from_str(line).unwrap();
|
|
let received_at = DateTime::from_timestamp_micros(line.received_at_us).unwrap();
|
|
(received_at, line.connection, line.text)
|
|
})
|
|
.collect();
|
|
frames.sort_by_key(|(received_at, _, _)| *received_at);
|
|
frames
|
|
}
|
|
|
|
/// The recorded live feed for the fixture's markets, on one connection
|
|
/// (tests/fixtures/README.md).
|
|
fn fixture_frames() -> Vec<(DateTime<Utc>, String)> {
|
|
read_frames("polymarket-frames.jsonl")
|
|
.into_iter()
|
|
.map(|(received_at, _, text)| (received_at, text))
|
|
.collect()
|
|
}
|
|
|
|
/// The same markets recorded on two connections at once, in receive order.
|
|
fn two_connections() -> Vec<Frame> {
|
|
read_frames("polymarket-frames-two.jsonl")
|
|
}
|
|
|
|
/// A book thread's feed holding `markets` on `connections`, as the
|
|
/// orchestrator's Adds leave it at `start`.
|
|
fn feed_on(markets: &[Market], connections: &[usize], start: DateTime<Utc>) -> Feed {
|
|
let mut feed = Feed::new("polymarket", 0, Chain::default());
|
|
for market in markets {
|
|
feed.add_market(market, connections, start);
|
|
}
|
|
feed
|
|
}
|
|
|
|
/// The one-connection recording's feed: its markets on connection 0.
|
|
fn feed_of(markets: &[Market]) -> Feed {
|
|
feed_on(markets, &[0], fixture_frames()[0].0)
|
|
}
|
|
|
|
fn replay<'a>(
|
|
feed: &mut Feed,
|
|
frames: impl IntoIterator<Item = &'a (DateTime<Utc>, String)>,
|
|
) -> Vec<Effect> {
|
|
frames
|
|
.into_iter()
|
|
.flat_map(|(received_at, text)| feed.on_frame(*received_at, text))
|
|
.collect()
|
|
}
|
|
|
|
fn replay_both<'a>(
|
|
feed: &mut Feed,
|
|
frames: impl IntoIterator<Item = &'a Frame>,
|
|
) -> Vec<Effect> {
|
|
frames
|
|
.into_iter()
|
|
.flat_map(|(received_at, connection, text)| {
|
|
feed.on_frame_from(*connection, *received_at, text)
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
fn resyncs(effects: &[Effect]) -> Vec<(usize, &str)> {
|
|
effects
|
|
.iter()
|
|
.filter_map(|effect| match effect {
|
|
Effect::Send(connection, Command::Resubscribe(token)) => {
|
|
Some((*connection, token.as_str()))
|
|
}
|
|
_ => None,
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
fn snapshot_symbols(effects: &[Effect]) -> Vec<&str> {
|
|
effects
|
|
.iter()
|
|
.filter_map(|effect| match effect {
|
|
Effect::Write(Output::Snapshot(snapshot)) => Some(snapshot.symbol.as_str()),
|
|
_ => None,
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// A Delta's content: exchange time, side, price, quantity.
|
|
type Content = (DateTime<Utc>, Side, i64, i64);
|
|
|
|
/// Per symbol, its Deltas' content, in order, from `since` (exchange
|
|
/// time) on.
|
|
fn deltas(effects: &[Effect], since: DateTime<Utc>) -> BTreeMap<String, Vec<Content>> {
|
|
let mut deltas: BTreeMap<_, Vec<_>> = BTreeMap::new();
|
|
for effect in effects {
|
|
if let Effect::Write(Output::Delta(delta)) = effect
|
|
&& delta.exchange_timestamp >= since
|
|
{
|
|
deltas.entry(delta.symbol.clone()).or_default().push((
|
|
delta.exchange_timestamp,
|
|
delta.side,
|
|
delta.price_level,
|
|
delta.quantity,
|
|
));
|
|
}
|
|
}
|
|
deltas
|
|
}
|
|
|
|
fn delta_stamps(effects: &[Effect]) -> Vec<(String, DateTime<Utc>)> {
|
|
effects
|
|
.iter()
|
|
.filter_map(|effect| match effect {
|
|
Effect::Write(Output::Delta(delta)) => {
|
|
Some((delta.symbol.clone(), delta.backtest_timestamp))
|
|
}
|
|
_ => None,
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Whether the changes of frame `index` are lost for good if dropped: no
|
|
/// later frame touches their levels, and their symbols change again
|
|
/// later, so a missing change is seen.
|
|
fn telling_loss<T>(frames: &[T], index: usize, text: impl Fn(&T) -> &str) -> bool {
|
|
let levels = |text: &str| -> Vec<(String, String, String)> {
|
|
wire::decode(text)
|
|
.unwrap_or_default()
|
|
.into_iter()
|
|
.flat_map(|event| match event {
|
|
Event::PriceChange(change) => change.price_changes,
|
|
_ => Vec::new(),
|
|
})
|
|
.map(|change| (change.asset_id, change.side, change.price))
|
|
.collect()
|
|
};
|
|
let lost = levels(text(&frames[index]));
|
|
let later: Vec<_> = frames[index + 1..]
|
|
.iter()
|
|
.flat_map(|frame| levels(text(frame)))
|
|
.collect();
|
|
!lost.is_empty()
|
|
&& lost.iter().all(|level| !later.contains(level))
|
|
&& lost.iter().all(|(token, _, _)| {
|
|
later
|
|
.iter()
|
|
.filter(|(theirs, _, _)| theirs == token)
|
|
.count()
|
|
>= 3
|
|
})
|
|
}
|
|
|
|
/// The tokens a frame's price changes touch.
|
|
fn changed_tokens(text: &str) -> Vec<String> {
|
|
wire::decode(text)
|
|
.unwrap_or_default()
|
|
.into_iter()
|
|
.flat_map(|event| match event {
|
|
Event::PriceChange(change) => change.price_changes,
|
|
_ => Vec::new(),
|
|
})
|
|
.map(|change| change.asset_id)
|
|
.collect()
|
|
}
|
|
|
|
fn hex(hash: Hash) -> String {
|
|
hash.iter().map(|byte| format!("{byte:02x}")).collect()
|
|
}
|
|
|
|
/// A frame of `changes` to `token`'s book `base`, all at `timestamp` (ms),
|
|
/// carrying the hash of the book after all of them (a batch, if several),
|
|
/// and that book.
|
|
fn changes_frame(
|
|
token: &str,
|
|
symbol: &Symbol,
|
|
base: &Book,
|
|
changes: &[(Side, i64, i64)],
|
|
timestamp: i64,
|
|
) -> (String, Book) {
|
|
let mut book = base.clone();
|
|
for &(side, price, quantity) in changes {
|
|
book.set(side, price, quantity);
|
|
}
|
|
let timestamp = timestamp.to_string();
|
|
let hash = hex(hash(token, symbol, &book, ×tamp));
|
|
let price_changes: Vec<_> = changes
|
|
.iter()
|
|
.map(|&(side, price, quantity)| {
|
|
serde_json::json!({"asset_id": token, "price": format_fixed(price),
|
|
"size": format_fixed(quantity),
|
|
"side": if side == Side::Buy { "BUY" } else { "SELL" }, "hash": hash})
|
|
})
|
|
.collect();
|
|
let frame = serde_json::json!({"event_type": "price_change", "market": symbol.market,
|
|
"timestamp": timestamp, "price_changes": price_changes});
|
|
(frame.to_string(), book)
|
|
}
|
|
|
|
/// A complete book frame for `book`, as the exchange would send it in
|
|
/// answer to a resync, 5 ms after the symbol's last change.
|
|
fn fresh_book(token: &str, symbol: &Symbol, book: &Book) -> String {
|
|
let at = symbol.exchange_timestamp + TimeDelta::milliseconds(5);
|
|
book_frame(token, symbol, book, at)
|
|
}
|
|
|
|
/// A complete book frame for `book`, last changed at `at` (exchange time).
|
|
fn book_frame(token: &str, symbol: &Symbol, book: &Book, at: DateTime<Utc>) -> String {
|
|
let timestamp = at.timestamp_millis().to_string();
|
|
let levels = |levels: &BTreeMap<i64, i64>| {
|
|
levels
|
|
.iter()
|
|
.map(|(&price, &size)| serde_json::json!({"price": format_fixed(price), "size": format_fixed(size)}))
|
|
.collect::<Vec<_>>()
|
|
};
|
|
serde_json::json!({
|
|
"event_type": "book",
|
|
"market": symbol.market,
|
|
"asset_id": token,
|
|
"timestamp": timestamp,
|
|
"hash": hex(hash(token, symbol, book, ×tamp)),
|
|
"bids": levels(&book.bids),
|
|
"asks": levels(&book.asks),
|
|
"tick_size": symbol.rule.tick_size,
|
|
"last_trade_price": symbol.rule.last_trade_price,
|
|
})
|
|
.to_string()
|
|
}
|
|
|
|
/// The first verified symbol with a bid side, and a free bid price below
|
|
/// 1 for synthetic changes; the next free one is `free + 1_000` or above.
|
|
fn verified_symbol(feed: &Feed) -> (String, i64) {
|
|
let (token, symbol) = feed
|
|
.shell
|
|
.symbols
|
|
.iter()
|
|
.find(|(_, symbol)| symbol.rule.state == State::Verified && symbol.book.bids.len() > 1)
|
|
.expect("a verified symbol");
|
|
let free = (1_000..)
|
|
.step_by(1_000)
|
|
.find(|price| {
|
|
!symbol.book.bids.contains_key(price)
|
|
&& !symbol.book.bids.contains_key(&(price + 1_000))
|
|
})
|
|
.unwrap();
|
|
(token.clone(), free)
|
|
}
|
|
|
|
// --- one connection: sprint 02's rules still hold ---
|
|
|
|
/// Split batches and trades make checks wait or fail with nothing lost:
|
|
/// the recording must replay without a single loss.
|
|
#[test]
|
|
fn replay_has_no_loss() {
|
|
let mut feed = feed_of(&fixture_markets());
|
|
replay(&mut feed, &fixture_frames());
|
|
assert_eq!(feed.rule.counts.losses, 0);
|
|
assert!(
|
|
feed.shell
|
|
.symbols
|
|
.values()
|
|
.all(|symbol| !matches!(symbol.rule.state, State::Unverifiable(_)))
|
|
);
|
|
assert!(
|
|
feed.rule.counts.verified > 1_000,
|
|
"only {} changes verified",
|
|
feed.rule.counts.verified
|
|
);
|
|
// Alone, a connection never has a duplicate but the server's repeats.
|
|
assert!(feed.rule.counts.duplicates < 10, "{:?}", feed.rule.counts);
|
|
}
|
|
|
|
#[test]
|
|
fn a_lost_frame_breaks_the_chain_and_its_fresh_book_counts_a_loss() {
|
|
let frames = fixture_frames();
|
|
let markets = fixture_markets();
|
|
let mut control = feed_of(&markets);
|
|
let mut lost = None;
|
|
for (index, (received_at, text)) in frames.iter().enumerate() {
|
|
let tokens = changed_tokens(text);
|
|
if lost.is_none()
|
|
&& index > frames.len() / 4
|
|
&& !tokens.is_empty()
|
|
&& telling_loss(&frames, index, |(_, text)| text)
|
|
&& tokens.iter().all(|token| {
|
|
control.shell.symbols[token].rule.state == State::Verified
|
|
&& control.shell.symbols[token].rule.held.is_empty()
|
|
})
|
|
{
|
|
lost = Some((index, tokens[0].clone()));
|
|
}
|
|
control.on_frame(*received_at, text);
|
|
}
|
|
let (lost, token) = lost.unwrap();
|
|
|
|
let mut feed = feed_of(&markets);
|
|
replay(&mut feed, &frames[..lost]);
|
|
let chains = feed.rule.counts.chains_broken;
|
|
let effects = replay(&mut feed, &frames[lost + 1..]);
|
|
assert!(
|
|
resyncs(&effects).contains(&(0, token.as_str())),
|
|
"the loss is never caught"
|
|
);
|
|
assert!(feed.rule.counts.chains_broken > chains);
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Following(0));
|
|
|
|
// The fresh book: the exchange's, which our control holds.
|
|
let book = fresh_book(
|
|
&token,
|
|
&feed.shell.symbols[&token],
|
|
&control.shell.symbols[&token].book,
|
|
);
|
|
let losses = feed.rule.counts.losses;
|
|
let effects = feed.on_frame(frames.last().unwrap().0, &book);
|
|
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
|
|
assert_eq!(feed.rule.counts.losses, losses + 1);
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Verified);
|
|
}
|
|
|
|
/// A feed half replayed, a verified symbol of it that holds nothing, and a
|
|
/// complete book of that symbol as it was before its last change: a book
|
|
/// behind our stream, as a resync's fresh book can be. Its fresh book is
|
|
/// asked from connection 0.
|
|
fn feed_past_a_stale_book() -> (Feed, String, DateTime<Utc>, String) {
|
|
let frames = fixture_frames();
|
|
let mut feed = feed_of(&fixture_markets());
|
|
let half = frames.len() / 2;
|
|
replay(&mut feed, &frames[..half]);
|
|
let (token, symbol) = feed
|
|
.shell
|
|
.symbols
|
|
.iter_mut()
|
|
.find(|(_, symbol)| {
|
|
symbol.rule.state == State::Verified
|
|
&& symbol.rule.held.is_empty()
|
|
&& symbol.rule.recent_changes.len() >= 2
|
|
})
|
|
.expect("a verified symbol with changes");
|
|
let token = token.clone();
|
|
let last = symbol.rule.recent_changes.back().unwrap();
|
|
let mut before = symbol.book.clone();
|
|
before.set(last.side, last.price, last.previous);
|
|
let before_hash = symbol.rule.recent_changes[symbol.rule.recent_changes.len() - 2].hash;
|
|
let mut stale: serde_json::Value =
|
|
serde_json::from_str(&fresh_book(&token, symbol, &before)).unwrap();
|
|
stale["hash"] = serde_json::json!(hex(before_hash));
|
|
symbol.rule.asked = Some((0, false));
|
|
(feed, token, frames[half].0, stale.to_string())
|
|
}
|
|
|
|
#[test]
|
|
fn a_book_behind_our_stream_is_not_a_loss() {
|
|
let (mut feed, token, now, stale_book) = feed_past_a_stale_book();
|
|
let ours = feed.shell.symbols[&token].book.clone();
|
|
let effects = feed.on_frame(now, &stale_book);
|
|
|
|
assert!(snapshot_symbols(&effects).is_empty());
|
|
assert_eq!(feed.rule.counts.books_behind, 1);
|
|
assert_eq!(feed.rule.counts.losses, 0);
|
|
assert_eq!(feed.shell.symbols[&token].book, ours);
|
|
assert_eq!(feed.shell.symbols[&token].rule.asked, None);
|
|
}
|
|
|
|
/// Being behind says nothing of our own book: if it was the one lost, the
|
|
/// book brought forward differs from it.
|
|
#[test]
|
|
fn a_book_behind_our_stream_still_catches_our_loss() {
|
|
let (mut feed, token, now, stale_book) = feed_past_a_stale_book();
|
|
let right = feed.shell.symbols[&token].book.clone();
|
|
feed.shell
|
|
.symbols
|
|
.get_mut(&token)
|
|
.unwrap()
|
|
.book
|
|
.set(Side::Buy, 1, 1);
|
|
let effects = feed.on_frame(now, &stale_book);
|
|
|
|
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
|
|
assert_eq!(feed.rule.counts.losses, 1);
|
|
assert_eq!(feed.shell.symbols[&token].book, right);
|
|
}
|
|
|
|
/// Seen live: a resync's fresh book behind by its levels but carrying a
|
|
/// newer timestamp, so its hash is none of our changes'. Ours stands, and
|
|
/// nothing stale is written.
|
|
#[test]
|
|
fn a_book_behind_by_levels_with_a_newer_timestamp_is_not_a_loss() {
|
|
let (mut feed, token, now, stale_book) = feed_past_a_stale_book();
|
|
let message: BookMessage = serde_json::from_str(&stale_book).unwrap();
|
|
let levels = parse_book(&message.bids, &message.asks).unwrap();
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let book = fresh_book(&token, symbol, &levels);
|
|
let ours = symbol.book.clone();
|
|
let effects = feed.on_frame(now, &book);
|
|
|
|
assert!(snapshot_symbols(&effects).is_empty());
|
|
let counts = &feed.rule.counts;
|
|
assert_eq!(
|
|
(counts.books_behind, counts.subscription_gaps, counts.losses),
|
|
(1, 0, 0)
|
|
);
|
|
assert_eq!(feed.shell.symbols[&token].book, ours);
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Verified);
|
|
}
|
|
|
|
/// The level test must not mistake our wrong book for a newer one: we
|
|
/// changed a level, then lost the exchange's next change to it. The fresh
|
|
/// book plus our earlier change would equal ours; ours rewound never does.
|
|
#[test]
|
|
fn a_lost_change_at_a_level_we_had_changed_is_not_taken_for_behind() {
|
|
let (mut feed, token, now, _) = feed_past_a_stale_book();
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let last = symbol.rule.recent_changes.back().unwrap();
|
|
let mut exchange = symbol.book.clone();
|
|
let lost_quantity = last.quantity + 1_000_000;
|
|
assert_ne!(last.previous, lost_quantity);
|
|
exchange.set(last.side, last.price, lost_quantity);
|
|
let book = fresh_book(&token, symbol, &exchange);
|
|
let effects = feed.on_frame(now, &book);
|
|
|
|
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
|
|
assert_eq!(feed.rule.counts.books_behind, 0);
|
|
assert_eq!(feed.shell.symbols[&token].book, exchange);
|
|
}
|
|
|
|
/// Seen live: our subscribe hides changes made just before the opening
|
|
/// book reaches us, so the first fresh book differs at a level none of our
|
|
/// changes touched, before any change was verified.
|
|
#[test]
|
|
fn a_differing_book_before_any_verified_change_is_a_subscription_gap() {
|
|
let frames = fixture_frames();
|
|
let mut feed = feed_of(&fixture_markets());
|
|
replay(&mut feed, &frames[..1]);
|
|
let (token, symbol) = feed
|
|
.shell
|
|
.symbols
|
|
.iter_mut()
|
|
.find(|(_, symbol)| symbol.rule.state == State::Verified)
|
|
.unwrap();
|
|
let token = token.clone();
|
|
assert!(!symbol.rule.verified_since_book);
|
|
symbol.rule.asked = Some((0, false));
|
|
let mut exchange = symbol.book.clone();
|
|
exchange.set(Side::Buy, 1_000, 123_000_000);
|
|
let book = fresh_book(&token, symbol, &exchange);
|
|
let effects = feed.on_frame(frames[1].0, &book);
|
|
|
|
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
|
|
assert_eq!(
|
|
(feed.rule.counts.subscription_gaps, feed.rule.counts.losses),
|
|
(1, 0)
|
|
);
|
|
assert_eq!(feed.shell.symbols[&token].book, exchange);
|
|
}
|
|
|
|
/// Once a change has been verified since the opening book, our book was
|
|
/// right then: a fresh book that differs is a loss.
|
|
#[test]
|
|
fn a_differing_book_after_a_verified_change_is_a_loss() {
|
|
let frames = fixture_frames();
|
|
let mut feed = feed_of(&fixture_markets());
|
|
replay(&mut feed, &frames[..frames.len() / 2]);
|
|
let (token, symbol) = feed
|
|
.shell
|
|
.symbols
|
|
.iter_mut()
|
|
.find(|(_, symbol)| {
|
|
symbol.rule.state == State::Verified && symbol.rule.verified_since_book
|
|
})
|
|
.unwrap();
|
|
let token = token.clone();
|
|
symbol.rule.asked = Some((0, false));
|
|
let mut exchange = symbol.book.clone();
|
|
exchange.set(Side::Buy, 1_000, 123_000_000);
|
|
let book = fresh_book(&token, symbol, &exchange);
|
|
let effects = feed.on_frame(frames[frames.len() / 2].0, &book);
|
|
|
|
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
|
|
assert_eq!(
|
|
(feed.rule.counts.subscription_gaps, feed.rule.counts.losses),
|
|
(0, 1)
|
|
);
|
|
assert_eq!(feed.shell.symbols[&token].book, exchange);
|
|
}
|
|
|
|
#[test]
|
|
fn a_book_with_an_empty_side_parses() {
|
|
let ask = LevelMessage {
|
|
price: "0.999".to_string(),
|
|
size: "5".to_string(),
|
|
};
|
|
let book = parse_book(&[], &[ask]).unwrap();
|
|
assert!(book.bids.is_empty());
|
|
assert_eq!(book.asks, BTreeMap::from([(999_000, 5_000_000)]));
|
|
}
|
|
|
|
#[test]
|
|
fn a_book_failing_its_own_hash_goes_unverifiable_without_resync() {
|
|
let mut markets = fixture_markets();
|
|
for market in &mut markets {
|
|
market.min_order_size = "6".to_string();
|
|
}
|
|
let mut feed = feed_of(&markets);
|
|
let frames = fixture_frames();
|
|
let opening = feed.on_frame(frames[0].0, &frames[0].1);
|
|
assert_eq!(snapshot_symbols(&opening).len(), 40);
|
|
|
|
let effects = replay(&mut feed, &frames[1..]);
|
|
assert_eq!(resyncs(&effects), []);
|
|
assert!(
|
|
feed.shell
|
|
.symbols
|
|
.values()
|
|
.all(|symbol| symbol.rule.state == State::Unverifiable(0))
|
|
);
|
|
assert!(
|
|
effects
|
|
.iter()
|
|
.any(|effect| matches!(effect, Effect::Write(Output::Delta(_))))
|
|
);
|
|
assert_eq!(feed.rule.counts.verified, 0);
|
|
}
|
|
|
|
#[test]
|
|
fn frames_that_are_not_events_are_skipped() {
|
|
let mut feed = feed_of(&fixture_markets());
|
|
let at = fixture_frames()[0].0;
|
|
for text in [
|
|
"PONG",
|
|
"NO NEW ASSETS\n",
|
|
"not json",
|
|
"{\"event_type\":\"best_bid_ask\"}",
|
|
"[]",
|
|
] {
|
|
assert!(feed.on_frame(at, text).is_empty(), "{text:?}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_hash_parses_from_its_hex_and_nothing_else() {
|
|
let text = "b75cdbc9bc53ef345ed7bc6fe7872140992cb182";
|
|
assert_eq!(parse_hash(text).map(hex).as_deref(), Some(text));
|
|
assert_eq!(parse_hash("b75c"), None);
|
|
assert_eq!(parse_hash(&"zz".repeat(20)), None);
|
|
}
|
|
|
|
// --- two connections: first one wins ---
|
|
|
|
/// The exchange time from which both connections had delivered their
|
|
/// opening books, with a margin: before it, two replays start from books
|
|
/// taken at different moments.
|
|
fn settled(frames: &[Frame]) -> DateTime<Utc> {
|
|
frames[0].0 + TimeDelta::seconds(5)
|
|
}
|
|
|
|
#[test]
|
|
fn two_connections_write_what_one_alone_writes_once() {
|
|
let frames = two_connections();
|
|
let markets = fixture_markets();
|
|
let start = frames[0].0;
|
|
let one_only: Vec<Frame> = frames
|
|
.iter()
|
|
.filter(|(_, connection, _)| *connection == 0)
|
|
.cloned()
|
|
.collect();
|
|
let mut alone = feed_on(&markets, &[0], start);
|
|
let alone_effects = replay_both(&mut alone, &one_only);
|
|
let mut both = feed_on(&markets, &[0, 1], start);
|
|
let both_effects = replay_both(&mut both, &frames);
|
|
|
|
// One opening Snapshot per symbol, not one per connection.
|
|
let opening =
|
|
|effects: &[Effect]| snapshot_symbols(&effects[..effects.len().min(200)]).len();
|
|
assert_eq!(opening(&both_effects), 40);
|
|
assert_eq!(opening(&alone_effects), 40);
|
|
let since = settled(&frames);
|
|
let (alone_deltas, both_deltas) =
|
|
(deltas(&alone_effects, since), deltas(&both_effects, since));
|
|
assert!(alone_deltas.values().map(Vec::len).sum::<usize>() > 1_000);
|
|
assert_eq!(alone_deltas, both_deltas);
|
|
// Every change came twice: the second copy is a duplicate.
|
|
assert!(
|
|
both.rule.counts.duplicates > both.rule.counts.verified / 2,
|
|
"{:?}",
|
|
both.rule.counts
|
|
);
|
|
assert_eq!(both.rule.counts.losses, 0);
|
|
assert!(!both.rule.lags.is_empty() || both.rule.counts.duplicates == 0);
|
|
}
|
|
|
|
/// The first copy's receive time is the one written.
|
|
#[test]
|
|
fn each_delta_takes_the_first_receive_time() {
|
|
let frames = two_connections();
|
|
let markets = fixture_markets();
|
|
let start = frames[0].0;
|
|
let mut both = feed_on(&markets, &[0, 1], start);
|
|
let both_effects = replay_both(&mut both, &frames);
|
|
for connection in [0, 1] {
|
|
let one: Vec<Frame> = frames
|
|
.iter()
|
|
.filter(|(_, on, _)| *on == connection)
|
|
.cloned()
|
|
.collect();
|
|
let mut alone = feed_on(&markets, &[connection], start);
|
|
let alone_effects = replay_both(&mut alone, &one);
|
|
// Delta by delta, in order per symbol: never later than one alone.
|
|
let mut alone_stamps: BTreeMap<String, VecDeque<DateTime<Utc>>> = BTreeMap::new();
|
|
for (symbol, at) in delta_stamps(&alone_effects) {
|
|
alone_stamps.entry(symbol).or_default().push_back(at);
|
|
}
|
|
let mut earlier = 0;
|
|
for (symbol, at) in delta_stamps(&both_effects) {
|
|
if let Some(theirs) = alone_stamps.get_mut(&symbol).and_then(VecDeque::pop_front) {
|
|
earlier += usize::from(at < theirs);
|
|
assert!(
|
|
at <= theirs + TimeDelta::milliseconds(60),
|
|
"{symbol}: {at} after {theirs}"
|
|
);
|
|
}
|
|
}
|
|
assert!(earlier > 0, "connection {connection} was never slower");
|
|
}
|
|
}
|
|
|
|
/// One change frame of a single verified symbol, past the opening, as both
|
|
/// connections delivered it: its index on each.
|
|
fn a_change_on_both(
|
|
frames: &[Frame],
|
|
feed_at: impl Fn(usize) -> Feed,
|
|
) -> (usize, usize, String) {
|
|
let quarter = frames.len() / 4;
|
|
for (index, (_, connection, text)) in frames.iter().enumerate().skip(quarter) {
|
|
let tokens = changed_tokens(text);
|
|
if *connection != 0
|
|
|| tokens.is_empty()
|
|
|| !telling_loss(frames, index, |(_, _, text)| text)
|
|
{
|
|
continue;
|
|
}
|
|
let feed = feed_at(index);
|
|
let settled = |token: &String| {
|
|
feed.shell.symbols[token].rule.state == State::Verified
|
|
&& feed.shell.symbols[token].rule.held.is_empty()
|
|
};
|
|
if !tokens.iter().all(settled) {
|
|
continue;
|
|
}
|
|
if let Some(other) = frames
|
|
.iter()
|
|
.enumerate()
|
|
.skip(quarter)
|
|
.position(|(_, (_, on, theirs))| *on == 1 && theirs == text)
|
|
{
|
|
return (index, other + quarter, tokens[0].clone());
|
|
}
|
|
}
|
|
panic!("no change on both connections");
|
|
}
|
|
|
|
#[test]
|
|
fn a_change_lost_on_one_connection_is_rescued() {
|
|
let frames = two_connections();
|
|
let markets = fixture_markets();
|
|
let start = frames[0].0;
|
|
let feed_at = |index: usize| {
|
|
let mut feed = feed_on(&markets, &[0, 1], start);
|
|
replay_both(&mut feed, &frames[..index]);
|
|
feed
|
|
};
|
|
let (on_0, _, token) = a_change_on_both(&frames, feed_at);
|
|
let mut full = feed_on(&markets, &[0, 1], start);
|
|
let full_effects = replay_both(&mut full, &frames);
|
|
let mut lossy = feed_on(&markets, &[0, 1], start);
|
|
let mut lossy_effects = replay_both(&mut lossy, &frames[..on_0]);
|
|
lossy_effects.extend(replay_both(&mut lossy, &frames[on_0 + 1..]));
|
|
|
|
let since = settled(&frames);
|
|
assert_eq!(
|
|
deltas(&full_effects, since),
|
|
deltas(&lossy_effects, since),
|
|
"{token}"
|
|
);
|
|
// Every copy has come a minute after the recording: count them.
|
|
let after = frames.last().unwrap().0 + TimeDelta::minutes(1);
|
|
full.tick(after);
|
|
lossy.tick(after);
|
|
assert_eq!(lossy.rule.counts.losses, 0);
|
|
assert_eq!(
|
|
lossy.rule.counts.chains_broken,
|
|
full.rule.counts.chains_broken
|
|
);
|
|
assert!(
|
|
lossy.rule.counts.rescued > full.rule.counts.rescued,
|
|
"{:?}",
|
|
lossy.rule.counts
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_change_lost_on_both_breaks_the_chain_and_resyncs_on_the_other() {
|
|
let frames = two_connections();
|
|
let markets = fixture_markets();
|
|
let start = frames[0].0;
|
|
let feed_at = |index: usize| {
|
|
let mut feed = feed_on(&markets, &[0, 1], start);
|
|
replay_both(&mut feed, &frames[..index]);
|
|
feed
|
|
};
|
|
let (on_0, on_1, token) = a_change_on_both(&frames, feed_at);
|
|
let mut control = feed_on(&markets, &[0, 1], start);
|
|
replay_both(&mut control, &frames);
|
|
let mut feed = feed_on(&markets, &[0, 1], start);
|
|
let kept: Vec<Frame> = frames
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|(index, _)| *index != on_0 && *index != on_1)
|
|
.map(|(_, frame)| frame.clone())
|
|
.collect();
|
|
let chains = control.rule.counts.chains_broken;
|
|
let effects = replay_both(&mut feed, &kept);
|
|
assert!(
|
|
feed.rule.counts.chains_broken > chains,
|
|
"{:?}",
|
|
feed.rule.counts
|
|
);
|
|
let followed = match feed.shell.symbols[&token].rule.state {
|
|
State::Following(followed) => followed,
|
|
state => panic!("{token} is {state:?}"),
|
|
};
|
|
// The fresh book is asked from the connection not followed.
|
|
assert!(
|
|
resyncs(&effects).contains(&(1 - followed, token.as_str())),
|
|
"{:?}",
|
|
resyncs(&effects)
|
|
);
|
|
assert_eq!(
|
|
feed.shell.symbols[&token].rule.asked,
|
|
Some((1 - followed, false))
|
|
);
|
|
|
|
let book = fresh_book(
|
|
&token,
|
|
&feed.shell.symbols[&token],
|
|
&control.shell.symbols[&token].book,
|
|
);
|
|
let effects = feed.on_frame_from(1 - followed, frames.last().unwrap().0, &book);
|
|
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
|
|
assert_eq!(feed.rule.counts.losses, 1);
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Verified);
|
|
}
|
|
|
|
#[test]
|
|
fn a_slow_connection_changes_nothing_but_the_duplicates() {
|
|
let frames = two_connections();
|
|
let markets = fixture_markets();
|
|
let start = frames[0].0;
|
|
let mut delayed: Vec<Frame> = frames
|
|
.iter()
|
|
.map(|(at, connection, text)| {
|
|
let at = if *connection == 1 {
|
|
*at + TimeDelta::seconds(2)
|
|
} else {
|
|
*at
|
|
};
|
|
(at, *connection, text.clone())
|
|
})
|
|
.collect();
|
|
delayed.sort_by_key(|(at, _, _)| *at);
|
|
let mut slow = feed_on(&markets, &[0, 1], start);
|
|
let slow_effects = replay_both(&mut slow, &delayed);
|
|
// Connection 0 alone sets the pace: compare with its own replay.
|
|
let one_only: Vec<Frame> = frames
|
|
.iter()
|
|
.filter(|(_, connection, _)| *connection == 0)
|
|
.cloned()
|
|
.collect();
|
|
let mut alone = feed_on(&markets, &[0], start);
|
|
let alone_effects = replay_both(&mut alone, &one_only);
|
|
|
|
let since = settled(&frames);
|
|
assert_eq!(deltas(&slow_effects, since), deltas(&alone_effects, since));
|
|
assert_eq!(
|
|
slow.rule.counts.chains_broken, alone.rule.counts.chains_broken,
|
|
"{:?}",
|
|
slow.rule.counts
|
|
);
|
|
assert_eq!(slow.rule.counts.losses, 0);
|
|
assert!(slow.rule.counts.duplicates > 1_000);
|
|
// Its receive times are connection 0's, symbol by symbol (a held
|
|
// change waits longer while connection 1 might still bring the one
|
|
// before it, so symbols interleave differently).
|
|
let by_symbol = |effects: &[Effect]| {
|
|
let mut stamps: BTreeMap<String, Vec<DateTime<Utc>>> = BTreeMap::new();
|
|
for (symbol, at) in delta_stamps(effects) {
|
|
stamps.entry(symbol).or_default().push(at);
|
|
}
|
|
stamps
|
|
};
|
|
assert_eq!(by_symbol(&slow_effects), by_symbol(&alone_effects));
|
|
}
|
|
|
|
/// A verified symbol on connections 0 and 1, its book, a free bid price,
|
|
/// and an exchange time past its last change.
|
|
fn two_connection_symbol() -> (Feed, String, Book, i64, i64, DateTime<Utc>) {
|
|
let frames = two_connections();
|
|
let mut feed = feed_on(&fixture_markets(), &[0, 1], frames[0].0);
|
|
replay_both(&mut feed, &frames);
|
|
let (token, free) = verified_symbol(&feed);
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let at = frames.last().unwrap().0 + TimeDelta::seconds(1);
|
|
let millis = symbol.exchange_timestamp.timestamp_millis() + 1_000;
|
|
let book = symbol.book.clone();
|
|
(feed, token, book, free, millis, at)
|
|
}
|
|
|
|
#[test]
|
|
fn a_change_held_for_the_one_before_it_is_stamped_after_it() {
|
|
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let (x, after_x) = changes_frame(
|
|
&token,
|
|
symbol,
|
|
&book,
|
|
&[(Side::Buy, free, 1_000_000)],
|
|
millis,
|
|
);
|
|
let (y, after_y) = changes_frame(
|
|
&token,
|
|
symbol,
|
|
&after_x,
|
|
&[(Side::Buy, free + 1_000, 2_000_000)],
|
|
millis + 1,
|
|
);
|
|
// Connection 0 missed X: its Y waits.
|
|
assert!(feed.on_frame_from(0, at, &y).is_empty());
|
|
assert!(feed.rule.waiting.contains(&token));
|
|
// Connection 1 brings X, 10 ms later: X, then Y.
|
|
let later = at + TimeDelta::milliseconds(10);
|
|
let effects = feed.on_frame_from(1, later, &x);
|
|
assert_eq!(
|
|
delta_stamps(&effects),
|
|
[(token.clone(), later), (token.clone(), later)]
|
|
);
|
|
assert_eq!(feed.shell.symbols[&token].book, after_y);
|
|
// Connection 1's own Y comes after: a duplicate.
|
|
let duplicates = feed.rule.counts.duplicates;
|
|
assert!(
|
|
feed.on_frame_from(1, later + TimeDelta::milliseconds(1), &y)
|
|
.is_empty()
|
|
);
|
|
assert_eq!(feed.rule.counts.duplicates, duplicates + 1);
|
|
// Once every copy could have come: X came once, rescued; Y twice.
|
|
let rescued = feed.rule.counts.rescued;
|
|
feed.tick(at + TimeDelta::minutes(2));
|
|
assert_eq!(feed.rule.counts.rescued, rescued + 1);
|
|
}
|
|
|
|
/// A burst on a connection that missed the change before it: the held
|
|
/// changes stop at the cap, the chain breaks, and it is all quick.
|
|
#[test]
|
|
fn a_burst_behind_a_missing_change_is_capped() {
|
|
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let (_, mut after) = changes_frame(
|
|
&token,
|
|
symbol,
|
|
&book,
|
|
&[(Side::Buy, free, 1_000_000)],
|
|
millis,
|
|
);
|
|
let mut burst = Vec::new();
|
|
for step in 1..=5_000 {
|
|
let (frame, next) = changes_frame(
|
|
&token,
|
|
symbol,
|
|
&after,
|
|
&[(Side::Buy, free + 1_000, step * 1_000)],
|
|
millis + step,
|
|
);
|
|
burst.push(frame);
|
|
after = next;
|
|
}
|
|
let started = std::time::Instant::now();
|
|
let mut most = 0;
|
|
for (step, frame) in burst.iter().enumerate() {
|
|
feed.on_frame_from(0, at + TimeDelta::microseconds(step as i64), frame);
|
|
let held = feed.shell.symbols[&token]
|
|
.rule
|
|
.held
|
|
.get(&0)
|
|
.map_or(0, Vec::len);
|
|
most = most.max(held);
|
|
}
|
|
assert!(most <= MAX_HELD + 1, "{most} held");
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Following(0));
|
|
assert!(
|
|
started.elapsed() < std::time::Duration::from_secs(10),
|
|
"{:?}",
|
|
started.elapsed()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_batch_is_written_once_it_fits() {
|
|
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let changes = [
|
|
(Side::Buy, free, 1_000_000),
|
|
(Side::Buy, free + 1_000, 1_000_000),
|
|
];
|
|
let (whole, after) = changes_frame(&token, symbol, &book, &changes, millis);
|
|
// The exchange sends a batch one change per frame, each with the
|
|
// batch's hash.
|
|
let value: serde_json::Value = serde_json::from_str(&whole).unwrap();
|
|
let part = |index: usize| {
|
|
let mut value = value.clone();
|
|
value["price_changes"] = serde_json::json!([value["price_changes"][index]]);
|
|
value.to_string()
|
|
};
|
|
assert!(feed.on_frame_from(0, at, &part(0)).is_empty());
|
|
let effects = feed.on_frame_from(0, at + TimeDelta::milliseconds(1), &part(1));
|
|
assert_eq!(delta_stamps(&effects).len(), 2);
|
|
assert_eq!(feed.shell.symbols[&token].book, after);
|
|
assert!(!feed.rule.waiting.contains(&token));
|
|
}
|
|
|
|
#[test]
|
|
fn a_change_older_than_the_book_is_dropped() {
|
|
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let (old, _) = changes_frame(
|
|
&token,
|
|
symbol,
|
|
&book,
|
|
&[(Side::Buy, free, 1_000_000)],
|
|
millis - 60_000,
|
|
);
|
|
let duplicates = feed.rule.counts.duplicates;
|
|
assert!(feed.on_frame_from(1, at, &old).is_empty());
|
|
assert_eq!(feed.rule.counts.duplicates, duplicates + 1);
|
|
assert!(!feed.rule.waiting.contains(&token));
|
|
}
|
|
|
|
#[test]
|
|
fn held_changes_past_their_ceiling_break_the_chain_when_the_book_stood_still() {
|
|
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let (_, after_x) = changes_frame(
|
|
&token,
|
|
symbol,
|
|
&book,
|
|
&[(Side::Buy, free, 1_000_000)],
|
|
millis,
|
|
);
|
|
let (y, after_y) = changes_frame(
|
|
&token,
|
|
symbol,
|
|
&after_x,
|
|
&[(Side::Buy, free + 1_000, 2_000_000)],
|
|
millis + 1,
|
|
);
|
|
// Both missed X and hold the same Y: nobody can bring X, 50 ms.
|
|
feed.on_frame_from(0, at, &y);
|
|
feed.on_frame_from(1, at, &y);
|
|
assert!(feed.idle(at + TimeDelta::milliseconds(40)).is_empty());
|
|
let effects = feed.idle(at + TimeDelta::milliseconds(60));
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Following(0));
|
|
assert_eq!(resyncs(&effects), [(1, token.as_str())]);
|
|
// Y is written unchecked, on the book that lacks X.
|
|
assert_eq!(delta_stamps(&effects), [(token.clone(), at)]);
|
|
assert_ne!(feed.shell.symbols[&token].book, after_y);
|
|
// Connection 1's changes are dropped while 0 is followed.
|
|
let (z, _) = changes_frame(
|
|
&token,
|
|
&feed.shell.symbols[&token],
|
|
&after_y,
|
|
&[(Side::Buy, free, 3_000_000)],
|
|
millis + 2,
|
|
);
|
|
assert!(
|
|
feed.on_frame_from(1, at + TimeDelta::milliseconds(70), &z)
|
|
.is_empty()
|
|
);
|
|
assert_eq!(
|
|
feed.on_frame_from(0, at + TimeDelta::milliseconds(71), &z)
|
|
.len(),
|
|
1
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_connection_behind_has_its_held_changes_dropped_at_the_ceiling() {
|
|
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let (x, after_x) = changes_frame(
|
|
&token,
|
|
symbol,
|
|
&book,
|
|
&[(Side::Buy, free, 1_000_000)],
|
|
millis,
|
|
);
|
|
// Connection 0 holds a change that will never fit (made on another
|
|
// book), while connection 1 moves the book on.
|
|
let mut elsewhere = after_x.clone();
|
|
elsewhere.set(Side::Buy, free + 1_000, 1);
|
|
let (stray, _) = changes_frame(
|
|
&token,
|
|
symbol,
|
|
&elsewhere,
|
|
&[(Side::Sell, 999_000, 5_000_000)],
|
|
millis,
|
|
);
|
|
feed.on_frame_from(0, at, &stray);
|
|
let effects = feed.on_frame_from(1, at + TimeDelta::milliseconds(5), &x);
|
|
assert_eq!(delta_stamps(&effects).len(), 1);
|
|
// Connection 1 could still deliver: a second, then dropped.
|
|
assert!(feed.idle(at + TimeDelta::milliseconds(900)).is_empty());
|
|
let effects = feed.idle(at + TimeDelta::milliseconds(1_100));
|
|
assert!(effects.is_empty());
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Verified);
|
|
assert_eq!(feed.rule.counts.held_dropped, 1);
|
|
assert!(!feed.rule.waiting.contains(&token));
|
|
}
|
|
|
|
#[test]
|
|
fn a_book_not_asked_for_changes_nothing() {
|
|
let (mut feed, token, book, _, _, at) = two_connection_symbol();
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let mut other = book.clone();
|
|
other.set(Side::Buy, 1_000, 5_000_000);
|
|
let unasked = fresh_book(&token, symbol, &other);
|
|
assert!(feed.on_frame_from(1, at, &unasked).is_empty());
|
|
assert_eq!(feed.shell.symbols[&token].book, book);
|
|
}
|
|
|
|
#[test]
|
|
fn one_connection_down_changes_nothing_and_both_down_await_a_book() {
|
|
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
|
|
assert!(feed.on_received(Received::Down(1)).is_empty());
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Verified);
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let (x, after_x) = changes_frame(
|
|
&token,
|
|
symbol,
|
|
&book,
|
|
&[(Side::Buy, free, 1_000_000)],
|
|
millis,
|
|
);
|
|
// Alone, a change that doesn't fit waits 50 ms, not a second.
|
|
assert_eq!(feed.on_frame_from(0, at, &x).len(), 1);
|
|
assert_eq!(feed.shell.symbols[&token].book, after_x);
|
|
|
|
feed.on_received(Received::Down(0));
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::AwaitingBook);
|
|
// The reconnect's opening book is compared with ours: equal, nothing.
|
|
feed.on_received(Received::Up(0));
|
|
let same = fresh_book(&token, &feed.shell.symbols[&token], &after_x);
|
|
assert!(snapshot_symbols(&feed.on_frame_from(0, at, &same)).is_empty());
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Verified);
|
|
// Differing, a loss.
|
|
feed.on_received(Received::Down(0));
|
|
feed.on_received(Received::Down(1));
|
|
let mut other = after_x.clone();
|
|
other.set(Side::Buy, free, 7_000_000);
|
|
let differing = fresh_book(&token, &feed.shell.symbols[&token], &other);
|
|
let losses = feed.rule.counts.losses;
|
|
assert_eq!(
|
|
snapshot_symbols(&feed.on_frame_from(1, at, &differing)),
|
|
[token.as_str()]
|
|
);
|
|
assert_eq!(feed.rule.counts.losses, losses + 1);
|
|
}
|
|
|
|
/// A frame setting `token`'s bid at `price` to `quantity`, `millis` its
|
|
/// exchange time, from the book as it is now.
|
|
fn change(feed: &Feed, token: &str, millis: i64, price: i64, quantity: i64) -> String {
|
|
let symbol = &feed.shell.symbols[token];
|
|
let changes = [(Side::Buy, price, quantity)];
|
|
changes_frame(token, symbol, &symbol.book, &changes, millis).0
|
|
}
|
|
|
|
fn resyncs_of<'a>(effects: &'a [Effect], token: &str) -> Vec<(usize, &'a str)> {
|
|
resyncs(effects)
|
|
.into_iter()
|
|
.filter(|&(_, theirs)| theirs == token)
|
|
.collect()
|
|
}
|
|
|
|
/// From the next wall-clock minute after `at`, which the feed has begun:
|
|
/// the following minute is counted whole.
|
|
fn begin_minute(feed: &mut Feed, at: DateTime<Utc>) -> DateTime<Utc> {
|
|
let minute = next_minute(at);
|
|
feed.idle(minute);
|
|
minute
|
|
}
|
|
|
|
/// `count` changes of `token`, from 10 s after `start`, on the
|
|
/// connections `on` only.
|
|
fn changes(
|
|
feed: &mut Feed,
|
|
token: &str,
|
|
on: &[usize],
|
|
start: DateTime<Utc>,
|
|
price: i64,
|
|
count: i64,
|
|
) {
|
|
let millis = feed.shell.symbols[token]
|
|
.exchange_timestamp
|
|
.timestamp_millis()
|
|
+ 1_000;
|
|
for step in 0..count {
|
|
let at = start + TimeDelta::seconds(10 + step);
|
|
let frame = change(feed, token, millis + step, price, (step + 1) * 1_000_000);
|
|
for &connection in on {
|
|
feed.on_frame_from(connection, at, &frame);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn quiet_reports(effects: &[Effect]) -> Vec<(&str, usize)> {
|
|
effects
|
|
.iter()
|
|
.filter_map(|effect| match effect {
|
|
Effect::Report(Report::Quiet { market, connection }) => {
|
|
Some((market.as_str(), *connection))
|
|
}
|
|
_ => None,
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
fn a_subscription_quiet_on_one_connection_is_resubscribed_then_reported() {
|
|
let (mut feed, token, _, free, _, at) = two_connection_symbol();
|
|
let market = feed.shell.symbols[&token].market.clone();
|
|
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
|
|
begin_minute(&mut feed, at);
|
|
let (losses, gaps) = (feed.rule.counts.losses, feed.rule.counts.subscription_gaps);
|
|
|
|
// Two changes against none: too few to say.
|
|
changes(&mut feed, &token, &[0], minute(0), free, 2);
|
|
let effects = feed.idle(minute(1));
|
|
assert!(resyncs_of(&effects, &token).is_empty());
|
|
assert_eq!(feed.shell.quiet, 0);
|
|
|
|
// Three: quiet on connection 1, resubscribed there.
|
|
changes(&mut feed, &token, &[0], minute(1), free, 3);
|
|
let effects = feed.idle(minute(2));
|
|
assert_eq!(resyncs_of(&effects, &token), [(1, token.as_str())]);
|
|
assert!(quiet_reports(&effects).is_empty());
|
|
assert_eq!(feed.shell.quiet, 1);
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Verified);
|
|
// The book the resubscribe brings back is not asked for: ignored.
|
|
let book = feed.shell.symbols[&token].book.clone();
|
|
let back = fresh_book(&token, &feed.shell.symbols[&token], &book);
|
|
let effects = feed.on_frame_from(1, minute(2) + TimeDelta::seconds(1), &back);
|
|
assert!(snapshot_symbols(&effects).is_empty());
|
|
|
|
// Still quiet the minute after: reported for a move, not resubscribed.
|
|
changes(&mut feed, &token, &[0], minute(2), free, 3);
|
|
let effects = feed.idle(minute(3));
|
|
assert!(resyncs_of(&effects, &token).is_empty());
|
|
assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]);
|
|
assert_eq!(
|
|
(feed.rule.counts.losses, feed.rule.counts.subscription_gaps),
|
|
(losses, gaps)
|
|
);
|
|
}
|
|
|
|
/// Both tokens of a market go quiet together: one report for the market.
|
|
#[test]
|
|
fn a_market_quiet_on_both_its_tokens_is_reported_once() {
|
|
let (mut feed, token, _, free, _, at) = two_connection_symbol();
|
|
let market = feed.shell.symbols[&token].market.clone();
|
|
let tokens = feed.shell.markets[&market].names.clone();
|
|
assert!(
|
|
tokens
|
|
.iter()
|
|
.all(|token| feed.shell.symbols[token].rule.state == State::Verified)
|
|
);
|
|
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
|
|
begin_minute(&mut feed, at);
|
|
for n in 0..2 {
|
|
for token in &tokens {
|
|
changes(&mut feed, token, &[0], minute(n), free, 3);
|
|
}
|
|
let effects = feed.idle(minute(n + 1));
|
|
if n == 0 {
|
|
assert_eq!(feed.shell.quiet, 2);
|
|
} else {
|
|
assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_subscription_delivering_or_not_there_the_whole_minute_is_not_quiet() {
|
|
let (mut feed, token, _, free, _, at) = two_connection_symbol();
|
|
let markets = fixture_markets();
|
|
let market = markets
|
|
.iter()
|
|
.find(|market| market.tokens.contains(&token))
|
|
.unwrap();
|
|
// Connection 5 is up from before the minutes below.
|
|
feed.on_received(Received::Up(5));
|
|
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
|
|
begin_minute(&mut feed, at);
|
|
|
|
// Both deliver: neither is quiet; the symbols that didn't change
|
|
// aren't either.
|
|
changes(&mut feed, &token, &[0, 1], minute(0), free, 3);
|
|
let effects = feed.idle(minute(1));
|
|
assert!(resyncs(&effects).iter().all(|&(_, theirs)| theirs != token));
|
|
|
|
// Connection 1 dropped and came back within the minute: not judged
|
|
// for the changes it missed.
|
|
changes(&mut feed, &token, &[0], minute(1), free, 3);
|
|
feed.on_received(Received::Down(1));
|
|
feed.on_received(Received::Up(1));
|
|
let effects = feed.idle(minute(2));
|
|
assert!(resyncs_of(&effects, &token).is_empty());
|
|
|
|
// Connection 5, subscribed in the middle of the minute and delivering
|
|
// nothing yet: not judged either.
|
|
feed.move_market(market, 1, 5, minute(2) + TimeDelta::seconds(5));
|
|
changes(&mut feed, &token, &[0, 1], minute(2), free, 3);
|
|
let effects = feed.idle(minute(3));
|
|
assert!(resyncs_of(&effects, &token).is_empty());
|
|
assert_eq!(feed.shell.quiet, 0);
|
|
}
|
|
|
|
/// A verified symbol on connections 0 and 1, changed once and then still
|
|
/// for 30 s: the feed, its token, its book before the change and now, a
|
|
/// free bid price, and the time 30 s after.
|
|
fn still_symbol() -> (Feed, String, Book, Book, i64, DateTime<Utc>) {
|
|
let (mut feed, token, before, free, millis, at) = two_connection_symbol();
|
|
let x = change(&feed, &token, millis, free, 1_000_000);
|
|
feed.on_frame_from(0, at, &x);
|
|
feed.on_frame_from(1, at, &x);
|
|
let ours = feed.shell.symbols[&token].book.clone();
|
|
let later = at + TimeDelta::seconds(30);
|
|
// The minute's Snapshots, out of the way.
|
|
feed.idle(later);
|
|
(feed, token, before, ours, free, later)
|
|
}
|
|
|
|
/// Connection 1 drops and reconnects; its opening book of `token` holds
|
|
/// `book`.
|
|
fn reconnect_with(feed: &mut Feed, token: &str, book: &Book, at: DateTime<Utc>) -> Vec<Effect> {
|
|
feed.on_received(Received::Down(1));
|
|
feed.on_received(Received::Up(1));
|
|
let opening = fresh_book(token, &feed.shell.symbols[token], book);
|
|
feed.on_frame_from(1, at, &opening)
|
|
}
|
|
|
|
#[test]
|
|
fn a_reconnect_book_of_a_still_symbol_equal_to_ours_or_behind_changes_nothing() {
|
|
let (mut feed, token, before, ours, _, later) = still_symbol();
|
|
let losses = feed.rule.counts.losses;
|
|
let effects = reconnect_with(&mut feed, &token, &ours, later);
|
|
assert!(snapshot_symbols(&effects).is_empty());
|
|
assert_eq!(feed.rule.counts.reconnect_books, 1);
|
|
// The server's opening books lag: a state we passed is no loss.
|
|
let behind = feed.rule.counts.books_behind;
|
|
let effects = reconnect_with(&mut feed, &token, &before, later + TimeDelta::seconds(1));
|
|
assert!(snapshot_symbols(&effects).is_empty());
|
|
assert_eq!(feed.rule.counts.reconnect_books, 2);
|
|
assert_eq!(feed.rule.counts.books_behind, behind + 1);
|
|
assert_eq!(feed.shell.symbols[&token].book, ours);
|
|
// It lags further back than the changes we remember: no newer than
|
|
// our last change, it still holds nothing we missed.
|
|
let symbol = feed.shell.symbols.get_mut(&token).unwrap();
|
|
symbol.rule.recent_changes.clear();
|
|
let old = symbol.exchange_timestamp - TimeDelta::minutes(10);
|
|
feed.on_received(Received::Down(1));
|
|
feed.on_received(Received::Up(1));
|
|
let lagging = book_frame(&token, &feed.shell.symbols[&token], &before, old);
|
|
let effects = feed.on_frame_from(1, later + TimeDelta::seconds(2), &lagging);
|
|
assert!(snapshot_symbols(&effects).is_empty());
|
|
assert_eq!(feed.rule.counts.books_behind, behind + 2);
|
|
assert_eq!(feed.shell.symbols[&token].book, ours);
|
|
assert_eq!(feed.rule.counts.losses, losses);
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Verified);
|
|
}
|
|
|
|
/// A reconnect's book that fails its own hash proves nothing: the symbol
|
|
/// stays verified, with its book and its hash inputs.
|
|
#[test]
|
|
fn a_reconnect_book_failing_its_own_hash_changes_nothing() {
|
|
let (mut feed, token, _, ours, free, later) = still_symbol();
|
|
let mut other = ours.clone();
|
|
other.set(Side::Buy, free, 7_000_000);
|
|
let frame = fresh_book(&token, &feed.shell.symbols[&token], &other);
|
|
let mut frame: serde_json::Value = serde_json::from_str(&frame).unwrap();
|
|
frame["hash"] = "0".repeat(40).into();
|
|
frame["last_trade_price"] = "0.123".into();
|
|
let last_trade_price = feed.shell.symbols[&token].rule.last_trade_price.clone();
|
|
feed.on_received(Received::Down(1));
|
|
feed.on_received(Received::Up(1));
|
|
let effects = feed.on_frame_from(1, later, &frame.to_string());
|
|
assert!(effects.is_empty());
|
|
let symbol = &feed.shell.symbols[&token];
|
|
assert_eq!(symbol.rule.state, State::Verified);
|
|
assert_eq!(symbol.book, ours);
|
|
assert_eq!(symbol.rule.last_trade_price, last_trade_price);
|
|
}
|
|
|
|
/// Only a reconnect's book of a symbol with nothing asked is judged: not
|
|
/// the opening book of a subscription made since the connection came up
|
|
/// (a move's), nor any while a fresh book is asked.
|
|
#[test]
|
|
fn a_book_that_is_no_reconnects_or_comes_while_one_is_asked_is_not_judged() {
|
|
let (mut feed, token, _, ours, free, later) = still_symbol();
|
|
let markets = fixture_markets();
|
|
let market = markets
|
|
.iter()
|
|
.find(|market| market.tokens.contains(&token))
|
|
.unwrap();
|
|
let mut other = ours.clone();
|
|
other.set(Side::Buy, free, 7_000_000);
|
|
feed.on_received(Received::Up(5));
|
|
feed.move_market(market, 0, 5, later);
|
|
let opening = fresh_book(&token, &feed.shell.symbols[&token], &other);
|
|
let effects = feed.on_frame_from(5, later + TimeDelta::seconds(1), &opening);
|
|
assert!(snapshot_symbols(&effects).is_empty());
|
|
assert_eq!(feed.rule.counts.reconnect_books, 0);
|
|
|
|
feed.shell.symbols.get_mut(&token).unwrap().rule.asked = Some((5, false));
|
|
let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(2));
|
|
assert!(snapshot_symbols(&effects).is_empty());
|
|
assert_eq!(feed.rule.counts.reconnect_books, 0);
|
|
assert_eq!(feed.shell.symbols[&token].book, ours);
|
|
}
|
|
|
|
#[test]
|
|
fn a_reconnect_book_of_a_still_symbol_that_differs_is_a_loss_and_resubscribes_the_other() {
|
|
let (mut feed, token, _, ours, free, later) = still_symbol();
|
|
let (losses, gaps) = (feed.rule.counts.losses, feed.rule.counts.subscription_gaps);
|
|
// Even before any change of it was verified: a loss, not the gap of
|
|
// a subscribe, which is long past.
|
|
feed.shell
|
|
.symbols
|
|
.get_mut(&token)
|
|
.unwrap()
|
|
.rule
|
|
.verified_since_book = false;
|
|
let mut other = ours.clone();
|
|
other.set(Side::Buy, free, 7_000_000);
|
|
let effects = reconnect_with(&mut feed, &token, &other, later);
|
|
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
|
|
assert_eq!(resyncs_of(&effects, &token), [(0, token.as_str())]);
|
|
assert_eq!(feed.rule.counts.losses, losses + 1);
|
|
assert_eq!(feed.rule.counts.subscription_gaps, gaps);
|
|
assert_eq!(feed.rule.counts.reconnect_losses, 1);
|
|
assert_eq!(feed.shell.symbols[&token].book, other);
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Verified);
|
|
}
|
|
|
|
#[test]
|
|
fn a_reconnect_book_of_a_symbol_changed_lately_is_ignored() {
|
|
let (mut feed, token, _, ours, free, later) = still_symbol();
|
|
let millis = feed.shell.symbols[&token]
|
|
.exchange_timestamp
|
|
.timestamp_millis()
|
|
+ 1_000;
|
|
let x = change(&feed, &token, millis, free, 2_000_000);
|
|
feed.on_frame_from(0, later, &x);
|
|
let mut other = ours.clone();
|
|
other.set(Side::Buy, free, 7_000_000);
|
|
let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(1));
|
|
assert!(snapshot_symbols(&effects).is_empty());
|
|
assert_eq!(feed.rule.counts.reconnect_books, 0);
|
|
// And the next book of it from that connection is no reconnect's.
|
|
feed.idle(later + TimeDelta::seconds(30));
|
|
let opening = fresh_book(&token, &feed.shell.symbols[&token], &other);
|
|
let effects = feed.on_frame_from(1, later + TimeDelta::seconds(31), &opening);
|
|
assert!(snapshot_symbols(&effects).is_empty());
|
|
assert_eq!(feed.rule.counts.reconnect_books, 0);
|
|
}
|
|
|
|
#[test]
|
|
fn the_followed_connection_down_switches_with_a_snapshot() {
|
|
let (mut feed, token, book, _, _, at) = two_connection_symbol();
|
|
feed.shell.symbols.get_mut(&token).unwrap().rule.state = State::Following(0);
|
|
let effects = feed.on_received(Received::Down(0));
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Following(1));
|
|
assert!(resyncs(&effects).contains(&(1, token.as_str())));
|
|
// Its fresh book is written even if it equals ours.
|
|
let fresh = fresh_book(&token, &feed.shell.symbols[&token], &book);
|
|
let effects = feed.on_frame_from(1, at, &fresh);
|
|
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
|
|
assert!(feed.rule.counts.switches >= 1);
|
|
assert_eq!(feed.shell.symbols[&token].rule.state, State::Verified);
|
|
}
|
|
|
|
#[test]
|
|
fn a_move_subscribes_first_and_leaves_once_the_new_connection_delivers() {
|
|
let (mut feed, token, book, free, millis, at) = two_connection_symbol();
|
|
let markets = fixture_markets();
|
|
let market = markets
|
|
.iter()
|
|
.find(|market| market.tokens.contains(&token))
|
|
.unwrap();
|
|
let effects = feed.move_market(market, 0, 5, at);
|
|
assert!(matches!(
|
|
effects.as_slice(),
|
|
[Effect::Send(5, Command::Subscribe(tokens))] if *tokens == market.tokens
|
|
));
|
|
// Both still deliver until connection 5's book comes.
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let (x, after_x) = changes_frame(
|
|
&token,
|
|
symbol,
|
|
&book,
|
|
&[(Side::Buy, free, 1_000_000)],
|
|
millis,
|
|
);
|
|
assert_eq!(feed.on_frame_from(0, at, &x).len(), 1);
|
|
let opening = fresh_book(&token, &feed.shell.symbols[&token], &after_x);
|
|
let effects = feed.on_frame_from(5, at, &opening);
|
|
assert!(matches!(
|
|
effects.as_slice(),
|
|
[Effect::Send(0, Command::Unsubscribe(tokens))] if *tokens == market.tokens
|
|
));
|
|
assert!(snapshot_symbols(&effects).is_empty());
|
|
let mut on = feed.shell.symbols[&token].connections.clone();
|
|
on.sort();
|
|
assert_eq!(on, [1, 5]);
|
|
}
|
|
|
|
/// The connection left can't answer a fresh book any more: it is asked
|
|
/// from another.
|
|
#[test]
|
|
fn a_move_ending_while_a_fresh_book_is_asked_from_the_left_one_asks_again() {
|
|
let (mut feed, token, book, _, _, at) = two_connection_symbol();
|
|
let markets = fixture_markets();
|
|
let market = markets
|
|
.iter()
|
|
.find(|market| market.tokens.contains(&token))
|
|
.unwrap();
|
|
let symbol = feed.shell.symbols.get_mut(&token).unwrap();
|
|
symbol.rule.state = State::Following(1);
|
|
symbol.rule.asked = Some((0, false));
|
|
feed.move_market(market, 0, 5, at);
|
|
let opening = fresh_book(&token, &feed.shell.symbols[&token], &book);
|
|
let effects = feed.on_frame_from(5, at, &opening);
|
|
assert!(
|
|
resyncs(&effects).contains(&(1, token.as_str())),
|
|
"{:?}",
|
|
resyncs(&effects)
|
|
);
|
|
assert_eq!(feed.shell.symbols[&token].rule.asked, Some((1, false)));
|
|
}
|
|
|
|
/// Replaying the output by time keeps its order: a change held across the
|
|
/// minute is not stamped before the minute's Snapshot.
|
|
#[test]
|
|
fn a_change_held_across_the_minute_is_stamped_after_its_snapshot() {
|
|
let (mut feed, token, book, free, millis, _) = two_connection_symbol();
|
|
let minute = next_minute(feed.shell.last_received_at.unwrap());
|
|
let symbol = &feed.shell.symbols[&token];
|
|
let changes = [
|
|
(Side::Buy, free, 1_000_000),
|
|
(Side::Buy, free + 1_000, 1_000_000),
|
|
];
|
|
let (whole, _) = changes_frame(&token, symbol, &book, &changes, millis);
|
|
let value: serde_json::Value = serde_json::from_str(&whole).unwrap();
|
|
let part = |index: usize| {
|
|
let mut value = value.clone();
|
|
value["price_changes"] = serde_json::json!([value["price_changes"][index]]);
|
|
value.to_string()
|
|
};
|
|
// A batch's first change waits for its second across the minute;
|
|
// another event comes after it, so the minute's Snapshot is stamped
|
|
// later than that first change.
|
|
feed.on_frame_from(0, minute - TimeDelta::milliseconds(2), &part(0));
|
|
let other = serde_json::json!({"event_type": "price_change", "market": "0xother",
|
|
"timestamp": "1", "price_changes": []});
|
|
feed.on_frame_from(0, minute - TimeDelta::milliseconds(1), &other.to_string());
|
|
let effects = feed.on_frame_from(0, minute + TimeDelta::milliseconds(1), &part(1));
|
|
let snapshot_at = effects
|
|
.iter()
|
|
.find_map(|effect| match effect {
|
|
Effect::Write(Output::Snapshot(snapshot)) if snapshot.symbol == token => {
|
|
Some(snapshot.received_at)
|
|
}
|
|
_ => None,
|
|
})
|
|
.expect("the minute's Snapshot");
|
|
let stamps: Vec<_> = delta_stamps(&effects)
|
|
.into_iter()
|
|
.filter(|(symbol, _)| *symbol == token)
|
|
.collect();
|
|
assert_eq!(stamps.len(), 2);
|
|
for (_, at) in stamps {
|
|
assert!(at >= snapshot_at, "{at} before {snapshot_at}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_market_on_two_connections_reports_its_load_on_one() {
|
|
let minute = "2026-11-10T16:00:00Z".parse::<DateTime<Utc>>().unwrap();
|
|
let market = Market {
|
|
condition_id: "0xm".to_string(),
|
|
tokens: vec!["0xma".to_string(), "0xmb".to_string()],
|
|
min_order_size: "5".to_string(),
|
|
neg_risk: false,
|
|
};
|
|
let mut feed = Feed::new("polymarket", 0, Chain::default());
|
|
feed.add_market(&market, &[0, 1], minute);
|
|
let change = serde_json::json!({"event_type": "price_change", "market": "0xm",
|
|
"timestamp": "1", "price_changes": []})
|
|
.to_string();
|
|
feed.on_frame_from(0, minute + TimeDelta::seconds(10), &change);
|
|
feed.on_frame_from(1, minute + TimeDelta::seconds(10), &change);
|
|
let effects = feed.tick(minute + TimeDelta::seconds(60));
|
|
let Some(Effect::Report(Report::Loads(loads))) = effects.first() else {
|
|
panic!("no report");
|
|
};
|
|
assert_eq!(loads["0xm"], change.len() as u64 / 60);
|
|
}
|
|
|
|
// --- markets, loads, the output ---
|
|
|
|
/// A market's opening books, as a subscribe brings them.
|
|
fn opening_books(frames: &[(DateTime<Utc>, String)], market: &Market) -> String {
|
|
let opening: Vec<serde_json::Value> = serde_json::from_str(&frames[0].1).unwrap();
|
|
let books: Vec<_> = opening
|
|
.iter()
|
|
.filter(|book| {
|
|
market
|
|
.tokens
|
|
.iter()
|
|
.any(|token| book["asset_id"] == **token)
|
|
})
|
|
.collect();
|
|
serde_json::to_string(&books).unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn a_market_added_mid_run_is_subscribed_and_its_first_book_written() {
|
|
let frames = fixture_frames();
|
|
let markets = fixture_markets();
|
|
let mut feed = feed_of(&markets[1..]);
|
|
replay(&mut feed, &frames[1..100]);
|
|
|
|
let effects = feed.add_market(&markets[0], &[0], frames[100].0);
|
|
assert!(matches!(
|
|
effects.as_slice(),
|
|
[Effect::Send(0, Command::Subscribe(tokens))] if *tokens == markets[0].tokens
|
|
));
|
|
let effects = feed.on_frame(frames[100].0, &opening_books(&frames, &markets[0]));
|
|
let mut written = snapshot_symbols(&effects);
|
|
written.sort();
|
|
let mut tokens: Vec<&str> = markets[0].tokens.iter().map(String::as_str).collect();
|
|
tokens.sort();
|
|
assert_eq!(written, tokens);
|
|
assert!(
|
|
markets[0]
|
|
.tokens
|
|
.iter()
|
|
.all(|token| feed.shell.symbols[token].rule.state == State::Verified)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_removed_market_is_unsubscribed_and_written_no_more() {
|
|
let frames = fixture_frames();
|
|
let markets = fixture_markets();
|
|
let mut feed = feed_of(&markets);
|
|
replay(&mut feed, &frames[..1]);
|
|
|
|
let removed = &markets[0].tokens;
|
|
let effects = feed.remove_market(&markets[0]);
|
|
assert!(matches!(
|
|
effects.as_slice(),
|
|
[Effect::Send(0, Command::Unsubscribe(tokens))] if tokens == removed
|
|
));
|
|
// The rest of the recording crosses two minutes: no Delta, Snapshot or
|
|
// resync names a removed token.
|
|
let effects = replay(&mut feed, &frames[1..]);
|
|
let mentions = |effect: &Effect| match effect {
|
|
Effect::Write(Output::Snapshot(snapshot)) => removed.contains(&snapshot.symbol),
|
|
Effect::Write(Output::Delta(delta)) => removed.contains(&delta.symbol),
|
|
Effect::Send(_, Command::Resubscribe(token)) => removed.contains(token),
|
|
Effect::Send(..) => false,
|
|
Effect::Report(Report::Loads(loads)) => loads.contains_key(&markets[0].condition_id),
|
|
Effect::Report(Report::Quiet { market, .. }) => *market == markets[0].condition_id,
|
|
};
|
|
assert!(!effects.iter().any(mentions));
|
|
// The minute Snapshots of the 38 others, twice.
|
|
assert_eq!(snapshot_symbols(&effects).len(), 38 * 2);
|
|
}
|
|
|
|
#[test]
|
|
fn each_minute_reports_the_bytes_per_second_of_every_market() {
|
|
let minute = "2026-11-10T16:00:00Z".parse::<DateTime<Utc>>().unwrap();
|
|
let at = |seconds| minute + TimeDelta::seconds(seconds);
|
|
let market = |id: &str| Market {
|
|
condition_id: id.to_string(),
|
|
tokens: vec![format!("{id}a"), format!("{id}b")],
|
|
min_order_size: "5".to_string(),
|
|
neg_risk: false,
|
|
};
|
|
let (busy, quiet, late) = (market("0xbusy"), market("0xquiet"), market("0xlate"));
|
|
let mut feed = Feed::new("polymarket", 0, Chain::default());
|
|
feed.add_market(&busy, &[0], at(0));
|
|
// Added halfway through the minute: measured over its 30 s.
|
|
feed.add_market(&quiet, &[0], at(30));
|
|
// Added 15 s before it: kept for the next report.
|
|
feed.add_market(&late, &[0], at(45));
|
|
let change = |id: &str| {
|
|
serde_json::json!({"event_type": "price_change", "market": id, "timestamp": "1",
|
|
"price_changes": []})
|
|
.to_string()
|
|
};
|
|
let trade = serde_json::json!({"event_type": "last_trade_price", "market": "0xbusy",
|
|
"asset_id": "0xbusya", "price": "0.5", "size": "1", "side": "BUY",
|
|
"timestamp": "1"})
|
|
.to_string();
|
|
let tick_size = serde_json::json!({"event_type": "tick_size_change", "market": "0xquiet",
|
|
"asset_id": "0xquieta", "old_tick_size": "0.01", "new_tick_size": "0.001",
|
|
"timestamp": "1"})
|
|
.to_string();
|
|
// One frame, two markets: each takes half.
|
|
let shared = format!("[{},{}]", change("0xbusy"), change("0xquiet"));
|
|
let mut reports = Vec::new();
|
|
let mut on_frame = |feed: &mut Feed, seconds, text: &str| {
|
|
for effect in feed.on_frame(at(seconds), text) {
|
|
if let Effect::Report(Report::Loads(loads)) = effect {
|
|
reports.push(loads);
|
|
}
|
|
}
|
|
};
|
|
on_frame(&mut feed, 10, &change("0xbusy"));
|
|
on_frame(&mut feed, 20, &trade);
|
|
on_frame(&mut feed, 40, &tick_size);
|
|
on_frame(&mut feed, 50, &change("0xlate"));
|
|
on_frame(&mut feed, 55, &change("0xunknown"));
|
|
// The next minute: the first report, then nothing more until the one
|
|
// after, which starts again from zero.
|
|
on_frame(&mut feed, 60, &shared);
|
|
on_frame(&mut feed, 120, &change("0xunknown"));
|
|
|
|
let bytes = |text: &String| text.len() as u64;
|
|
let half = bytes(&shared) / 2;
|
|
assert_eq!(
|
|
reports,
|
|
[
|
|
Loads::from([
|
|
(
|
|
"0xbusy".to_string(),
|
|
(bytes(&change("0xbusy")) + bytes(&trade)) / 60
|
|
),
|
|
("0xquiet".to_string(), bytes(&tick_size) / 30),
|
|
]),
|
|
Loads::from([
|
|
("0xbusy".to_string(), half / 60),
|
|
("0xquiet".to_string(), half / 60),
|
|
("0xlate".to_string(), bytes(&change("0xlate")) / 75),
|
|
]),
|
|
]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn the_recording_reports_every_market_once_measured_long_enough() {
|
|
let frames = fixture_frames();
|
|
let markets = fixture_markets();
|
|
let mut feed = feed_of(&markets);
|
|
let reports: Vec<Loads> = replay(&mut feed, &frames)
|
|
.into_iter()
|
|
.filter_map(|effect| match effect {
|
|
Effect::Report(Report::Loads(loads)) => Some(loads),
|
|
_ => None,
|
|
})
|
|
.collect();
|
|
// The recording crosses two minutes, the first too soon after its
|
|
// start for any market to be measured.
|
|
let start = frames[0].0;
|
|
assert!(next_minute(start) - start < MIN_MEASURED);
|
|
assert_eq!(reports.len(), 2);
|
|
assert!(reports[0].is_empty());
|
|
let loads = &reports[1];
|
|
assert_eq!(loads.len(), markets.len());
|
|
assert!(
|
|
markets
|
|
.iter()
|
|
.all(|market| loads.contains_key(&market.condition_id))
|
|
);
|
|
assert!(loads.values().sum::<u64>() > 0);
|
|
}
|
|
|
|
#[test]
|
|
fn the_output_reconciles() {
|
|
// The book and writer threads as main runs them, fed the two
|
|
// connections' recording.
|
|
let frames = two_connections();
|
|
let (events_tx, events_rx) = crossbeam::channel::unbounded();
|
|
let (outputs_tx, outputs_rx) = crossbeam::channel::unbounded();
|
|
let (connection_tx, _connection_rx) = crossbeam::channel::unbounded();
|
|
let (_commands_tx, commands_rx) = crossbeam::channel::unbounded();
|
|
let (reports_tx, _reports_rx) = crossbeam::channel::unbounded();
|
|
for (received_at, connection, text) in &frames {
|
|
for arrival in arrivals(*received_at, *connection, text) {
|
|
events_tx.send(Received::Event(arrival)).unwrap();
|
|
}
|
|
}
|
|
drop(events_tx);
|
|
let feed = feed_on(&fixture_markets(), &[0, 1], frames[0].0);
|
|
let book = std::thread::spawn(move || {
|
|
serve(
|
|
feed,
|
|
vec![connection_tx.clone(), connection_tx],
|
|
events_rx,
|
|
commands_rx,
|
|
outputs_tx,
|
|
reports_tx,
|
|
)
|
|
});
|
|
let path = std::env::temp_dir().join(format!("reconcile-{}.sbe", std::process::id()));
|
|
let written = crate::write(path.to_str().unwrap(), outputs_rx);
|
|
book.join().unwrap();
|
|
written.unwrap();
|
|
let report = engineering_challenge::reconcile::reconcile(&path, None);
|
|
std::fs::remove_file(&path).unwrap();
|
|
|
|
let report = report.unwrap();
|
|
// The opening Snapshots, then one per symbol at each of the two
|
|
// minutes crossed (20 of the 40 tokens had an order book then).
|
|
assert_eq!(report.snapshots, 20 * 3, "{report:?}");
|
|
assert!(report.deltas > 100, "{report:?}");
|
|
assert_eq!(report.orphan_deltas, 0);
|
|
assert_eq!(report.unexplained, []);
|
|
}
|
|
}
|