feat(polymarket): subscription monitoring and connection recycling

The exchange can fail without telling us, and first one wins hides a
subscription gone quiet on one of a symbol's two connections.

- A connection that receives nothing for 30 s, not even the keepalive's
  answer, reconnects.
- Each book thread counts a symbol's changes per connection every minute:
  a subscription with none against three or more on the other is
  resubscribed, and moved to another connection if it stays quiet.
- The orchestrator recycles one connection at a time, each about hourly,
  and never while another connection is down or just back.
- A reconnect's opening book of a symbol unchanged for 10 s is judged like
  a resync's answer: one newer than ours that differs shows a symbol quiet
  on both connections, written as a Snapshot and logged as a loss.
This commit is contained in:
pascal committed 2026-11-11 17:42:27 +00:00
1 parent 02747b882f
commit 480ee3ad2d
10 files changed
+1346 -102

No files matched your search

+591 -41
View File
@@ -2,8 +2,11 @@
//! per symbol, first one wins: a change is taken from whichever connection
//! delivers it first, Polymarket's book hash saying where it belongs
//! (ADR 0003). It resyncs when a change fits on no connection in time, and
//! emits Deltas and Snapshots. The rules are in
//! `roadmap/04. redundant connections.md`, the protocol facts and the hash
//! emits Deltas and Snapshots. It also looks for what first one wins hides:
//! a subscription gone quiet on one connection, each minute, and symbols
//! quiet on both, in a reconnect's opening books. The rules are in
//! `roadmap/04. redundant connections.md` and
//! `roadmap/05. subscription monitoring.md`, the protocol facts and the hash
//! formula in `docs/notes.md`, the terms in `GLOSSARY.md`.
use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
@@ -47,15 +50,34 @@ const MAX_HELD: usize = 512;
const TAILS_TRIED: usize = 4;
/// When a change's copies are counted: no second copy came later than 0.7 s.
const COPIES_BY: TimeDelta = TimeDelta::seconds(2);
/// A connection that delivered none of a symbol's changes in a minute while
/// another delivered at least this many is quiet for it (sprint 05: crossed
/// once in 21 live minutes; at 1, copies landing in the next minute would
/// count).
const QUIET_AGAINST: u32 = 3;
/// A reconnect's opening book is judged only for a symbol unchanged this
/// long: a moving symbol's books run ahead of or behind our stream.
const STILL_FOR: TimeDelta = TimeDelta::seconds(10);
/// What the book thread does with an event or a command: write output, ask a
/// connection (by number) for something, or report its markets' loads to the
/// orchestrator.
/// connection (by number) for something, or report to the orchestrator.
#[derive(Debug)]
pub enum Effect {
Write(Output),
Send(usize, Command),
Report(Loads),
Report(Report),
}
/// What a book thread tells the orchestrator.
#[derive(Debug, PartialEq, Eq)]
pub enum Report {
Loads(Loads),
/// The market stayed quiet on the connection after being resubscribed
/// there: its subscription should move.
Quiet {
market: String,
connection: usize,
},
}
/// Each market's load (`conditionId` → bytes per second on one connection),
@@ -130,6 +152,33 @@ impl RecentChange {
}
}
/// A symbol's subscription on one of its connections, as watched for going
/// quiet.
#[derive(Debug, Clone, Copy)]
struct Watch {
connection: usize,
/// Its changes received from this connection this minute: every one,
/// whatever first one wins then made of it.
received: u32,
/// When it was last subscribed there: a minute it didn't cover whole is
/// not judged.
since: DateTime<Utc>,
/// When the connection last delivered a complete book of it: its first
/// since the connection came up is the reconnect's.
book_at: DateTime<Utc>,
}
impl Watch {
fn new(connection: usize, since: DateTime<Utc>) -> Self {
Self {
connection,
received: 0,
since,
book_at: DateTime::UNIX_EPOCH,
}
}
}
/// A change as applied, with the exchange's hash of the book after it.
struct RecentChange {
side: Side,
@@ -172,6 +221,10 @@ struct Symbol {
had_book: bool,
/// The connections it is subscribed on (three while a move overlaps).
connections: Vec<usize>,
/// One per connection in `connections`.
watches: Vec<Watch>,
/// The connection it was quiet on last minute, and resubscribed.
quiet_on: Option<usize>,
/// The receive time of its last Delta: a held change is stamped no earlier,
/// so replaying the output by time keeps its order.
last_written: DateTime<Utc>,
@@ -184,7 +237,7 @@ struct Symbol {
}
impl Symbol {
fn new(market: &Market, connections: &[usize]) -> Self {
fn new(market: &Market, connections: &[usize], now: DateTime<Utc>) -> Self {
Self {
state: State::AwaitingBook,
book: Book::default(),
@@ -196,6 +249,11 @@ impl Symbol {
verified_since_book: false,
had_book: false,
connections: connections.to_vec(),
watches: connections
.iter()
.map(|&connection| Watch::new(connection, now))
.collect(),
quiet_on: None,
last_written: DateTime::UNIX_EPOCH,
market: market.condition_id.clone(),
tick_size: String::new(),
@@ -216,7 +274,39 @@ impl Symbol {
// A switch already asked stays one until its book comes.
let switch = switch || self.asked.is_some_and(|(_, switch)| switch);
self.asked = Some((connection, switch));
effects.push(Effect::Send(connection, Command::Resync(token.to_string())));
effects.push(Effect::Send(
connection,
Command::Resubscribe(token.to_string()),
));
}
fn watch_mut(&mut self, connection: usize) -> Option<&mut Watch> {
self.watches
.iter_mut()
.find(|watch| watch.connection == connection)
}
/// The connection quiet for it over the minute begun at `started`: one
/// that delivered none of its changes while another delivered at least
/// [`QUIET_AGAINST`], both up and subscribed for the whole minute.
fn quiet(&self, started: DateTime<Utc>, up_since: &[Option<DateTime<Utc>>]) -> Option<usize> {
let judged: Vec<&Watch> = self
.watches
.iter()
.filter(|watch| {
watch.since <= started
&& up_since
.get(watch.connection)
.copied()
.flatten()
.is_some_and(|up| up <= started)
})
.collect();
let busiest = judged.iter().map(|watch| watch.received).max()?;
(busiest >= QUIET_AGAINST)
.then(|| judged.iter().find(|watch| watch.received == 0))
.flatten()
.map(|watch| watch.connection)
}
/// One of its connections that is up, other than `except`.
@@ -404,6 +494,15 @@ struct Counts {
subscription_gaps: u64,
losses: u64,
switches: u64,
/// Subscriptions found quiet, each resubscribed.
quiet: u64,
/// Markets quiet again after their resubscribe: reported for a move.
quiet_reported: u64,
/// A reconnect's opening books judged, those of still symbols.
reconnect_books: u64,
/// Of them, books that differed from every state of ours: both
/// subscriptions were quiet.
reconnect_losses: u64,
}
pub struct Feed {
@@ -414,6 +513,13 @@ pub struct Feed {
markets: HashMap<String, Tracked>,
/// By connection number: whether it is up.
up: Vec<bool>,
/// By connection number: since when it is up, if it is.
up_since: Vec<Option<DateTime<Utc>>>,
/// When the minute being counted began: its quiet subscriptions are
/// judged at its end. None before the first.
minute_started: Option<DateTime<Utc>>,
/// Changes received this minute, from all connections.
received_this_minute: u64,
/// The symbols with held changes: their ceilings are checked.
waiting: BTreeSet<String>,
/// The receive time of the last event applied: the time a Snapshot's book
@@ -436,6 +542,9 @@ impl Feed {
symbols: BTreeMap::new(),
markets: HashMap::new(),
up: Vec::new(),
up_since: Vec::new(),
minute_started: None,
received_this_minute: 0,
waiting: BTreeSet::new(),
last_received_at: None,
next_minute_at: None,
@@ -456,7 +565,7 @@ impl Feed {
) -> Vec<Effect> {
for token in &market.tokens {
self.symbols
.insert(token.clone(), Symbol::new(market, connections));
.insert(token.clone(), Symbol::new(market, connections, now));
}
let tracked = Tracked {
tokens: market.tokens.clone(),
@@ -476,7 +585,13 @@ impl Feed {
/// Moves a market's subscription from one connection to another: it
/// subscribes on `to` now, and leaves `from` once `to` delivers a book of
/// the market. The book stays as it is: no Snapshot, no subscription gap.
pub fn move_market(&mut self, market: &Market, from: usize, to: usize) -> Vec<Effect> {
pub fn move_market(
&mut self,
market: &Market,
from: usize,
to: usize,
now: DateTime<Utc>,
) -> Vec<Effect> {
let Some(tracked) = self.markets.get_mut(&market.condition_id) else {
return Vec::new();
};
@@ -493,6 +608,7 @@ impl Feed {
&& !symbol.connections.contains(&to)
{
symbol.connections.push(to);
symbol.watches.push(Watch::new(to, now));
}
}
effects.push(Effect::Send(to, Command::Subscribe(market.tokens.clone())));
@@ -513,6 +629,9 @@ impl Feed {
for token in &tokens {
if let Some(symbol) = self.symbols.get_mut(token) {
symbol.connections.retain(|&on| on != connection);
symbol
.watches
.retain(|watch| watch.connection != connection);
symbol.held.remove(&connection);
Self::lose(
token,
@@ -558,16 +677,25 @@ impl Feed {
match received {
Received::Event(arrival) => self.on_event(arrival),
Received::Up(connection) => {
self.set_up(connection, true);
// Its first events come after this marker; any other
// connection's before it were received earlier.
let at = self.last_received_at.unwrap_or(DateTime::UNIX_EPOCH);
self.set_up(connection, true, at);
Vec::new()
}
Received::Down(connection) => self.on_down(connection),
}
}
fn set_up(&mut self, connection: usize, up: bool) {
fn set_up(&mut self, connection: usize, up: bool, at: DateTime<Utc>) {
if self.up.len() <= connection {
self.up.resize(connection + 1, false);
self.up_since.resize(connection + 1, None);
}
if up && !self.up[connection] {
self.up_since[connection] = Some(at);
} else if !up {
self.up_since[connection] = None;
}
self.up[connection] = up;
}
@@ -576,7 +704,7 @@ impl Feed {
let (received_at, connection) = (arrival.received_at, arrival.connection);
// A connection that delivers is up, its marker possibly still behind.
if !is_up(&self.up, connection) {
self.set_up(connection, true);
self.set_up(connection, true, received_at);
}
// Minute Snapshots due before this event hold the books as they were.
let mut effects = self.tick(received_at);
@@ -631,7 +759,7 @@ impl Feed {
}
fn on_down(&mut self, connection: usize) -> Vec<Effect> {
self.set_up(connection, false);
self.set_up(connection, false, DateTime::UNIX_EPOCH);
let mut effects = Vec::new();
for (token, symbol) in &mut self.symbols {
if !symbol.connections.contains(&connection) {
@@ -692,14 +820,16 @@ impl Feed {
}
/// Once `now` reaches the next wall-clock minute: reports the loads,
/// emits the minute Snapshots, and asks again for the fresh books still
/// not come.
/// emits the minute Snapshots, asks again for the fresh books still not
/// come, and resubscribes (or reports) the subscriptions quiet over the
/// minute that ends.
pub fn tick(&mut self, now: DateTime<Utc>) -> Vec<Effect> {
let next = *self.next_minute_at.get_or_insert_with(|| next_minute(now));
if now < next {
return Vec::new();
}
self.next_minute_at = Some(next_minute(now));
let started = self.minute_started.replace(now);
let mut loads = Loads::new();
for (market, tracked) in &mut self.markets {
let connections = tracked
@@ -711,13 +841,48 @@ impl Feed {
loads.insert(market.clone(), load);
}
}
let mut effects = vec![Effect::Report(loads)];
let mut effects = vec![Effect::Report(Report::Loads(loads))];
let Some(received_at) = self.last_received_at else {
return effects;
};
self.log_minute();
let mut reported = BTreeSet::new();
for (token, symbol) in &mut self.symbols {
symbol.count_old_copies(now, &mut self.counts);
let quiet = started
.filter(|_| symbol.state == State::Verified && symbol.asked.is_none())
.and_then(|started| symbol.quiet(started, &self.up_since));
for watch in &mut symbol.watches {
watch.received = 0;
}
match quiet {
// Resubscribed already, and still quiet: it should move.
Some(connection) if symbol.quiet_on == Some(connection) => {
symbol.quiet_on = None;
if reported.insert((symbol.market.clone(), connection)) {
self.counts.quiet_reported += 1;
effects.push(Effect::Report(Report::Quiet {
market: symbol.market.clone(),
connection,
}));
}
}
// The other connection carries it meanwhile: the book that
// comes back is ignored, no Snapshot, no subscription gap.
Some(connection) => {
log!("polymarket: {token}: quiet on connection {connection}, resubscribed");
self.counts.quiet += 1;
symbol.quiet_on = Some(connection);
if let Some(watch) = symbol.watch_mut(connection) {
watch.since = now;
}
effects.push(Effect::Send(
connection,
Command::Resubscribe(token.clone()),
));
}
None => symbol.quiet_on = None,
}
if symbol.state != State::AwaitingBook {
effects.push(Effect::Write(symbol.book.snapshot(
token,
@@ -759,8 +924,10 @@ impl Feed {
log!(
"polymarket: book thread {}: {} symbols; so far {} changes written ({} verified), {} duplicates, \
{} rescued, {} held dropped, {} chains broken, {} resyncs, {} full books behind ours, \
{} subscription gaps, {} losses, {} switches; second copies {:.1} ms late at the median, \
{:.1} ms at p99; at most {} events waited this minute",
{} subscription gaps, {} losses, {} switches, {} quiet subscriptions resubscribed, \
{} markets reported quiet again, {} reconnect books judged ({} losses); second copies \
{:.1} ms late at the median, {:.1} ms at p99; this minute {} changes received, at most \
{} events waited",
self.number,
self.symbols.len(),
counts.written,
@@ -774,12 +941,18 @@ impl Feed {
counts.subscription_gaps,
counts.losses,
counts.switches,
counts.quiet,
counts.quiet_reported,
counts.reconnect_books,
counts.reconnect_losses,
lag(0.5),
lag(0.99),
self.received_this_minute,
self.peak_waiting
);
self.lags.clear();
self.peak_waiting = 0;
self.received_this_minute = 0;
}
/// Held changes past their ceiling: dropped if the book moved on without
@@ -862,11 +1035,14 @@ impl Feed {
counts.resyncs += 1;
}
/// A full book counts only when asked for: a symbol's first (from any of
/// its connections), or the answer to a resync. It replaces ours, brought
/// forward first if it is behind our stream, and is written as a Snapshot
/// when it is the symbol's first, when it differs from ours, or when the
/// connection followed changed. Then it is checked against its own hash.
/// A full book counts when asked for (a symbol's first, from any of its
/// connections, or the answer to a resync), or when it is a reconnect's
/// opening book of a symbol unchanged for [`STILL_FOR`]. It replaces
/// ours, brought forward first if it is behind our stream, and is written
/// as a Snapshot when it is the symbol's first, when it differs from
/// ours, or when the connection followed changed. Then it is checked
/// against its own hash. A reconnect's book that is ours, or behind
/// ours, changes nothing at all.
fn on_book(
&mut self,
received_at: DateTime<Utc>,
@@ -895,7 +1071,24 @@ impl Feed {
let complete = message.last_trade_price.is_some();
let asked = symbol.asked.filter(|&(asked, _)| asked == connection);
let first = symbol.state == State::AwaitingBook && symbol.connections.contains(&connection);
if !complete || (asked.is_none() && !first) {
// The connection's first complete book since it came up, for a
// symbol subscribed there before: the reconnect's opening book.
let up_since = self.up_since.get(connection).copied().flatten();
let reconnect = complete
&& symbol.watch_mut(connection).is_some_and(|watch| {
// A book at the very instant it came up came before it.
let opening = up_since.is_some_and(|up| watch.since < up && watch.book_at <= up);
watch.book_at = received_at;
opening
});
// Judged only for a still, verified symbol, nothing held or asked.
let reconnect = reconnect
&& asked.is_none()
&& symbol.asked.is_none()
&& symbol.state == State::Verified
&& symbol.held.is_empty()
&& received_at - symbol.advanced_at >= STILL_FOR;
if !complete || (asked.is_none() && !first && !reconnect) {
return;
}
let (Some(book), Some(exchange_timestamp), Some(book_hash)) = (
@@ -903,6 +1096,10 @@ impl Feed {
parse_millis(&message.timestamp),
parse_hash(&message.hash),
) else {
// Nobody asked for a reconnect's book: unreadable, it is ignored.
if reconnect {
return;
}
// A protocol error: ask again. A book that stays unreadable is asked
// for once per answer, and its symbol stays unchecked meanwhile.
log!("polymarket: {token}: unreadable book, asking for a fresh one");
@@ -910,6 +1107,28 @@ impl Feed {
self.counts.resyncs += 1;
return;
};
if reconnect {
self.counts.reconnect_books += 1;
// The exchange's opening books lag by minutes (docs/notes.md): one
// that is ours, or a state we passed, leaves ours, and the hash
// inputs, as they are.
let brought = symbol
.recent_changes
.iter()
.rposition(|change| change.hash == book_hash)
.map(|position| symbol.brought_forward(&book, position + 1));
if book == symbol.book || brought.as_ref() == Some(&symbol.book) {
return;
}
// Our chain holds up to our last change, so a book no newer than
// it holds nothing we missed, however far back it lags: further
// than the changes we remember, too.
if exchange_timestamp <= symbol.exchange_timestamp || symbol.was_ours(&book) {
self.counts.books_behind += 1;
return;
}
}
let inputs = reconnect.then(|| (symbol.tick_size.clone(), symbol.last_trade_price.clone()));
if let Some(tick_size) = message.tick_size {
symbol.tick_size = tick_size;
}
@@ -918,6 +1137,25 @@ impl Feed {
}
// Checked as received, before being brought forward below.
let verified = hash(&token, symbol, &book, &message.timestamp) == book_hash;
if let Some(inputs) = inputs {
// A book failing its own hash proves nothing against a verified
// one: ours stands, with its hash inputs.
if !verified {
(symbol.tick_size, symbol.last_trade_price) = inputs;
return;
}
// Newer than ours and no state of ours: both subscriptions were
// quiet. The other one is renewed too; its book is not waited for.
self.counts.reconnect_losses += 1;
for other in symbol.connections.clone() {
if other != connection {
if let Some(watch) = symbol.watch_mut(other) {
watch.since = received_at;
}
effects.push(Effect::Send(other, Command::Resubscribe(token.clone())));
}
}
}
let switch = asked.is_some_and(|(_, switch)| switch);
let compared = !first || symbol.had_book;
@@ -947,7 +1185,7 @@ impl Feed {
let differs = compared && book != symbol.book;
let cause = if switch {
Some(Cause::Switch)
} else if differs && !symbol.verified_since_book && !symbol.had_book {
} else if differs && !reconnect && !symbol.verified_since_book && !symbol.had_book {
// A subscribe hides a few ms of changes (docs/notes.md): until a
// change is verified, a differing book is that gap.
Some(Cause::SubscriptionGap)
@@ -978,8 +1216,13 @@ impl Feed {
effects.push(Effect::Write(snapshot));
symbol.last_written = symbol.last_written.max(received_at);
if let Some(cause) = cause {
let found = if reconnect {
", found by a reconnect"
} else {
""
};
let detail = format!(
" (its time {}, ours {})",
"{found} (its time {}, ours {})",
book_time.format("%H:%M:%S%.3f"),
exchange_timestamp.format("%H:%M:%S%.3f")
);
@@ -1015,6 +1258,10 @@ impl Feed {
let Some(symbol) = self.symbols.get_mut(&token) else {
continue;
};
if let Some(watch) = symbol.watch_mut(connection) {
watch.received += 1;
}
self.received_this_minute += 1;
let side = match change.side.as_str() {
"BUY" => Some(Side::Buy),
"SELL" => Some(Side::Sell),
@@ -1146,7 +1393,7 @@ pub fn run(
events: Receiver<Received>,
commands: Receiver<BookCommand>,
outputs: Sender<Output>,
reports: Sender<Loads>,
reports: Sender<Report>,
) {
serve(feed, Vec::new(), events, commands, outputs, reports)
}
@@ -1159,7 +1406,7 @@ fn serve(
events: Receiver<Received>,
commands: Receiver<BookCommand>,
outputs: Sender<Output>,
reports: Sender<Loads>,
reports: Sender<Report>,
) {
let no_commands = never();
let mut orchestrator_gone = false;
@@ -1190,7 +1437,9 @@ fn serve(
Vec::new()
}
Ok(BookCommand::Add(market, on)) => feed.add_market(&market, &on, Utc::now()),
Ok(BookCommand::Move { market, from, to }) => feed.move_market(&market, from, to),
Ok(BookCommand::Move { market, from, to }) => {
feed.move_market(&market, from, to, Utc::now())
}
Ok(BookCommand::Remove(market)) => feed.remove_market(&market),
// It is stopping: the connections' end still decides ours.
Err(_) => {
@@ -1210,7 +1459,7 @@ fn serve(
// The connection may be gone already, while stopping, and the
// orchestrator too.
Effect::Send(connection, command) => _ = connections[connection].send(command),
Effect::Report(loads) => _ = reports.send(loads),
Effect::Report(report) => _ = reports.send(report),
}
}
}
@@ -1417,7 +1666,7 @@ mod tests {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Send(connection, Command::Resync(token)) => {
Effect::Send(connection, Command::Resubscribe(token)) => {
Some((*connection, token.as_str()))
}
_ => None,
@@ -1549,9 +1798,13 @@ mod tests {
/// A complete book frame for `book`, as the exchange would send it in
/// answer to a resync, 5 ms after the symbol's last change.
fn fresh_book(token: &str, symbol: &Symbol, book: &Book) -> String {
let timestamp = (symbol.exchange_timestamp + TimeDelta::milliseconds(5))
.timestamp_millis()
.to_string();
let at = symbol.exchange_timestamp + TimeDelta::milliseconds(5);
book_frame(token, symbol, book, at)
}
/// A complete book frame for `book`, last changed at `at` (exchange time).
fn book_frame(token: &str, symbol: &Symbol, book: &Book, at: DateTime<Utc>) -> String {
let timestamp = at.timestamp_millis().to_string();
let levels = |levels: &BTreeMap<i64, i64>| {
levels
.iter()
@@ -2379,6 +2632,302 @@ mod tests {
assert_eq!(feed.counts.losses, losses + 1);
}
/// A frame setting `token`'s bid at `price` to `quantity`, `millis` its
/// exchange time, from the book as it is now.
fn change(feed: &Feed, token: &str, millis: i64, price: i64, quantity: i64) -> String {
let symbol = &feed.symbols[token];
let changes = [(Side::Buy, price, quantity)];
changes_frame(token, symbol, &symbol.book, &changes, millis).0
}
fn resyncs_of<'a>(effects: &'a [Effect], token: &str) -> Vec<(usize, &'a str)> {
resyncs(effects)
.into_iter()
.filter(|&(_, theirs)| theirs == token)
.collect()
}
/// From the next wall-clock minute after `at`, which the feed has begun:
/// the following minute is counted whole.
fn begin_minute(feed: &mut Feed, at: DateTime<Utc>) -> DateTime<Utc> {
let minute = next_minute(at);
feed.idle(minute);
minute
}
/// `count` changes of `token`, from 10 s after `start`, on the
/// connections `on` only.
fn changes(
feed: &mut Feed,
token: &str,
on: &[usize],
start: DateTime<Utc>,
price: i64,
count: i64,
) {
let millis = feed.symbols[token].exchange_timestamp.timestamp_millis() + 1_000;
for step in 0..count {
let at = start + TimeDelta::seconds(10 + step);
let frame = change(feed, token, millis + step, price, (step + 1) * 1_000_000);
for &connection in on {
feed.on_frame_from(connection, at, &frame);
}
}
}
fn quiet_reports(effects: &[Effect]) -> Vec<(&str, usize)> {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Report(Report::Quiet { market, connection }) => {
Some((market.as_str(), *connection))
}
_ => None,
})
.collect()
}
#[test]
fn a_subscription_quiet_on_one_connection_is_resubscribed_then_reported() {
let (mut feed, token, _, free, _, at) = two_connection_symbol();
let market = feed.symbols[&token].market.clone();
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
begin_minute(&mut feed, at);
let (losses, gaps) = (feed.counts.losses, feed.counts.subscription_gaps);
// Two changes against none: too few to say.
changes(&mut feed, &token, &[0], minute(0), free, 2);
let effects = feed.idle(minute(1));
assert!(resyncs_of(&effects, &token).is_empty());
assert_eq!(feed.counts.quiet, 0);
// Three: quiet on connection 1, resubscribed there.
changes(&mut feed, &token, &[0], minute(1), free, 3);
let effects = feed.idle(minute(2));
assert_eq!(resyncs_of(&effects, &token), [(1, token.as_str())]);
assert!(quiet_reports(&effects).is_empty());
assert_eq!(feed.counts.quiet, 1);
assert_eq!(feed.symbols[&token].state, State::Verified);
// The book the resubscribe brings back is not asked for: ignored.
let book = feed.symbols[&token].book.clone();
let back = fresh_book(&token, &feed.symbols[&token], &book);
let effects = feed.on_frame_from(1, minute(2) + TimeDelta::seconds(1), &back);
assert!(snapshot_symbols(&effects).is_empty());
// Still quiet the minute after: reported for a move, not resubscribed.
changes(&mut feed, &token, &[0], minute(2), free, 3);
let effects = feed.idle(minute(3));
assert!(resyncs_of(&effects, &token).is_empty());
assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]);
assert_eq!(
(feed.counts.losses, feed.counts.subscription_gaps),
(losses, gaps)
);
}
/// Both tokens of a market go quiet together: one report for the market.
#[test]
fn a_market_quiet_on_both_its_tokens_is_reported_once() {
let (mut feed, token, _, free, _, at) = two_connection_symbol();
let market = feed.symbols[&token].market.clone();
let tokens = feed.markets[&market].tokens.clone();
assert!(
tokens
.iter()
.all(|token| feed.symbols[token].state == State::Verified)
);
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
begin_minute(&mut feed, at);
for n in 0..2 {
for token in &tokens {
changes(&mut feed, token, &[0], minute(n), free, 3);
}
let effects = feed.idle(minute(n + 1));
if n == 0 {
assert_eq!(feed.counts.quiet, 2);
} else {
assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]);
}
}
}
#[test]
fn a_subscription_delivering_or_not_there_the_whole_minute_is_not_quiet() {
let (mut feed, token, _, free, _, at) = two_connection_symbol();
let markets = fixture_markets();
let market = markets
.iter()
.find(|market| market.tokens.contains(&token))
.unwrap();
// Connection 5 is up from before the minutes below.
feed.on_received(Received::Up(5));
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
begin_minute(&mut feed, at);
// Both deliver: neither is quiet; the symbols that didn't change
// aren't either.
changes(&mut feed, &token, &[0, 1], minute(0), free, 3);
let effects = feed.idle(minute(1));
assert!(resyncs(&effects).iter().all(|&(_, theirs)| theirs != token));
// Connection 1 dropped and came back within the minute: not judged
// for the changes it missed.
changes(&mut feed, &token, &[0], minute(1), free, 3);
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let effects = feed.idle(minute(2));
assert!(resyncs_of(&effects, &token).is_empty());
// Connection 5, subscribed in the middle of the minute and delivering
// nothing yet: not judged either.
feed.move_market(market, 1, 5, minute(2) + TimeDelta::seconds(5));
changes(&mut feed, &token, &[0, 1], minute(2), free, 3);
let effects = feed.idle(minute(3));
assert!(resyncs_of(&effects, &token).is_empty());
assert_eq!(feed.counts.quiet, 0);
}
/// A verified symbol on connections 0 and 1, changed once and then still
/// for 30 s: the feed, its token, its book before the change and now, a
/// free bid price, and the time 30 s after.
fn still_symbol() -> (Feed, String, Book, Book, i64, DateTime<Utc>) {
let (mut feed, token, before, free, millis, at) = two_connection_symbol();
let x = change(&feed, &token, millis, free, 1_000_000);
feed.on_frame_from(0, at, &x);
feed.on_frame_from(1, at, &x);
let ours = feed.symbols[&token].book.clone();
let later = at + TimeDelta::seconds(30);
// The minute's Snapshots, out of the way.
feed.idle(later);
(feed, token, before, ours, free, later)
}
/// Connection 1 drops and reconnects; its opening book of `token` holds
/// `book`.
fn reconnect_with(feed: &mut Feed, token: &str, book: &Book, at: DateTime<Utc>) -> Vec<Effect> {
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let opening = fresh_book(token, &feed.symbols[token], book);
feed.on_frame_from(1, at, &opening)
}
#[test]
fn a_reconnect_book_of_a_still_symbol_equal_to_ours_or_behind_changes_nothing() {
let (mut feed, token, before, ours, _, later) = still_symbol();
let losses = feed.counts.losses;
let effects = reconnect_with(&mut feed, &token, &ours, later);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 1);
// The server's opening books lag: a state we passed is no loss.
let behind = feed.counts.books_behind;
let effects = reconnect_with(&mut feed, &token, &before, later + TimeDelta::seconds(1));
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 2);
assert_eq!(feed.counts.books_behind, behind + 1);
assert_eq!(feed.symbols[&token].book, ours);
// It lags further back than the changes we remember: no newer than
// our last change, it still holds nothing we missed.
let symbol = feed.symbols.get_mut(&token).unwrap();
symbol.recent_changes.clear();
let old = symbol.exchange_timestamp - TimeDelta::minutes(10);
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let lagging = book_frame(&token, &feed.symbols[&token], &before, old);
let effects = feed.on_frame_from(1, later + TimeDelta::seconds(2), &lagging);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.books_behind, behind + 2);
assert_eq!(feed.symbols[&token].book, ours);
assert_eq!(feed.counts.losses, losses);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
/// A reconnect's book that fails its own hash proves nothing: the symbol
/// stays verified, with its book and its hash inputs.
#[test]
fn a_reconnect_book_failing_its_own_hash_changes_nothing() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
let frame = fresh_book(&token, &feed.symbols[&token], &other);
let mut frame: serde_json::Value = serde_json::from_str(&frame).unwrap();
frame["hash"] = "0".repeat(40).into();
frame["last_trade_price"] = "0.123".into();
let last_trade_price = feed.symbols[&token].last_trade_price.clone();
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let effects = feed.on_frame_from(1, later, &frame.to_string());
assert!(effects.is_empty());
let symbol = &feed.symbols[&token];
assert_eq!(symbol.state, State::Verified);
assert_eq!(symbol.book, ours);
assert_eq!(symbol.last_trade_price, last_trade_price);
}
/// Only a reconnect's book of a symbol with nothing asked is judged: not
/// the opening book of a subscription made since the connection came up
/// (a move's), nor any while a fresh book is asked.
#[test]
fn a_book_that_is_no_reconnects_or_comes_while_one_is_asked_is_not_judged() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let markets = fixture_markets();
let market = markets
.iter()
.find(|market| market.tokens.contains(&token))
.unwrap();
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
feed.on_received(Received::Up(5));
feed.move_market(market, 0, 5, later);
let opening = fresh_book(&token, &feed.symbols[&token], &other);
let effects = feed.on_frame_from(5, later + TimeDelta::seconds(1), &opening);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
feed.symbols.get_mut(&token).unwrap().asked = Some((5, false));
let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(2));
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
assert_eq!(feed.symbols[&token].book, ours);
}
#[test]
fn a_reconnect_book_of_a_still_symbol_that_differs_is_a_loss_and_resubscribes_the_other() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let (losses, gaps) = (feed.counts.losses, feed.counts.subscription_gaps);
// Even before any change of it was verified: a loss, not the gap of
// a subscribe, which is long past.
feed.symbols.get_mut(&token).unwrap().verified_since_book = false;
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
let effects = reconnect_with(&mut feed, &token, &other, later);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!(resyncs_of(&effects, &token), [(0, token.as_str())]);
assert_eq!(feed.counts.losses, losses + 1);
assert_eq!(feed.counts.subscription_gaps, gaps);
assert_eq!(feed.counts.reconnect_losses, 1);
assert_eq!(feed.symbols[&token].book, other);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
#[test]
fn a_reconnect_book_of_a_symbol_changed_lately_is_ignored() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let millis = feed.symbols[&token].exchange_timestamp.timestamp_millis() + 1_000;
let x = change(&feed, &token, millis, free, 2_000_000);
feed.on_frame_from(0, later, &x);
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(1));
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
// And the next book of it from that connection is no reconnect's.
feed.idle(later + TimeDelta::seconds(30));
let opening = fresh_book(&token, &feed.symbols[&token], &other);
let effects = feed.on_frame_from(1, later + TimeDelta::seconds(31), &opening);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
}
#[test]
fn the_followed_connection_down_switches_with_a_snapshot() {
let (mut feed, token, book, _, _, at) = two_connection_symbol();
@@ -2402,7 +2951,7 @@ mod tests {
.iter()
.find(|market| market.tokens.contains(&token))
.unwrap();
let effects = feed.move_market(market, 0, 5);
let effects = feed.move_market(market, 0, 5, at);
assert!(matches!(
effects.as_slice(),
[Effect::Send(5, Command::Subscribe(tokens))] if *tokens == market.tokens
@@ -2442,7 +2991,7 @@ mod tests {
let symbol = feed.symbols.get_mut(&token).unwrap();
symbol.state = State::Following(1);
symbol.asked = Some((0, false));
feed.move_market(market, 0, 5);
feed.move_market(market, 0, 5, at);
let opening = fresh_book(&token, &feed.symbols[&token], &book);
let effects = feed.on_frame_from(5, at, &opening);
assert!(
@@ -2515,7 +3064,7 @@ mod tests {
feed.on_frame_from(0, minute + TimeDelta::seconds(10), &change);
feed.on_frame_from(1, minute + TimeDelta::seconds(10), &change);
let effects = feed.tick(minute + TimeDelta::seconds(60));
let Some(Effect::Report(loads)) = effects.first() else {
let Some(Effect::Report(Report::Loads(loads))) = effects.first() else {
panic!("no report");
};
assert_eq!(loads["0xm"], change.len() as u64 / 60);
@@ -2583,9 +3132,10 @@ mod tests {
let mentions = |effect: &Effect| match effect {
Effect::Write(Output::Snapshot(snapshot)) => removed.contains(&snapshot.symbol),
Effect::Write(Output::Delta(delta)) => removed.contains(&delta.symbol),
Effect::Send(_, Command::Resync(token)) => removed.contains(token),
Effect::Send(_, Command::Resubscribe(token)) => removed.contains(token),
Effect::Send(..) => false,
Effect::Report(loads) => loads.contains_key(&markets[0].condition_id),
Effect::Report(Report::Loads(loads)) => loads.contains_key(&markets[0].condition_id),
Effect::Report(Report::Quiet { market, .. }) => *market == markets[0].condition_id,
};
assert!(!effects.iter().any(mentions));
// The minute Snapshots of the 38 others, twice.
@@ -2627,7 +3177,7 @@ mod tests {
let mut reports = Vec::new();
let mut on_frame = |feed: &mut Feed, seconds, text: &str| {
for effect in feed.on_frame(at(seconds), text) {
if let Effect::Report(loads) = effect {
if let Effect::Report(Report::Loads(loads)) = effect {
reports.push(loads);
}
}
@@ -2671,7 +3221,7 @@ mod tests {
let reports: Vec<Loads> = replay(&mut feed, &frames)
.into_iter()
.filter_map(|effect| match effect {
Effect::Report(loads) => Some(loads),
Effect::Report(Report::Loads(loads)) => Some(loads),
_ => None,
})
.collect();