feat(polymarket): subscription monitoring and connection recycling

The exchange can fail without telling us, and first one wins hides a
subscription gone quiet on one of a symbol's two connections.

- A connection that receives nothing for 30 s, not even the keepalive's
  answer, reconnects.
- Each book thread counts a symbol's changes per connection every minute:
  a subscription with none against three or more on the other is
  resubscribed, and moved to another connection if it stays quiet.
- The orchestrator recycles one connection at a time, each about hourly,
  and never while another connection is down or just back.
- A reconnect's opening book of a symbol unchanged for 10 s is judged like
  a resync's answer: one newer than ours that differs shows a symbol quiet
  on both connections, written as a Snapshot and logged as a loss.
This commit is contained in:
pascal committed 2026-11-11 17:42:27 +00:00
1 parent 02747b882f
commit 480ee3ad2d
10 files changed
+1346 -102

No files matched your search

+8
View File
@@ -71,6 +71,14 @@ mod tests {
let logged = format!("12:00:00.001 polymarket: {text}");
assert_eq!(parse(&logged), Some(("123".to_string(), at, cause)));
}
// A reconnect's loss says how it was found, after its cause.
let text = line(
"123",
at,
Cause::Loss,
", found by a reconnect (its time 12:00:00.000)",
);
assert_eq!(parse(&text), Some(("123".to_string(), at, Cause::Loss)));
assert_eq!(
parse("12:00:00.001 polymarket: connection 3: connected"),
None
+591 -41
View File
@@ -2,8 +2,11 @@
//! per symbol, first one wins: a change is taken from whichever connection
//! delivers it first, Polymarket's book hash saying where it belongs
//! (ADR 0003). It resyncs when a change fits on no connection in time, and
//! emits Deltas and Snapshots. The rules are in
//! `roadmap/04. redundant connections.md`, the protocol facts and the hash
//! emits Deltas and Snapshots. It also looks for what first one wins hides:
//! a subscription gone quiet on one connection, each minute, and symbols
//! quiet on both, in a reconnect's opening books. The rules are in
//! `roadmap/04. redundant connections.md` and
//! `roadmap/05. subscription monitoring.md`, the protocol facts and the hash
//! formula in `docs/notes.md`, the terms in `GLOSSARY.md`.
use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
@@ -47,15 +50,34 @@ const MAX_HELD: usize = 512;
const TAILS_TRIED: usize = 4;
/// When a change's copies are counted: no second copy came later than 0.7 s.
const COPIES_BY: TimeDelta = TimeDelta::seconds(2);
/// A connection that delivered none of a symbol's changes in a minute while
/// another delivered at least this many is quiet for it (sprint 05: crossed
/// once in 21 live minutes; at 1, copies landing in the next minute would
/// count).
const QUIET_AGAINST: u32 = 3;
/// A reconnect's opening book is judged only for a symbol unchanged this
/// long: a moving symbol's books run ahead of or behind our stream.
const STILL_FOR: TimeDelta = TimeDelta::seconds(10);
/// What the book thread does with an event or a command: write output, ask a
/// connection (by number) for something, or report its markets' loads to the
/// orchestrator.
/// connection (by number) for something, or report to the orchestrator.
#[derive(Debug)]
pub enum Effect {
Write(Output),
Send(usize, Command),
Report(Loads),
Report(Report),
}
/// What a book thread tells the orchestrator.
#[derive(Debug, PartialEq, Eq)]
pub enum Report {
Loads(Loads),
/// The market stayed quiet on the connection after being resubscribed
/// there: its subscription should move.
Quiet {
market: String,
connection: usize,
},
}
/// Each market's load (`conditionId` → bytes per second on one connection),
@@ -130,6 +152,33 @@ impl RecentChange {
}
}
/// A symbol's subscription on one of its connections, as watched for going
/// quiet.
#[derive(Debug, Clone, Copy)]
struct Watch {
connection: usize,
/// Its changes received from this connection this minute: every one,
/// whatever first one wins then made of it.
received: u32,
/// When it was last subscribed there: a minute it didn't cover whole is
/// not judged.
since: DateTime<Utc>,
/// When the connection last delivered a complete book of it: its first
/// since the connection came up is the reconnect's.
book_at: DateTime<Utc>,
}
impl Watch {
fn new(connection: usize, since: DateTime<Utc>) -> Self {
Self {
connection,
received: 0,
since,
book_at: DateTime::UNIX_EPOCH,
}
}
}
/// A change as applied, with the exchange's hash of the book after it.
struct RecentChange {
side: Side,
@@ -172,6 +221,10 @@ struct Symbol {
had_book: bool,
/// The connections it is subscribed on (three while a move overlaps).
connections: Vec<usize>,
/// One per connection in `connections`.
watches: Vec<Watch>,
/// The connection it was quiet on last minute, and resubscribed.
quiet_on: Option<usize>,
/// The receive time of its last Delta: a held change is stamped no earlier,
/// so replaying the output by time keeps its order.
last_written: DateTime<Utc>,
@@ -184,7 +237,7 @@ struct Symbol {
}
impl Symbol {
fn new(market: &Market, connections: &[usize]) -> Self {
fn new(market: &Market, connections: &[usize], now: DateTime<Utc>) -> Self {
Self {
state: State::AwaitingBook,
book: Book::default(),
@@ -196,6 +249,11 @@ impl Symbol {
verified_since_book: false,
had_book: false,
connections: connections.to_vec(),
watches: connections
.iter()
.map(|&connection| Watch::new(connection, now))
.collect(),
quiet_on: None,
last_written: DateTime::UNIX_EPOCH,
market: market.condition_id.clone(),
tick_size: String::new(),
@@ -216,7 +274,39 @@ impl Symbol {
// A switch already asked stays one until its book comes.
let switch = switch || self.asked.is_some_and(|(_, switch)| switch);
self.asked = Some((connection, switch));
effects.push(Effect::Send(connection, Command::Resync(token.to_string())));
effects.push(Effect::Send(
connection,
Command::Resubscribe(token.to_string()),
));
}
fn watch_mut(&mut self, connection: usize) -> Option<&mut Watch> {
self.watches
.iter_mut()
.find(|watch| watch.connection == connection)
}
/// The connection quiet for it over the minute begun at `started`: one
/// that delivered none of its changes while another delivered at least
/// [`QUIET_AGAINST`], both up and subscribed for the whole minute.
fn quiet(&self, started: DateTime<Utc>, up_since: &[Option<DateTime<Utc>>]) -> Option<usize> {
let judged: Vec<&Watch> = self
.watches
.iter()
.filter(|watch| {
watch.since <= started
&& up_since
.get(watch.connection)
.copied()
.flatten()
.is_some_and(|up| up <= started)
})
.collect();
let busiest = judged.iter().map(|watch| watch.received).max()?;
(busiest >= QUIET_AGAINST)
.then(|| judged.iter().find(|watch| watch.received == 0))
.flatten()
.map(|watch| watch.connection)
}
/// One of its connections that is up, other than `except`.
@@ -404,6 +494,15 @@ struct Counts {
subscription_gaps: u64,
losses: u64,
switches: u64,
/// Subscriptions found quiet, each resubscribed.
quiet: u64,
/// Markets quiet again after their resubscribe: reported for a move.
quiet_reported: u64,
/// A reconnect's opening books judged, those of still symbols.
reconnect_books: u64,
/// Of them, books that differed from every state of ours: both
/// subscriptions were quiet.
reconnect_losses: u64,
}
pub struct Feed {
@@ -414,6 +513,13 @@ pub struct Feed {
markets: HashMap<String, Tracked>,
/// By connection number: whether it is up.
up: Vec<bool>,
/// By connection number: since when it is up, if it is.
up_since: Vec<Option<DateTime<Utc>>>,
/// When the minute being counted began: its quiet subscriptions are
/// judged at its end. None before the first.
minute_started: Option<DateTime<Utc>>,
/// Changes received this minute, from all connections.
received_this_minute: u64,
/// The symbols with held changes: their ceilings are checked.
waiting: BTreeSet<String>,
/// The receive time of the last event applied: the time a Snapshot's book
@@ -436,6 +542,9 @@ impl Feed {
symbols: BTreeMap::new(),
markets: HashMap::new(),
up: Vec::new(),
up_since: Vec::new(),
minute_started: None,
received_this_minute: 0,
waiting: BTreeSet::new(),
last_received_at: None,
next_minute_at: None,
@@ -456,7 +565,7 @@ impl Feed {
) -> Vec<Effect> {
for token in &market.tokens {
self.symbols
.insert(token.clone(), Symbol::new(market, connections));
.insert(token.clone(), Symbol::new(market, connections, now));
}
let tracked = Tracked {
tokens: market.tokens.clone(),
@@ -476,7 +585,13 @@ impl Feed {
/// Moves a market's subscription from one connection to another: it
/// subscribes on `to` now, and leaves `from` once `to` delivers a book of
/// the market. The book stays as it is: no Snapshot, no subscription gap.
pub fn move_market(&mut self, market: &Market, from: usize, to: usize) -> Vec<Effect> {
pub fn move_market(
&mut self,
market: &Market,
from: usize,
to: usize,
now: DateTime<Utc>,
) -> Vec<Effect> {
let Some(tracked) = self.markets.get_mut(&market.condition_id) else {
return Vec::new();
};
@@ -493,6 +608,7 @@ impl Feed {
&& !symbol.connections.contains(&to)
{
symbol.connections.push(to);
symbol.watches.push(Watch::new(to, now));
}
}
effects.push(Effect::Send(to, Command::Subscribe(market.tokens.clone())));
@@ -513,6 +629,9 @@ impl Feed {
for token in &tokens {
if let Some(symbol) = self.symbols.get_mut(token) {
symbol.connections.retain(|&on| on != connection);
symbol
.watches
.retain(|watch| watch.connection != connection);
symbol.held.remove(&connection);
Self::lose(
token,
@@ -558,16 +677,25 @@ impl Feed {
match received {
Received::Event(arrival) => self.on_event(arrival),
Received::Up(connection) => {
self.set_up(connection, true);
// Its first events come after this marker; any other
// connection's before it were received earlier.
let at = self.last_received_at.unwrap_or(DateTime::UNIX_EPOCH);
self.set_up(connection, true, at);
Vec::new()
}
Received::Down(connection) => self.on_down(connection),
}
}
fn set_up(&mut self, connection: usize, up: bool) {
fn set_up(&mut self, connection: usize, up: bool, at: DateTime<Utc>) {
if self.up.len() <= connection {
self.up.resize(connection + 1, false);
self.up_since.resize(connection + 1, None);
}
if up && !self.up[connection] {
self.up_since[connection] = Some(at);
} else if !up {
self.up_since[connection] = None;
}
self.up[connection] = up;
}
@@ -576,7 +704,7 @@ impl Feed {
let (received_at, connection) = (arrival.received_at, arrival.connection);
// A connection that delivers is up, its marker possibly still behind.
if !is_up(&self.up, connection) {
self.set_up(connection, true);
self.set_up(connection, true, received_at);
}
// Minute Snapshots due before this event hold the books as they were.
let mut effects = self.tick(received_at);
@@ -631,7 +759,7 @@ impl Feed {
}
fn on_down(&mut self, connection: usize) -> Vec<Effect> {
self.set_up(connection, false);
self.set_up(connection, false, DateTime::UNIX_EPOCH);
let mut effects = Vec::new();
for (token, symbol) in &mut self.symbols {
if !symbol.connections.contains(&connection) {
@@ -692,14 +820,16 @@ impl Feed {
}
/// Once `now` reaches the next wall-clock minute: reports the loads,
/// emits the minute Snapshots, and asks again for the fresh books still
/// not come.
/// emits the minute Snapshots, asks again for the fresh books still not
/// come, and resubscribes (or reports) the subscriptions quiet over the
/// minute that ends.
pub fn tick(&mut self, now: DateTime<Utc>) -> Vec<Effect> {
let next = *self.next_minute_at.get_or_insert_with(|| next_minute(now));
if now < next {
return Vec::new();
}
self.next_minute_at = Some(next_minute(now));
let started = self.minute_started.replace(now);
let mut loads = Loads::new();
for (market, tracked) in &mut self.markets {
let connections = tracked
@@ -711,13 +841,48 @@ impl Feed {
loads.insert(market.clone(), load);
}
}
let mut effects = vec![Effect::Report(loads)];
let mut effects = vec![Effect::Report(Report::Loads(loads))];
let Some(received_at) = self.last_received_at else {
return effects;
};
self.log_minute();
let mut reported = BTreeSet::new();
for (token, symbol) in &mut self.symbols {
symbol.count_old_copies(now, &mut self.counts);
let quiet = started
.filter(|_| symbol.state == State::Verified && symbol.asked.is_none())
.and_then(|started| symbol.quiet(started, &self.up_since));
for watch in &mut symbol.watches {
watch.received = 0;
}
match quiet {
// Resubscribed already, and still quiet: it should move.
Some(connection) if symbol.quiet_on == Some(connection) => {
symbol.quiet_on = None;
if reported.insert((symbol.market.clone(), connection)) {
self.counts.quiet_reported += 1;
effects.push(Effect::Report(Report::Quiet {
market: symbol.market.clone(),
connection,
}));
}
}
// The other connection carries it meanwhile: the book that
// comes back is ignored, no Snapshot, no subscription gap.
Some(connection) => {
log!("polymarket: {token}: quiet on connection {connection}, resubscribed");
self.counts.quiet += 1;
symbol.quiet_on = Some(connection);
if let Some(watch) = symbol.watch_mut(connection) {
watch.since = now;
}
effects.push(Effect::Send(
connection,
Command::Resubscribe(token.clone()),
));
}
None => symbol.quiet_on = None,
}
if symbol.state != State::AwaitingBook {
effects.push(Effect::Write(symbol.book.snapshot(
token,
@@ -759,8 +924,10 @@ impl Feed {
log!(
"polymarket: book thread {}: {} symbols; so far {} changes written ({} verified), {} duplicates, \
{} rescued, {} held dropped, {} chains broken, {} resyncs, {} full books behind ours, \
{} subscription gaps, {} losses, {} switches; second copies {:.1} ms late at the median, \
{:.1} ms at p99; at most {} events waited this minute",
{} subscription gaps, {} losses, {} switches, {} quiet subscriptions resubscribed, \
{} markets reported quiet again, {} reconnect books judged ({} losses); second copies \
{:.1} ms late at the median, {:.1} ms at p99; this minute {} changes received, at most \
{} events waited",
self.number,
self.symbols.len(),
counts.written,
@@ -774,12 +941,18 @@ impl Feed {
counts.subscription_gaps,
counts.losses,
counts.switches,
counts.quiet,
counts.quiet_reported,
counts.reconnect_books,
counts.reconnect_losses,
lag(0.5),
lag(0.99),
self.received_this_minute,
self.peak_waiting
);
self.lags.clear();
self.peak_waiting = 0;
self.received_this_minute = 0;
}
/// Held changes past their ceiling: dropped if the book moved on without
@@ -862,11 +1035,14 @@ impl Feed {
counts.resyncs += 1;
}
/// A full book counts only when asked for: a symbol's first (from any of
/// its connections), or the answer to a resync. It replaces ours, brought
/// forward first if it is behind our stream, and is written as a Snapshot
/// when it is the symbol's first, when it differs from ours, or when the
/// connection followed changed. Then it is checked against its own hash.
/// A full book counts when asked for (a symbol's first, from any of its
/// connections, or the answer to a resync), or when it is a reconnect's
/// opening book of a symbol unchanged for [`STILL_FOR`]. It replaces
/// ours, brought forward first if it is behind our stream, and is written
/// as a Snapshot when it is the symbol's first, when it differs from
/// ours, or when the connection followed changed. Then it is checked
/// against its own hash. A reconnect's book that is ours, or behind
/// ours, changes nothing at all.
fn on_book(
&mut self,
received_at: DateTime<Utc>,
@@ -895,7 +1071,24 @@ impl Feed {
let complete = message.last_trade_price.is_some();
let asked = symbol.asked.filter(|&(asked, _)| asked == connection);
let first = symbol.state == State::AwaitingBook && symbol.connections.contains(&connection);
if !complete || (asked.is_none() && !first) {
// The connection's first complete book since it came up, for a
// symbol subscribed there before: the reconnect's opening book.
let up_since = self.up_since.get(connection).copied().flatten();
let reconnect = complete
&& symbol.watch_mut(connection).is_some_and(|watch| {
// A book at the very instant it came up came before it.
let opening = up_since.is_some_and(|up| watch.since < up && watch.book_at <= up);
watch.book_at = received_at;
opening
});
// Judged only for a still, verified symbol, nothing held or asked.
let reconnect = reconnect
&& asked.is_none()
&& symbol.asked.is_none()
&& symbol.state == State::Verified
&& symbol.held.is_empty()
&& received_at - symbol.advanced_at >= STILL_FOR;
if !complete || (asked.is_none() && !first && !reconnect) {
return;
}
let (Some(book), Some(exchange_timestamp), Some(book_hash)) = (
@@ -903,6 +1096,10 @@ impl Feed {
parse_millis(&message.timestamp),
parse_hash(&message.hash),
) else {
// Nobody asked for a reconnect's book: unreadable, it is ignored.
if reconnect {
return;
}
// A protocol error: ask again. A book that stays unreadable is asked
// for once per answer, and its symbol stays unchecked meanwhile.
log!("polymarket: {token}: unreadable book, asking for a fresh one");
@@ -910,6 +1107,28 @@ impl Feed {
self.counts.resyncs += 1;
return;
};
if reconnect {
self.counts.reconnect_books += 1;
// The exchange's opening books lag by minutes (docs/notes.md): one
// that is ours, or a state we passed, leaves ours, and the hash
// inputs, as they are.
let brought = symbol
.recent_changes
.iter()
.rposition(|change| change.hash == book_hash)
.map(|position| symbol.brought_forward(&book, position + 1));
if book == symbol.book || brought.as_ref() == Some(&symbol.book) {
return;
}
// Our chain holds up to our last change, so a book no newer than
// it holds nothing we missed, however far back it lags: further
// than the changes we remember, too.
if exchange_timestamp <= symbol.exchange_timestamp || symbol.was_ours(&book) {
self.counts.books_behind += 1;
return;
}
}
let inputs = reconnect.then(|| (symbol.tick_size.clone(), symbol.last_trade_price.clone()));
if let Some(tick_size) = message.tick_size {
symbol.tick_size = tick_size;
}
@@ -918,6 +1137,25 @@ impl Feed {
}
// Checked as received, before being brought forward below.
let verified = hash(&token, symbol, &book, &message.timestamp) == book_hash;
if let Some(inputs) = inputs {
// A book failing its own hash proves nothing against a verified
// one: ours stands, with its hash inputs.
if !verified {
(symbol.tick_size, symbol.last_trade_price) = inputs;
return;
}
// Newer than ours and no state of ours: both subscriptions were
// quiet. The other one is renewed too; its book is not waited for.
self.counts.reconnect_losses += 1;
for other in symbol.connections.clone() {
if other != connection {
if let Some(watch) = symbol.watch_mut(other) {
watch.since = received_at;
}
effects.push(Effect::Send(other, Command::Resubscribe(token.clone())));
}
}
}
let switch = asked.is_some_and(|(_, switch)| switch);
let compared = !first || symbol.had_book;
@@ -947,7 +1185,7 @@ impl Feed {
let differs = compared && book != symbol.book;
let cause = if switch {
Some(Cause::Switch)
} else if differs && !symbol.verified_since_book && !symbol.had_book {
} else if differs && !reconnect && !symbol.verified_since_book && !symbol.had_book {
// A subscribe hides a few ms of changes (docs/notes.md): until a
// change is verified, a differing book is that gap.
Some(Cause::SubscriptionGap)
@@ -978,8 +1216,13 @@ impl Feed {
effects.push(Effect::Write(snapshot));
symbol.last_written = symbol.last_written.max(received_at);
if let Some(cause) = cause {
let found = if reconnect {
", found by a reconnect"
} else {
""
};
let detail = format!(
" (its time {}, ours {})",
"{found} (its time {}, ours {})",
book_time.format("%H:%M:%S%.3f"),
exchange_timestamp.format("%H:%M:%S%.3f")
);
@@ -1015,6 +1258,10 @@ impl Feed {
let Some(symbol) = self.symbols.get_mut(&token) else {
continue;
};
if let Some(watch) = symbol.watch_mut(connection) {
watch.received += 1;
}
self.received_this_minute += 1;
let side = match change.side.as_str() {
"BUY" => Some(Side::Buy),
"SELL" => Some(Side::Sell),
@@ -1146,7 +1393,7 @@ pub fn run(
events: Receiver<Received>,
commands: Receiver<BookCommand>,
outputs: Sender<Output>,
reports: Sender<Loads>,
reports: Sender<Report>,
) {
serve(feed, Vec::new(), events, commands, outputs, reports)
}
@@ -1159,7 +1406,7 @@ fn serve(
events: Receiver<Received>,
commands: Receiver<BookCommand>,
outputs: Sender<Output>,
reports: Sender<Loads>,
reports: Sender<Report>,
) {
let no_commands = never();
let mut orchestrator_gone = false;
@@ -1190,7 +1437,9 @@ fn serve(
Vec::new()
}
Ok(BookCommand::Add(market, on)) => feed.add_market(&market, &on, Utc::now()),
Ok(BookCommand::Move { market, from, to }) => feed.move_market(&market, from, to),
Ok(BookCommand::Move { market, from, to }) => {
feed.move_market(&market, from, to, Utc::now())
}
Ok(BookCommand::Remove(market)) => feed.remove_market(&market),
// It is stopping: the connections' end still decides ours.
Err(_) => {
@@ -1210,7 +1459,7 @@ fn serve(
// The connection may be gone already, while stopping, and the
// orchestrator too.
Effect::Send(connection, command) => _ = connections[connection].send(command),
Effect::Report(loads) => _ = reports.send(loads),
Effect::Report(report) => _ = reports.send(report),
}
}
}
@@ -1417,7 +1666,7 @@ mod tests {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Send(connection, Command::Resync(token)) => {
Effect::Send(connection, Command::Resubscribe(token)) => {
Some((*connection, token.as_str()))
}
_ => None,
@@ -1549,9 +1798,13 @@ mod tests {
/// A complete book frame for `book`, as the exchange would send it in
/// answer to a resync, 5 ms after the symbol's last change.
fn fresh_book(token: &str, symbol: &Symbol, book: &Book) -> String {
let timestamp = (symbol.exchange_timestamp + TimeDelta::milliseconds(5))
.timestamp_millis()
.to_string();
let at = symbol.exchange_timestamp + TimeDelta::milliseconds(5);
book_frame(token, symbol, book, at)
}
/// A complete book frame for `book`, last changed at `at` (exchange time).
fn book_frame(token: &str, symbol: &Symbol, book: &Book, at: DateTime<Utc>) -> String {
let timestamp = at.timestamp_millis().to_string();
let levels = |levels: &BTreeMap<i64, i64>| {
levels
.iter()
@@ -2379,6 +2632,302 @@ mod tests {
assert_eq!(feed.counts.losses, losses + 1);
}
/// A frame setting `token`'s bid at `price` to `quantity`, `millis` its
/// exchange time, from the book as it is now.
fn change(feed: &Feed, token: &str, millis: i64, price: i64, quantity: i64) -> String {
let symbol = &feed.symbols[token];
let changes = [(Side::Buy, price, quantity)];
changes_frame(token, symbol, &symbol.book, &changes, millis).0
}
fn resyncs_of<'a>(effects: &'a [Effect], token: &str) -> Vec<(usize, &'a str)> {
resyncs(effects)
.into_iter()
.filter(|&(_, theirs)| theirs == token)
.collect()
}
/// From the next wall-clock minute after `at`, which the feed has begun:
/// the following minute is counted whole.
fn begin_minute(feed: &mut Feed, at: DateTime<Utc>) -> DateTime<Utc> {
let minute = next_minute(at);
feed.idle(minute);
minute
}
/// `count` changes of `token`, from 10 s after `start`, on the
/// connections `on` only.
fn changes(
feed: &mut Feed,
token: &str,
on: &[usize],
start: DateTime<Utc>,
price: i64,
count: i64,
) {
let millis = feed.symbols[token].exchange_timestamp.timestamp_millis() + 1_000;
for step in 0..count {
let at = start + TimeDelta::seconds(10 + step);
let frame = change(feed, token, millis + step, price, (step + 1) * 1_000_000);
for &connection in on {
feed.on_frame_from(connection, at, &frame);
}
}
}
fn quiet_reports(effects: &[Effect]) -> Vec<(&str, usize)> {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Report(Report::Quiet { market, connection }) => {
Some((market.as_str(), *connection))
}
_ => None,
})
.collect()
}
#[test]
fn a_subscription_quiet_on_one_connection_is_resubscribed_then_reported() {
let (mut feed, token, _, free, _, at) = two_connection_symbol();
let market = feed.symbols[&token].market.clone();
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
begin_minute(&mut feed, at);
let (losses, gaps) = (feed.counts.losses, feed.counts.subscription_gaps);
// Two changes against none: too few to say.
changes(&mut feed, &token, &[0], minute(0), free, 2);
let effects = feed.idle(minute(1));
assert!(resyncs_of(&effects, &token).is_empty());
assert_eq!(feed.counts.quiet, 0);
// Three: quiet on connection 1, resubscribed there.
changes(&mut feed, &token, &[0], minute(1), free, 3);
let effects = feed.idle(minute(2));
assert_eq!(resyncs_of(&effects, &token), [(1, token.as_str())]);
assert!(quiet_reports(&effects).is_empty());
assert_eq!(feed.counts.quiet, 1);
assert_eq!(feed.symbols[&token].state, State::Verified);
// The book the resubscribe brings back is not asked for: ignored.
let book = feed.symbols[&token].book.clone();
let back = fresh_book(&token, &feed.symbols[&token], &book);
let effects = feed.on_frame_from(1, minute(2) + TimeDelta::seconds(1), &back);
assert!(snapshot_symbols(&effects).is_empty());
// Still quiet the minute after: reported for a move, not resubscribed.
changes(&mut feed, &token, &[0], minute(2), free, 3);
let effects = feed.idle(minute(3));
assert!(resyncs_of(&effects, &token).is_empty());
assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]);
assert_eq!(
(feed.counts.losses, feed.counts.subscription_gaps),
(losses, gaps)
);
}
/// Both tokens of a market go quiet together: one report for the market.
#[test]
fn a_market_quiet_on_both_its_tokens_is_reported_once() {
let (mut feed, token, _, free, _, at) = two_connection_symbol();
let market = feed.symbols[&token].market.clone();
let tokens = feed.markets[&market].tokens.clone();
assert!(
tokens
.iter()
.all(|token| feed.symbols[token].state == State::Verified)
);
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
begin_minute(&mut feed, at);
for n in 0..2 {
for token in &tokens {
changes(&mut feed, token, &[0], minute(n), free, 3);
}
let effects = feed.idle(minute(n + 1));
if n == 0 {
assert_eq!(feed.counts.quiet, 2);
} else {
assert_eq!(quiet_reports(&effects), [(market.as_str(), 1)]);
}
}
}
#[test]
fn a_subscription_delivering_or_not_there_the_whole_minute_is_not_quiet() {
let (mut feed, token, _, free, _, at) = two_connection_symbol();
let markets = fixture_markets();
let market = markets
.iter()
.find(|market| market.tokens.contains(&token))
.unwrap();
// Connection 5 is up from before the minutes below.
feed.on_received(Received::Up(5));
let minute = |n: i64| next_minute(at) + TimeDelta::minutes(n);
begin_minute(&mut feed, at);
// Both deliver: neither is quiet; the symbols that didn't change
// aren't either.
changes(&mut feed, &token, &[0, 1], minute(0), free, 3);
let effects = feed.idle(minute(1));
assert!(resyncs(&effects).iter().all(|&(_, theirs)| theirs != token));
// Connection 1 dropped and came back within the minute: not judged
// for the changes it missed.
changes(&mut feed, &token, &[0], minute(1), free, 3);
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let effects = feed.idle(minute(2));
assert!(resyncs_of(&effects, &token).is_empty());
// Connection 5, subscribed in the middle of the minute and delivering
// nothing yet: not judged either.
feed.move_market(market, 1, 5, minute(2) + TimeDelta::seconds(5));
changes(&mut feed, &token, &[0, 1], minute(2), free, 3);
let effects = feed.idle(minute(3));
assert!(resyncs_of(&effects, &token).is_empty());
assert_eq!(feed.counts.quiet, 0);
}
/// A verified symbol on connections 0 and 1, changed once and then still
/// for 30 s: the feed, its token, its book before the change and now, a
/// free bid price, and the time 30 s after.
fn still_symbol() -> (Feed, String, Book, Book, i64, DateTime<Utc>) {
let (mut feed, token, before, free, millis, at) = two_connection_symbol();
let x = change(&feed, &token, millis, free, 1_000_000);
feed.on_frame_from(0, at, &x);
feed.on_frame_from(1, at, &x);
let ours = feed.symbols[&token].book.clone();
let later = at + TimeDelta::seconds(30);
// The minute's Snapshots, out of the way.
feed.idle(later);
(feed, token, before, ours, free, later)
}
/// Connection 1 drops and reconnects; its opening book of `token` holds
/// `book`.
fn reconnect_with(feed: &mut Feed, token: &str, book: &Book, at: DateTime<Utc>) -> Vec<Effect> {
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let opening = fresh_book(token, &feed.symbols[token], book);
feed.on_frame_from(1, at, &opening)
}
#[test]
fn a_reconnect_book_of_a_still_symbol_equal_to_ours_or_behind_changes_nothing() {
let (mut feed, token, before, ours, _, later) = still_symbol();
let losses = feed.counts.losses;
let effects = reconnect_with(&mut feed, &token, &ours, later);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 1);
// The server's opening books lag: a state we passed is no loss.
let behind = feed.counts.books_behind;
let effects = reconnect_with(&mut feed, &token, &before, later + TimeDelta::seconds(1));
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 2);
assert_eq!(feed.counts.books_behind, behind + 1);
assert_eq!(feed.symbols[&token].book, ours);
// It lags further back than the changes we remember: no newer than
// our last change, it still holds nothing we missed.
let symbol = feed.symbols.get_mut(&token).unwrap();
symbol.recent_changes.clear();
let old = symbol.exchange_timestamp - TimeDelta::minutes(10);
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let lagging = book_frame(&token, &feed.symbols[&token], &before, old);
let effects = feed.on_frame_from(1, later + TimeDelta::seconds(2), &lagging);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.books_behind, behind + 2);
assert_eq!(feed.symbols[&token].book, ours);
assert_eq!(feed.counts.losses, losses);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
/// A reconnect's book that fails its own hash proves nothing: the symbol
/// stays verified, with its book and its hash inputs.
#[test]
fn a_reconnect_book_failing_its_own_hash_changes_nothing() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
let frame = fresh_book(&token, &feed.symbols[&token], &other);
let mut frame: serde_json::Value = serde_json::from_str(&frame).unwrap();
frame["hash"] = "0".repeat(40).into();
frame["last_trade_price"] = "0.123".into();
let last_trade_price = feed.symbols[&token].last_trade_price.clone();
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(1));
let effects = feed.on_frame_from(1, later, &frame.to_string());
assert!(effects.is_empty());
let symbol = &feed.symbols[&token];
assert_eq!(symbol.state, State::Verified);
assert_eq!(symbol.book, ours);
assert_eq!(symbol.last_trade_price, last_trade_price);
}
/// Only a reconnect's book of a symbol with nothing asked is judged: not
/// the opening book of a subscription made since the connection came up
/// (a move's), nor any while a fresh book is asked.
#[test]
fn a_book_that_is_no_reconnects_or_comes_while_one_is_asked_is_not_judged() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let markets = fixture_markets();
let market = markets
.iter()
.find(|market| market.tokens.contains(&token))
.unwrap();
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
feed.on_received(Received::Up(5));
feed.move_market(market, 0, 5, later);
let opening = fresh_book(&token, &feed.symbols[&token], &other);
let effects = feed.on_frame_from(5, later + TimeDelta::seconds(1), &opening);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
feed.symbols.get_mut(&token).unwrap().asked = Some((5, false));
let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(2));
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
assert_eq!(feed.symbols[&token].book, ours);
}
#[test]
fn a_reconnect_book_of_a_still_symbol_that_differs_is_a_loss_and_resubscribes_the_other() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let (losses, gaps) = (feed.counts.losses, feed.counts.subscription_gaps);
// Even before any change of it was verified: a loss, not the gap of
// a subscribe, which is long past.
feed.symbols.get_mut(&token).unwrap().verified_since_book = false;
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
let effects = reconnect_with(&mut feed, &token, &other, later);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!(resyncs_of(&effects, &token), [(0, token.as_str())]);
assert_eq!(feed.counts.losses, losses + 1);
assert_eq!(feed.counts.subscription_gaps, gaps);
assert_eq!(feed.counts.reconnect_losses, 1);
assert_eq!(feed.symbols[&token].book, other);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
#[test]
fn a_reconnect_book_of_a_symbol_changed_lately_is_ignored() {
let (mut feed, token, _, ours, free, later) = still_symbol();
let millis = feed.symbols[&token].exchange_timestamp.timestamp_millis() + 1_000;
let x = change(&feed, &token, millis, free, 2_000_000);
feed.on_frame_from(0, later, &x);
let mut other = ours.clone();
other.set(Side::Buy, free, 7_000_000);
let effects = reconnect_with(&mut feed, &token, &other, later + TimeDelta::seconds(1));
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
// And the next book of it from that connection is no reconnect's.
feed.idle(later + TimeDelta::seconds(30));
let opening = fresh_book(&token, &feed.symbols[&token], &other);
let effects = feed.on_frame_from(1, later + TimeDelta::seconds(31), &opening);
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(feed.counts.reconnect_books, 0);
}
#[test]
fn the_followed_connection_down_switches_with_a_snapshot() {
let (mut feed, token, book, _, _, at) = two_connection_symbol();
@@ -2402,7 +2951,7 @@ mod tests {
.iter()
.find(|market| market.tokens.contains(&token))
.unwrap();
let effects = feed.move_market(market, 0, 5);
let effects = feed.move_market(market, 0, 5, at);
assert!(matches!(
effects.as_slice(),
[Effect::Send(5, Command::Subscribe(tokens))] if *tokens == market.tokens
@@ -2442,7 +2991,7 @@ mod tests {
let symbol = feed.symbols.get_mut(&token).unwrap();
symbol.state = State::Following(1);
symbol.asked = Some((0, false));
feed.move_market(market, 0, 5);
feed.move_market(market, 0, 5, at);
let opening = fresh_book(&token, &feed.symbols[&token], &book);
let effects = feed.on_frame_from(5, at, &opening);
assert!(
@@ -2515,7 +3064,7 @@ mod tests {
feed.on_frame_from(0, minute + TimeDelta::seconds(10), &change);
feed.on_frame_from(1, minute + TimeDelta::seconds(10), &change);
let effects = feed.tick(minute + TimeDelta::seconds(60));
let Some(Effect::Report(loads)) = effects.first() else {
let Some(Effect::Report(Report::Loads(loads))) = effects.first() else {
panic!("no report");
};
assert_eq!(loads["0xm"], change.len() as u64 / 60);
@@ -2583,9 +3132,10 @@ mod tests {
let mentions = |effect: &Effect| match effect {
Effect::Write(Output::Snapshot(snapshot)) => removed.contains(&snapshot.symbol),
Effect::Write(Output::Delta(delta)) => removed.contains(&delta.symbol),
Effect::Send(_, Command::Resync(token)) => removed.contains(token),
Effect::Send(_, Command::Resubscribe(token)) => removed.contains(token),
Effect::Send(..) => false,
Effect::Report(loads) => loads.contains_key(&markets[0].condition_id),
Effect::Report(Report::Loads(loads)) => loads.contains_key(&markets[0].condition_id),
Effect::Report(Report::Quiet { market, .. }) => *market == markets[0].condition_id,
};
assert!(!effects.iter().any(mentions));
// The minute Snapshots of the 38 others, twice.
@@ -2627,7 +3177,7 @@ mod tests {
let mut reports = Vec::new();
let mut on_frame = |feed: &mut Feed, seconds, text: &str| {
for effect in feed.on_frame(at(seconds), text) {
if let Effect::Report(loads) = effect {
if let Effect::Report(Report::Loads(loads)) = effect {
reports.push(loads);
}
}
@@ -2671,7 +3221,7 @@ mod tests {
let reports: Vec<Loads> = replay(&mut feed, &frames)
.into_iter()
.filter_map(|effect| match effect {
Effect::Report(loads) => Some(loads),
Effect::Report(Report::Loads(loads)) => Some(loads),
_ => None,
})
.collect();
+206 -29
View File
@@ -1,7 +1,7 @@
//! The orchestrator: the one owner of what is tracked. It keeps the tracked
//! markets equal to Gamma's live ones, decides which connections carry each
//! market and owns every timer; the two fetchers only fetch (ADR 0002, ADR
//! 0003, roadmap/04).
//! market and owns every timer, recycling connections among them; the two
//! fetchers only fetch (ADR 0002, ADR 0003, roadmap/04, roadmap/05).
use std::cmp::Reverse;
use std::collections::{HashMap, HashSet, VecDeque};
@@ -14,8 +14,9 @@ use chrono::{DateTime, TimeDelta, Utc};
use crossbeam::channel::{Sender, select, unbounded};
use engineering_challenge::book::Output;
use super::feed::{self, BookCommand, Feed, Loads};
use super::feed::{self, BookCommand, Feed, Loads, Report};
use super::gamma::{self, Finding, Request};
use super::socket::{Command, Notice};
use super::wire::Received;
use super::{BOOK_THREADS, Market, book_thread, socket};
@@ -50,6 +51,9 @@ const RESCAN_EVERY: Duration = Duration::from_secs(60 * 60);
/// `createdAt` (docs/notes.md); known markets are skipped, so the margin costs
/// only the pages it reads again.
const RECENT_MARGIN: TimeDelta = TimeDelta::minutes(15);
/// Each connection is recycled about this often, one at a time, to shed what
/// accumulates on the exchange's side of it.
const RECYCLE_EVERY: Duration = Duration::from_secs(60 * 60);
/// So SIGINT is seen while no timer is due.
const WAKE_EVERY: Duration = Duration::from_secs(1);
@@ -70,10 +74,10 @@ pub fn run(outputs: Sender<Output>, stop: &Arc<AtomicBool>) -> usize {
})
.collect();
drop(findings_tx);
let (reports_tx, reports) = unbounded::<Loads>();
let (closes_tx, closes) = unbounded::<usize>();
let mut threads = Threads::start(&outputs, &reports_tx, &closes_tx, stop);
drop(closes_tx);
let (reports_tx, reports) = unbounded::<Report>();
let (notices_tx, notices) = unbounded::<Notice>();
let mut threads = Threads::start(&outputs, &reports_tx, &notices_tx, stop);
drop(notices_tx);
// The book threads hold the only senders the writer waits for.
drop(outputs);
drop(reports_tx);
@@ -91,6 +95,7 @@ pub fn run(outputs: Sender<Output>, stop: &Arc<AtomicBool>) -> usize {
let mut queue: VecDeque<BookCommand> = VecDeque::new();
let mut next_deal = Instant::now();
let (mut added, mut removed, mut moved, mut splits) = (0, 0, 0, 0);
let (mut recycled, mut quiet_moves, mut silent) = (0, 0, 0);
// The minute checks cover the first scan from its start.
let mut recent_since = Utc::now() - RECENT_MARGIN;
let mut recent_asked_at: Option<DateTime<Utc>> = None;
@@ -134,12 +139,22 @@ pub fn run(outputs: Sender<Output>, stop: &Arc<AtomicBool>) -> usize {
let busiest = tracker.connections.iter().map(|one| one.load).max();
log!(
"polymarket: orchestrator: {} markets on {} connections; {added} added, {removed} \
removed, {moved} moved, {splits} splits this minute; busiest connection at {} bytes/s",
removed, {moved} moved ({quiet_moves} quiet), {splits} splits, {recycled} recycled, \
{silent} silent this minute; busiest connection at {} bytes/s",
tracker.markets.len(),
tracker.connections.len(),
busiest.unwrap_or(0)
);
(added, removed, moved, splits) = (0, 0, 0, 0);
(recycled, quiet_moves, silent) = (0, 0, 0);
}
// Recycling starts once the startup deal is sent.
if first_scan.is_none() && queue.is_empty() {
tracker.start_recycling(now);
}
if let Some(connection) = tracker.recycle(now) {
threads.recycle(connection);
recycled += 1;
}
if !queue.is_empty() && now >= next_deal {
for command in queue.drain(..DEAL_AT_ONCE.min(queue.len())) {
@@ -217,13 +232,26 @@ pub fn run(outputs: Sender<Output>, stop: &Arc<AtomicBool>) -> usize {
// Both fetchers are gone: the check above stops the run.
Err(_) => {}
},
recv(reports) -> loads => {
if let Ok(loads) = loads {
tracker.report(&loads);
recv(reports) -> report => match report {
Ok(Report::Loads(loads)) => tracker.report(&loads),
Ok(Report::Quiet { market, connection }) => {
if let Some(command) = tracker.quiet(&market, connection, Instant::now()) {
log!(
"polymarket: orchestrator: {market} still quiet on connection \
{connection} after a resubscribe, moved"
);
quiet_moves += 1;
moved += 1;
commands.push(command);
}
}
Err(_) => {}
},
recv(closes) -> closed => {
if let Ok(connection) = closed {
recv(notices) -> notice => match notice {
Ok(Notice::Up(connection)) => tracker.up(connection, Instant::now()),
Ok(Notice::Down(connection)) => tracker.down(connection),
Ok(Notice::Silent(_)) => silent += 1,
Ok(Notice::TooSlow(connection)) => {
let moves = tracker.closed(connection, Instant::now());
if !moves.is_empty() {
log!(
@@ -236,6 +264,7 @@ pub fn run(outputs: Sender<Output>, stop: &Arc<AtomicBool>) -> usize {
}
commands.extend(moves);
}
Err(_) => {}
},
default(timeout) => {}
}
@@ -261,7 +290,9 @@ struct Threads {
/// Every book thread's events channel, by number: each connection gets a
/// clone of them all.
events: Vec<Sender<Received>>,
closes: Sender<usize>,
notices: Sender<Notice>,
/// Each connection's commands, by number, for its recycles.
connection_commands: Vec<Sender<Command>>,
book_threads: Vec<JoinHandle<()>>,
connections: Vec<JoinHandle<()>>,
stop: Arc<AtomicBool>,
@@ -270,14 +301,15 @@ struct Threads {
impl Threads {
fn start(
outputs: &Sender<Output>,
reports: &Sender<Loads>,
closes: &Sender<usize>,
reports: &Sender<Report>,
notices: &Sender<Notice>,
stop: &Arc<AtomicBool>,
) -> Self {
let mut threads = Self {
book_commands: Vec::new(),
events: Vec::new(),
closes: closes.clone(),
notices: notices.clone(),
connection_commands: Vec::new(),
book_threads: Vec::new(),
connections: Vec::new(),
stop: Arc::clone(stop),
@@ -302,23 +334,29 @@ impl Threads {
let number = self.connections.len();
let (commands, commands_rx) = unbounded();
let (events, stop) = (self.events.clone(), Arc::clone(&self.stop));
let closes = self.closes.clone();
let notices = self.notices.clone();
self.connections.push(thread::spawn(move || {
socket::run(
socket::MARKET_CHANNEL_URL,
number,
&events,
&commands_rx,
&closes,
&notices,
socket::SILENT_AFTER,
&stop,
)
}));
self.connection_commands.push(commands.clone());
for book_thread in &self.book_commands {
_ = book_thread.send(BookCommand::Connected(number, commands.clone()));
}
}
}
fn recycle(&self, connection: usize) {
_ = self.connection_commands[connection].send(Command::Recycle);
}
/// To the market's book thread. One that died keeps its markets unserved,
/// and its connections stop the run.
fn send(&self, command: BookCommand) {
@@ -360,16 +398,19 @@ struct Tracked {
moved_at: Option<Instant>,
}
/// A connection's tokens and load, against its two limits, and its 1013
/// closes.
/// A connection's tokens and load, against its two limits, its 1013 closes,
/// and whether and since when it is up.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
struct ConnectionLoad {
struct ConnectionState {
tokens: usize,
/// The sum of its markets' loads, bytes per second.
load: u64,
/// When it was last closed with 1013.
closed_at: Option<Instant>,
split_at: Option<Instant>,
up: bool,
/// When it last came up: the one longest ago is recycled next.
connected_at: Option<Instant>,
}
/// Which connections carry each market and what it weighs, and which markets
@@ -379,10 +420,14 @@ struct Tracker {
/// By `conditionId`.
markets: HashMap<String, Tracked>,
/// By connection number.
connections: Vec<ConnectionLoad>,
connections: Vec<ConnectionState>,
/// While a rescan runs: the markets known at its start, not seen since.
unseen: Option<HashSet<String>>,
random: Random,
/// When the next recycle is due; None until recycling starts.
next_recycle: Option<Instant>,
/// The connection recycled and not yet up again.
recycling: Option<usize>,
}
impl Tracker {
@@ -393,6 +438,8 @@ impl Tracker {
connections: Vec::new(),
unseen: None,
random: Random(seed | 1),
next_recycle: None,
recycling: None,
}
}
@@ -410,7 +457,7 @@ impl Tracker {
let count = (self.subscriptions * tokens)
.div_ceil(STARTUP_FILL)
.max(self.subscriptions);
self.connections = vec![ConnectionLoad::default(); count];
self.connections = vec![ConnectionState::default(); count];
let mut placed = vec![Vec::new(); markets.len()];
for _ in 0..self.subscriptions {
let mut order: Vec<usize> = (0..markets.len()).collect();
@@ -474,7 +521,7 @@ impl Tracker {
})
.collect();
if with_room.is_empty() {
self.connections.push(ConnectionLoad::default());
self.connections.push(ConnectionState::default());
return self.connections.len() - 1;
}
with_room[self.random.below(with_room.len())]
@@ -604,6 +651,73 @@ impl Tracker {
.collect()
}
/// A market the book thread found quiet on `connection` again after a
/// resubscribe: its subscription there moves, beyond rebalancing's
/// limits. None if it no longer is there.
fn quiet(&mut self, id: &str, connection: usize, now: Instant) -> Option<BookCommand> {
let tracked = self.markets.get(id)?;
// A market moved lately stays, as for rebalancing: its last move may
// still be under way, and a second one would end it early.
let stays = tracked
.moved_at
.is_some_and(|at| now.duration_since(at) < MOVED_STAYS);
(tracked.connections.contains(&connection) && !stays)
.then(|| self.move_off(id, connection, now))
}
/// A connection came up: its subscribes are sent, its opening books
/// still to come (0.3 to 20 s, docs/notes.md), so no recycle starts for
/// one interval.
fn up(&mut self, connection: usize, now: Instant) {
if let Some(up) = self.connections.get_mut(connection) {
up.up = true;
up.connected_at = Some(now);
}
if self.recycling == Some(connection) {
self.recycling = None;
}
let settled = now + self.recycle_interval();
if let Some(next) = &mut self.next_recycle {
*next = (*next).max(settled);
}
}
fn down(&mut self, connection: usize) {
if let Some(down) = self.connections.get_mut(connection) {
down.up = false;
}
}
fn start_recycling(&mut self, now: Instant) {
if self.next_recycle.is_none() {
self.next_recycle = Some(now + self.recycle_interval());
}
}
/// Each connection about once per [`RECYCLE_EVERY`], one at a time.
fn recycle_interval(&self) -> Duration {
RECYCLE_EVERY / self.connections.len().max(1) as u32
}
/// The connection to recycle now, if one is due: the one up the longest,
/// once the last recycled is up again, and only while every connection is
/// up and has been for an interval, so no market loses both of its
/// connections at once.
fn recycle(&mut self, now: Instant) -> Option<usize> {
let due = self.next_recycle.is_some_and(|next| now >= next);
if !due
|| self.recycling.is_some()
|| self.connections.iter().any(|connection| !connection.up)
{
return None;
}
let oldest = (0..self.connections.len())
.min_by_key(|&number| self.connections[number].connected_at)?;
self.recycling = Some(oldest);
self.next_recycle = Some(now + self.recycle_interval());
Some(oldest)
}
fn scan_started(&mut self) {
self.unseen = Some(self.markets.keys().cloned().collect());
}
@@ -795,7 +909,7 @@ mod tests {
for id in 0..10 {
tracker.add(market(id));
}
tracker.connections.push(ConnectionLoad::default());
tracker.connections.push(ConnectionState::default());
let budget = CONNECTION_BYTES_PER_SECOND;
// Both connections at 1.5 budgets, market 0 weighing half a budget.
let mut loads = vec![(0, budget / 2)];
@@ -831,7 +945,7 @@ mod tests {
tracker.add(market(id));
}
// Room elsewhere, on several connections.
tracker.connections.resize(6, ConnectionLoad::default());
tracker.connections.resize(6, ConnectionState::default());
report(
&mut tracker,
&[(0, CONNECTION_BYTES_PER_SECOND), (1, 1_000)],
@@ -850,8 +964,8 @@ mod tests {
fn the_busiest_connection_over_budget_is_rebalanced_first() {
let mut tracker = Tracker::new(1, 7);
tracker.deal(vec![market(0), market(1)]);
tracker.connections.push(ConnectionLoad::default());
tracker.connections.push(ConnectionLoad::default());
tracker.connections.push(ConnectionState::default());
tracker.connections.push(ConnectionState::default());
// One connection so far: put markets 2 and 3 on another.
for id in [2, 3] {
tracker.connections[0].tokens = CONNECTION_TOKENS;
@@ -931,6 +1045,69 @@ mod tests {
assert_eq!(moves(&commands), [("0x2", 1, 2)]);
}
#[test]
fn recycling_takes_the_oldest_connection_one_at_a_time_while_every_one_is_up() {
let mut tracker = tracker();
tracker.connections = vec![ConnectionState::default(); 3];
let start = Instant::now();
let at = |seconds: u64| start + Duration::from_secs(seconds);
// Up in the order 2, 0, 1.
for (seconds, connection) in [(0, 2), (1, 0), (2, 1)] {
tracker.up(connection, at(seconds));
}
assert_eq!(tracker.recycle(at(5_000)), None, "not started");
tracker.start_recycling(at(10));
// One every hour / 3 connections.
assert_eq!(tracker.recycle(at(1_209)), None);
assert_eq!(tracker.recycle(at(1_210)), Some(2));
// The next waits for the recycled one, whose Down may not even have
// come yet, to be up again...
assert_eq!(tracker.recycle(at(2_410)), None);
tracker.down(2);
assert_eq!(tracker.recycle(at(2_410)), None);
tracker.up(2, at(2_411));
// ...and for its opening books: an interval more.
assert_eq!(tracker.recycle(at(3_610)), None);
assert_eq!(tracker.recycle(at(3_611)), Some(0));
tracker.down(0);
tracker.up(0, at(3_612));
// A connection down for its own reasons holds every recycle, and its
// return delays the next like any other.
tracker.down(2);
assert_eq!(tracker.recycle(at(6_000)), None);
tracker.up(2, at(6_001));
assert_eq!(tracker.recycle(at(7_200)), None);
assert_eq!(tracker.recycle(at(7_201)), Some(1));
}
#[test]
fn a_market_quiet_again_moves_off_that_connection_unless_it_moved_lately() {
let mut tracker = tracker();
tracker.deal((0..50).map(market).collect());
let now = Instant::now();
let on = connections(&tracker, 0);
let first = [tracker.quiet("0x0", on[0], now).expect("a move")];
let [(id, from, to)] = moves(&first)[..] else {
panic!("one move");
};
assert_eq!((id, from), ("0x0", on[0]));
assert!(!on.contains(&to));
// Moved a minute ago, its move perhaps still under way: it stays,
// as for rebalancing.
let on = connections(&tracker, 0);
let soon = now + Duration::from_secs(60);
assert!(tracker.quiet("0x0", on[1], soon).is_none());
assert!(tracker.quiet("0x0", on[1], now + MOVED_STAYS).is_some());
let now = now + MOVED_STAYS + MOVED_STAYS;
// Reported for a connection it isn't on (any more): nothing.
let on = connections(&tracker, 0);
let elsewhere = (0..tracker.connections.len())
.find(|connection| !on.contains(connection))
.unwrap();
assert!(tracker.quiet("0x0", elsewhere, now).is_none());
assert!(tracker.quiet("0xunknown", 0, now).is_none());
}
#[test]
fn a_connection_closed_twice_within_ten_minutes_is_split() {
let mut tracker = Tracker::new(1, 7);
+171 -18
View File
@@ -1,6 +1,7 @@
//! A connection: reads the market channel, stamps each frame's receive time,
//! decodes it and sends each event to the book thread of its market; and
//! sends the exchange what the book threads ask for (subscriptions, resyncs).
//! It reconnects when it drops, falls silent, or is recycled.
use std::collections::BTreeSet;
use std::io::ErrorKind;
@@ -29,6 +30,9 @@ const READ_TIMEOUT: Duration = Duration::from_millis(200);
const PING_EVERY: Duration = Duration::from_secs(10);
const RECONNECT_AFTER: Duration = Duration::from_secs(1);
const CLOSE_TIMEOUT: Duration = Duration::from_secs(2);
/// Nothing received this long, not even the answer to three keepalives: the
/// connection is silent, and reconnects.
pub const SILENT_AFTER: Duration = Duration::from_secs(30);
/// Tokens per subscribe or unsubscribe message: a first message of more than
/// 500 gets no opening books (docs/notes.md).
const CHUNK: usize = 500;
@@ -39,27 +43,46 @@ pub enum Command {
Subscribe(Vec<String>),
Unsubscribe(Vec<String>),
/// Unsubscribe and subscribe again: the server then sends a fresh book.
Resync(String),
/// A resync asks for that book; a quiet subscription only for the new
/// subscription.
Resubscribe(String),
/// Close and reconnect, subscribing every token again.
Recycle,
}
/// What a connection tells the orchestrator.
#[derive(Debug, PartialEq, Eq)]
pub enum Notice {
/// Connected, its tokens subscribed.
Up(usize),
Down(usize),
/// Closed by the server with 1013: it couldn't send to us fast enough.
TooSlow(usize),
/// Closed by us: nothing came for [`SILENT_AFTER`].
Silent(usize),
}
type Socket = WebSocket<MaybeTlsStream<TcpStream>>;
/// Runs until `stop` is set or a book thread is gone, reconnecting (and
/// subscribing its tokens again) whenever the connection drops, and telling
/// every book thread when it comes up and goes down. `book_threads` holds
/// every book thread's channel, by number; `closes` hears of each 1013 close.
/// subscribing its tokens again) whenever the connection drops, falls silent
/// for `silent_after` or is recycled, and telling every book thread and the
/// orchestrator (`notices`) when it comes up and goes down. `book_threads`
/// holds every book thread's channel, by number.
pub fn run(
url: &str,
number: usize,
book_threads: &[Sender<Received>],
commands: &Receiver<Command>,
closes: &Sender<usize>,
notices: &Sender<Notice>,
silent_after: Duration,
stop: &AtomicBool,
) {
let tell = |received: fn(usize) -> Received| {
let tell = |received: fn(usize) -> Received, notice: fn(usize) -> Notice| {
for book_thread in book_threads {
_ = book_thread.send(received(number));
}
_ = notices.send(notice(number));
};
let mut tokens = BTreeSet::new();
while !stop.load(Ordering::Relaxed) {
@@ -74,21 +97,23 @@ pub fn run(
"polymarket: connection {number}: connected, {} symbols",
tokens.len()
);
tell(Received::Up);
tell(Received::Up, Notice::Up);
let ended = read(
&mut socket,
number,
&mut tokens,
book_threads,
commands,
closes,
notices,
silent_after,
stop,
);
if !matches!(ended, Ok(Ended::Stopped)) {
tell(Received::Down);
tell(Received::Down, Notice::Down);
}
match ended {
Ok(Ended::Stopped) => return close(&mut socket, number, book_threads),
Ok(Ended::Recycled) => log!("polymarket: connection {number}: recycled"),
Ok(Ended::BookThreadGone) => {
// Only a panic ends a book thread before us: its
// markets would go unserved, so the run stops.
@@ -109,6 +134,7 @@ pub fn run(
enum Ended {
Stopped,
BookThreadGone,
Recycled,
}
/// Updates the token set, and returns the messages that carry `command` out
@@ -127,12 +153,14 @@ fn apply(tokens: &mut BTreeSet<String>, command: &Command) -> Vec<String> {
// Subscribing a token already subscribed sends nothing back but
// `NO NEW ASSETS`: unsubscribing first makes the server send a fresh
// book.
Command::Resync(token) if tokens.contains(token) => [
Command::Resubscribe(token) if tokens.contains(token) => [
operations("unsubscribe", [token]),
operations("subscribe", [token]),
]
.concat(),
Command::Resync(_) => Vec::new(),
Command::Resubscribe(_) => Vec::new(),
// The read loop acts on it; disconnected, the reconnect does.
Command::Recycle => Vec::new(),
}
}
@@ -219,20 +247,27 @@ fn truncate(text: &str) -> &str {
.map_or(text, |(end, _)| &text[..end])
}
// The connection's whole context, each a different part of it.
#[allow(clippy::too_many_arguments)]
fn read(
socket: &mut Socket,
number: usize,
tokens: &mut BTreeSet<String>,
book_threads: &[Sender<Received>],
commands: &Receiver<Command>,
closes: &Sender<usize>,
notices: &Sender<Notice>,
silent_after: Duration,
stop: &AtomicBool,
) -> anyhow::Result<Ended> {
let mut last_ping = Instant::now();
let mut last_heard = Instant::now();
loop {
let message = socket.read();
// Stamp as soon as the frame is off the wire, before any processing.
let received_at = Utc::now();
if message.is_ok() {
last_heard = Instant::now();
}
match message {
Ok(Message::Text(text)) => {
if !route(received_at, text.as_str(), number, book_threads) {
@@ -246,7 +281,7 @@ fn read(
.as_ref()
.is_some_and(|frame| frame.code == CloseCode::Again)
{
_ = closes.send(number);
_ = notices.send(Notice::TooSlow(number));
}
anyhow::bail!("closed by the server: {frame:?}")
}
@@ -258,11 +293,23 @@ fn read(
if stop.load(Ordering::Relaxed) {
return Ok(Ended::Stopped);
}
if last_heard.elapsed() >= silent_after {
_ = notices.send(Notice::Silent(number));
anyhow::bail!("silent for {} s", silent_after.as_secs_f64());
}
if last_ping.elapsed() >= PING_EVERY {
socket.send(Message::text("PING"))?;
last_ping = Instant::now();
}
for command in commands.try_iter() {
if command == Command::Recycle {
// Not waited on: whatever still comes would arrive from a
// connection already down for the book threads. The commands
// behind it are applied before the reconnect.
_ = socket.close(None);
_ = socket.flush();
return Ok(Ended::Recycled);
}
for message in apply(tokens, &command) {
socket.send(Message::text(message))?;
}
@@ -351,9 +398,18 @@ mod tests {
.unwrap();
let stop = Arc::new(AtomicBool::new(false));
let client_stop = Arc::clone(&stop);
let (closes, _closes_rx) = unbounded();
let client =
thread::spawn(move || run(&url, 0, &[events_tx], &commands_rx, &closes, &client_stop));
let (notices, _notices_rx) = unbounded();
let client = thread::spawn(move || {
run(
&url,
0,
&[events_tx],
&commands_rx,
&notices,
SILENT_AFTER,
&client_stop,
)
});
let connections = server.join().unwrap();
stop.store(true, Ordering::Relaxed);
@@ -368,6 +424,103 @@ mod tests {
}
}
/// A local server that accepts two connections of `total` tokens, each
/// handled by `serve` once subscribed, and answers what the client
/// subscribed on each.
fn serving_twice(
total: usize,
serve: fn(&mut WebSocket<TcpStream>),
) -> (String, thread::JoinHandle<Vec<Vec<Vec<String>>>>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("ws://{}/ws/market", listener.local_addr().unwrap());
let server = thread::spawn(move || {
(0..2)
.map(|_| {
let (stream, _) = listener.accept().unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
let mut socket = tungstenite::accept(stream).unwrap();
let chunks = subscribed(&mut socket, total);
serve(&mut socket);
chunks
})
.collect()
});
(url, server)
}
/// Runs a connection of `tokens` against `url` until `until` returns,
/// then stops it: the notices it sent.
fn client(
url: String,
tokens: Vec<String>,
silent_after: Duration,
until: impl FnOnce(&Sender<Command>),
) -> Vec<Notice> {
let (events_tx, _events_rx) = unbounded();
let (commands_tx, commands_rx) = unbounded();
commands_tx.send(Command::Subscribe(tokens)).unwrap();
let stop = Arc::new(AtomicBool::new(false));
let client_stop = Arc::clone(&stop);
let (notices, notices_rx) = unbounded();
let client = thread::spawn(move || {
run(
&url,
0,
&[events_tx],
&commands_rx,
&notices,
silent_after,
&client_stop,
)
});
until(&commands_tx);
stop.store(true, Ordering::Relaxed);
client.join().unwrap();
notices_rx.try_iter().collect()
}
#[test]
fn a_silent_connection_reconnects_with_its_tokens() {
let tokens: Vec<String> = (0..10).map(|token| token.to_string()).collect();
// Silent: reads what comes (the keepalives) and answers nothing,
// until the client closes.
let (url, server) = serving_twice(tokens.len(), |socket| while socket.read().is_ok() {});
let notices = client(url, tokens.clone(), Duration::from_millis(500), |_| {
thread::sleep(Duration::from_millis(2_500));
});
let connections = server.join().unwrap();
assert_eq!(connections.len(), 2);
assert!(connections.iter().all(|chunks| chunks.concat() == tokens));
assert_eq!(
notices[..3],
[Notice::Up(0), Notice::Silent(0), Notice::Down(0)]
);
assert_eq!(notices[3], Notice::Up(0));
}
#[test]
fn a_recycle_closes_and_subscribes_every_token_again() {
let tokens: Vec<String> = (0..10).map(|token| token.to_string()).collect();
// Answers the keepalives, as the exchange does, until closed.
let (url, server) = serving_twice(tokens.len(), |socket| {
while let Ok(message) = socket.read() {
if message.is_text() {
_ = socket.send(Message::text("PONG"));
}
}
});
let notices = client(url, tokens.clone(), SILENT_AFTER, |commands| {
thread::sleep(Duration::from_millis(300));
commands.send(Command::Recycle).unwrap();
thread::sleep(Duration::from_millis(1_800));
});
let connections = server.join().unwrap();
assert!(connections.iter().all(|chunks| chunks.concat() == tokens));
assert_eq!(notices, [Notice::Up(0), Notice::Down(0), Notice::Up(0)]);
}
/// The recording's first frame holds the opening books of 40 tokens, 20
/// markets: each book goes to its market's book thread, with its share of
/// the frame's bytes.
@@ -433,7 +586,7 @@ mod tests {
[op("subscribe", &["a", "b"])]
);
assert_eq!(
sent(apply(&mut tokens, &Command::Resync("a".into()))),
sent(apply(&mut tokens, &Command::Resubscribe("a".into()))),
[op("unsubscribe", &["a"]), op("subscribe", &["a"])]
);
let unsubscribe = Command::Unsubscribe(vec!["a".into()]);
@@ -441,7 +594,7 @@ mod tests {
sent(apply(&mut tokens, &unsubscribe)),
[op("unsubscribe", &["a"])]
);
assert!(apply(&mut tokens, &Command::Resync("a".into())).is_empty());
assert!(apply(&mut tokens, &Command::Resubscribe("a".into())).is_empty());
assert_eq!(tokens.into_iter().collect::<Vec<_>>(), ["b"]);
}
}