feat(polymarket): track every live market over load-balanced lanes

An orchestrator keeps the tracked set equal to Gamma's live markets (a scan at
startup and every hour, new and closed markets every minute, reaching back 15
minutes for Gamma's indexing delay) and spreads them over lanes, each a
connection and a book thread, all writing through the one writer. Lanes report
each market's load in bytes per second; the orchestrator puts new markets on
the least-loaded lane and moves the busiest markets off lanes over a 300 KB/s
budget, busiest lane first. Differing books our own subscribe explains are
subscription gaps, not losses, and every first book after a move is logged,
so reconcile's mismatches are all accounted for in the log. A 75-minute live
run tracked ~195k markets (~390k symbols) in 89 lanes.
This commit is contained in:
pascal committed 2026-11-11 06:28:25 +00:00
1 parent adb7fee8d8
commit 8114a51dc2
18 files changed
+2826 -320

No files matched your search

+542 -105
View File
@@ -1,32 +1,58 @@
//! The book thread: parses market-channel frames, folds them into one book per
//! symbol, checks them against Polymarket's book hash, resyncs when a check
//! fails, and emits Deltas and Snapshots. The rules are in
//! `roadmap/02. book pipeline.md`; the hash formula in `docs/notes.md`.
//! fails, and emits Deltas and Snapshots. The protocol facts and the hash
//! formula are in `docs/notes.md`, the terms (loss, resync, subscription gap,
//! load) in `GLOSSARY.md`.
use std::collections::{BTreeMap, VecDeque};
use std::collections::{BTreeMap, HashMap, VecDeque};
use std::time::Duration;
use chrono::{DateTime, DurationRound, TimeDelta, Utc};
use crossbeam::channel::{Receiver, RecvTimeoutError, Sender};
use crossbeam::channel::{Receiver, Sender, never, select};
use engineering_challenge::Side;
use engineering_challenge::util::dtos::challenge_orderbook::PriceLevelUpdateDomain;
use serde::{Deserialize, Serialize};
use sha1::{Digest, Sha1};
use super::Market;
use super::socket::Frame;
use super::socket::{Command, Frame};
use engineering_challenge::book::{Book, Output, format_fixed, parse_fixed};
/// A market measured for less than this waits for the next load report: over
/// a few seconds, its opening books alone would read as a burst.
const MIN_MEASURED: TimeDelta = TimeDelta::seconds(30);
/// How far back a full book is recognised as behind our stream: the lag seen
/// live is a few ms, a handful of changes.
const RECENT_CHANGES: usize = 64;
/// What the book thread does with a frame: write output, or ask the socket
/// thread for a resync.
/// What the book thread does with a frame or a command: write output, ask its
/// connection for something, or report its load to the orchestrator.
#[derive(Debug)]
pub enum Effect {
Write(Output),
Resync(String),
Send(Command),
Report(Loads),
}
/// Each market's load (`conditionId` → bytes per second), every market of the
/// lane measured for at least [`MIN_MEASURED`].
pub type Loads = HashMap<String, u64>;
/// A lane's loads since its last report, for the orchestrator.
#[derive(Debug)]
pub struct LoadReport {
pub lane: usize,
pub loads: Loads,
}
/// What the orchestrator tells a lane's book thread.
#[derive(Debug)]
pub enum LaneCommand {
Add(Market),
/// An Add for a market moved here from another lane.
MoveIn(Market),
Remove(Market),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -34,7 +60,7 @@ enum State {
/// No book yet since subscribing: changes are dropped until it comes.
AwaitingBook,
/// Its last full book matched its hash: every change is checked.
Live,
Verified,
/// A check failed and a fresh book is asked for: changes are applied and
/// written unchecked until it comes. After a trade, it usually equals ours.
Resyncing,
@@ -53,6 +79,12 @@ struct Symbol {
/// Its last [`RECENT_CHANGES`] changes: a full book carrying the hash of
/// one of them is the exchange's state after it.
recent_changes: VecDeque<RecentChange>,
/// Whether a change has been verified since its opening book: until one
/// is, a differing fresh book is the gap our subscribe opened.
verified_since_book: bool,
/// Moved in from another lane, no book yet: its first book holds the
/// move's subscription gap, which only the old lane's book would show.
moved_in: bool,
// The other inputs of the book hash, as the exchange writes them.
market: String,
tick_size: String,
@@ -62,6 +94,30 @@ struct Symbol {
}
impl Symbol {
fn new(market: &Market) -> Self {
Self {
state: State::AwaitingBook,
book: Book::default(),
exchange_timestamp: DateTime::UNIX_EPOCH,
open_batch: None,
recent_changes: VecDeque::new(),
verified_since_book: false,
moved_in: false,
market: String::new(),
tick_size: String::new(),
last_trade_price: String::new(),
min_order_size: market.min_order_size.clone(),
neg_risk: market.neg_risk,
}
}
/// A change matched the exchange's hash: counted in `verified_changes`,
/// and the book was right since its opening book.
fn verified(&mut self, verified_changes: &mut u64) {
*verified_changes += 1;
self.verified_since_book = true;
}
/// Asks for a fresh book, unless one is already coming.
fn resync(&mut self, token: &str, effects: &mut Vec<Effect>) {
if matches!(self.state, State::AwaitingBook | State::Resyncing) {
@@ -77,7 +133,7 @@ impl Symbol {
self.state = State::Resyncing;
}
self.open_batch = None;
effects.push(Effect::Resync(token.to_string()));
effects.push(Effect::Send(Command::Resync(token.to_string())));
}
/// Whether `book` is ours as it was after one of our recent changes: ours
@@ -102,6 +158,29 @@ impl Symbol {
}
}
/// A market's frame bytes since `since`.
struct Measure {
bytes: u64,
since: DateTime<Utc>,
}
impl Measure {
/// Bytes per second since `since`, then starts again from `now`; none
/// before [`MIN_MEASURED`], the bytes kept for the next report.
fn take(&mut self, now: DateTime<Utc>) -> Option<u64> {
let measured = now - self.since;
if measured < MIN_MEASURED {
return None;
}
let load = self.bytes * 1_000 / measured.num_milliseconds() as u64;
*self = Measure {
bytes: 0,
since: now,
};
Some(load)
}
}
/// A change as applied, with the exchange's hash of the book after it.
struct RecentChange {
side: Side,
@@ -113,17 +192,25 @@ struct RecentChange {
}
pub struct Feed {
/// Its lane's number, for the logs.
lane: usize,
symbols: BTreeMap<String, Symbol>,
/// Bytes of frames per market since its last load report.
measures: HashMap<String, Measure>,
/// The receive time of the last frame applied: the time a Snapshot's book
/// is exact as of.
last_received_at: Option<DateTime<Utc>>,
next_snapshots: Option<DateTime<Utc>>,
/// The next wall-clock minute: its Snapshots, load report and log line.
next_minute_at: Option<DateTime<Utc>>,
verified_changes: u64,
resyncs: u64,
/// Full books that were a state we had already passed (see `on_book`).
books_behind: u64,
/// Fresh books ahead of our stream by changes our resync kept from us.
resync_gaps: u64,
/// Fresh books holding changes our own subscribe kept from us.
subscription_gaps: u64,
/// First books of markets moved in, each logged: whether one holds a
/// subscription gap only the old lane's book would tell.
books_after_moves: u64,
/// Full books whose levels differed from ours otherwise.
losses: u64,
/// The most frames waiting for the book thread since the last minute log:
@@ -132,42 +219,56 @@ pub struct Feed {
}
impl Feed {
pub fn new(markets: &[Market]) -> Self {
let symbols = markets
.iter()
.flat_map(|market| {
market.tokens.iter().map(|token| {
let symbol = Symbol {
state: State::AwaitingBook,
book: Book::default(),
exchange_timestamp: DateTime::UNIX_EPOCH,
open_batch: None,
recent_changes: VecDeque::new(),
market: String::new(),
tick_size: String::new(),
last_trade_price: String::new(),
min_order_size: market.min_order_size.clone(),
neg_risk: market.neg_risk,
};
(token.clone(), symbol)
})
})
.collect();
pub fn new(lane: usize) -> Self {
Self {
symbols,
lane,
symbols: BTreeMap::new(),
measures: HashMap::new(),
last_received_at: None,
next_snapshots: None,
next_minute_at: None,
verified_changes: 0,
resyncs: 0,
books_behind: 0,
resync_gaps: 0,
subscription_gaps: 0,
books_after_moves: 0,
losses: 0,
peak_waiting: 0,
}
}
pub fn tokens(&self) -> Vec<String> {
self.symbols.keys().cloned().collect()
/// Tracks a market's symbols, then asks the connection to subscribe them:
/// a book can't arrive for a symbol not yet known. Its load is measured
/// from `now`.
pub fn add_market(&mut self, market: &Market, now: DateTime<Utc>) -> Vec<Effect> {
for token in &market.tokens {
self.symbols.insert(token.clone(), Symbol::new(market));
}
let measure = Measure {
bytes: 0,
since: now,
};
self.measures.insert(market.condition_id.clone(), measure);
vec![Effect::Send(Command::Subscribe(market.tokens.clone()))]
}
/// [`Feed::add_market`] for a market moved in from another lane.
pub fn move_in(&mut self, market: &Market, now: DateTime<Utc>) -> Vec<Effect> {
let effects = self.add_market(market, now);
for token in &market.tokens {
self.symbols.get_mut(token).expect("just added").moved_in = true;
}
effects
}
/// Forgets a market's symbols and unsubscribes them; nothing is written
/// for them. Their frames still on the way are ignored, as any unknown
/// token's.
pub fn remove_market(&mut self, market: &Market) -> Vec<Effect> {
for token in &market.tokens {
self.symbols.remove(token);
}
self.measures.remove(&market.condition_id);
vec![Effect::Send(Command::Unsubscribe(market.tokens.clone()))]
}
pub fn on_frame(&mut self, received_at: DateTime<Utc>, text: &str) -> Vec<Effect> {
@@ -186,51 +287,74 @@ impl Feed {
}
},
};
// A frame is one market's change, or books of several: each book takes
// its share.
let share = (text.len() / events.len().max(1)) as u64;
for event in events {
let market = match &event {
Event::Book(book) => Some(&book.market),
Event::PriceChange(change) => Some(&change.market),
Event::TickSizeChange { market, .. } | Event::LastTradePrice { market } => {
market.as_ref()
}
Event::Other => None,
};
if let Some(measure) = market.and_then(|market| self.measures.get_mut(market)) {
measure.bytes += share;
}
match event {
Event::Book(book) => self.on_book(received_at, book, &mut effects),
Event::PriceChange(change) => {
self.on_price_change(received_at, change, &mut effects)
}
Event::TickSizeChange { asset_id } => {
Event::TickSizeChange { asset_id, .. } => {
if let Some(symbol) = self.symbols.get_mut(&asset_id) {
log!("polymarket: {asset_id}: tick size changed, resync");
symbol.resync(&asset_id, &mut effects);
}
}
// Among them `last_trade_price`: the hash takes a new last
// trade price before this event, and not always its value, so
// a fresh book brings it instead (docs/notes.md).
Event::Other => {}
// The hash takes a new last trade price before this event,
// and not always its value, so a fresh book brings it instead
// (docs/notes.md).
Event::LastTradePrice { .. } | Event::Other => {}
}
}
self.resyncs += count_resyncs(&effects[ticked..]);
effects
}
/// Once `now` reaches the next wall-clock minute: emits the minute
/// Snapshots, and asks again for the fresh books still not come.
/// Once `now` reaches the next wall-clock minute: reports the minute's
/// load, emits the minute Snapshots, and asks again for the fresh books
/// still not come.
pub fn tick(&mut self, now: DateTime<Utc>) -> Vec<Effect> {
let next = *self.next_snapshots.get_or_insert_with(|| next_minute(now));
let next = *self.next_minute_at.get_or_insert_with(|| next_minute(now));
if now < next {
return Vec::new();
}
self.next_snapshots = Some(next_minute(now));
self.next_minute_at = Some(next_minute(now));
let loads = self
.measures
.iter_mut()
.filter_map(|(market, measure)| Some((market.clone(), measure.take(now)?)))
.collect();
let mut effects = vec![Effect::Report(loads)];
let Some(received_at) = self.last_received_at else {
return Vec::new();
return effects;
};
log!(
"polymarket: so far {} changes verified, {} resyncs, {} full books behind ours, \
{} resync gaps, {} losses; at most {} frames waited this minute",
"polymarket: lane {}: {} symbols; so far {} changes verified, {} resyncs, {} full books behind ours, \
{} subscription gaps, {} books after moves, {} losses; at most {} frames waited this minute",
self.lane,
self.symbols.len(),
self.verified_changes,
self.resyncs,
self.books_behind,
self.resync_gaps,
self.subscription_gaps,
self.books_after_moves,
self.losses,
self.peak_waiting
);
self.peak_waiting = 0;
let mut effects = Vec::new();
for (token, symbol) in &mut self.symbols {
if symbol.state != State::AwaitingBook {
effects.push(Effect::Write(symbol.book.snapshot(
@@ -249,8 +373,8 @@ impl Feed {
/// A full book replaces ours, brought forward first if it is behind our
/// stream, and is written as a Snapshot when it is the symbol's first
/// since subscribing, or its levels differ from ours (a resync gap or a
/// loss). Then it is checked against its own hash, as received.
/// since subscribing, or its levels differ from ours (a subscription gap
/// or a loss). Then it is checked against its own hash, as received.
fn on_book(
&mut self,
received_at: DateTime<Utc>,
@@ -285,6 +409,13 @@ impl Feed {
let verified = hash(&token, symbol, &book, &message.timestamp) == message.hash;
let first = symbol.state == State::AwaitingBook;
if first && symbol.moved_in {
// The check matches it with the Snapshot below if that differs
// from the old lane's last.
symbol.moved_in = false;
self.books_after_moves += 1;
log!("polymarket: {token}: first book after a move, a subscription gap if it differs");
}
let mut book = book;
let book_time = exchange_timestamp;
let mut exchange_timestamp = exchange_timestamp;
@@ -312,17 +443,24 @@ impl Feed {
brought_forward = true;
}
let differs = !first && book != symbol.book;
// A fresh book answering our resync can also be ahead: between our
// unsubscribe and subscribe the server sent us none of its changes.
let gap = differs
&& !brought_forward
// A subscribe hides a few ms of changes (docs/notes.md): a fresh book
// answering our resync can be ahead of our stream, since between our
// unsubscribe and subscribe the server sent none of its changes; and
// so can any book before a change was verified since the opening one.
let opening = !symbol.verified_since_book;
let resync = !brought_forward
&& complete
&& symbol.state == State::Resyncing
&& book_time >= symbol.exchange_timestamp;
let kind = if gap { "resync gap" } else { "loss" };
let gap = differs && (opening || resync);
let kind = match (gap, opening) {
(false, _) => "loss",
(true, true) => "subscription gap (opening)",
(true, false) => "subscription gap (resync)",
};
if differs {
if gap {
self.resync_gaps += 1;
self.subscription_gaps += 1;
} else {
self.losses += 1;
}
@@ -347,7 +485,7 @@ impl Feed {
}
if verified {
symbol.state = State::Live;
symbol.state = State::Verified;
} else if complete {
if symbol.state != State::Unverifiable {
log!(
@@ -355,10 +493,10 @@ impl Feed {
);
}
symbol.state = State::Unverifiable;
} else if matches!(symbol.state, State::AwaitingBook | State::Live) {
// Its last trade price may be stale: a fresh book carries it. Live
// first, since it has a book now, which resync() requires.
symbol.state = State::Live;
} else if matches!(symbol.state, State::AwaitingBook | State::Verified) {
// Its last trade price may be stale: a fresh book carries it.
// Verified first, since it has a book now, which resync() requires.
symbol.state = State::Verified;
symbol.resync(&token, effects);
}
}
@@ -384,7 +522,7 @@ impl Feed {
.take_if(|(_, batch_hash)| *batch_hash != change.hash)
{
if hash(&token, symbol, &symbol.book, &timestamp) == batch_hash {
self.verified_changes += 1;
symbol.verified(&mut self.verified_changes);
} else {
symbol.resync(&token, effects);
}
@@ -416,9 +554,9 @@ impl Feed {
hash: change.hash.clone(),
});
symbol.exchange_timestamp = exchange_timestamp;
if symbol.state == State::Live {
if symbol.state == State::Verified {
if hash(&token, symbol, &symbol.book, &message.timestamp) == change.hash {
self.verified_changes += 1;
symbol.verified(&mut self.verified_changes);
} else {
symbol.open_batch = Some((message.timestamp.clone(), change.hash));
}
@@ -435,8 +573,8 @@ impl Feed {
}
}
/// The book thread: runs until the socket thread is gone, then drops its
/// senders so the writer thread finishes too.
/// A lane's book thread: runs until its connection is gone, then drops its
/// senders so the writer thread finishes once every lane has.
// ponytail: unbounded channels never block the socket read nor drop a frame;
// the ceiling is memory if a consumer stays behind. The minute log's peak of
// frames waiting makes that visible; bounded channels are a constructor change
@@ -444,20 +582,41 @@ impl Feed {
pub fn run(
mut feed: Feed,
frames: Receiver<Frame>,
commands: Receiver<LaneCommand>,
outputs: Sender<Output>,
resyncs: Sender<String>,
connection: Sender<Command>,
reports: Sender<LoadReport>,
) {
let no_commands = never();
let mut orchestrator_gone = false;
loop {
let until_next_minute = (next_minute(Utc::now()) - Utc::now())
.to_std()
.unwrap_or(Duration::ZERO);
let effects = match frames.recv_timeout(until_next_minute) {
Ok((received_at, text)) => {
feed.peak_waiting = feed.peak_waiting.max(frames.len());
feed.on_frame(received_at, text.as_str())
}
Err(RecvTimeoutError::Timeout) => feed.tick(Utc::now()),
Err(RecvTimeoutError::Disconnected) => return,
let commands = if orchestrator_gone {
&no_commands
} else {
&commands
};
let effects = select! {
recv(frames) -> frame => match frame {
Ok((received_at, text)) => {
feed.peak_waiting = feed.peak_waiting.max(frames.len());
feed.on_frame(received_at, text.as_str())
}
Err(_) => return,
},
recv(commands) -> command => match command {
Ok(LaneCommand::Add(market)) => feed.add_market(&market, Utc::now()),
Ok(LaneCommand::MoveIn(market)) => feed.move_in(&market, Utc::now()),
Ok(LaneCommand::Remove(market)) => feed.remove_market(&market),
// It is stopping: the connection's end still decides ours.
Err(_) => {
orchestrator_gone = true;
Vec::new()
}
},
default(until_next_minute) => feed.tick(Utc::now()),
};
for effect in effects {
match effect {
@@ -466,8 +625,15 @@ pub fn run(
return log!("polymarket: the writer thread is gone, stopping");
}
}
// The socket thread may be gone already, while stopping.
Effect::Resync(token) => _ = resyncs.send(token),
// The connection may be gone already, while stopping, and the
// orchestrator too.
Effect::Send(command) => _ = connection.send(command),
Effect::Report(loads) => {
_ = reports.send(LoadReport {
lane: feed.lane,
loads,
})
}
}
}
}
@@ -531,11 +697,19 @@ enum Events {
enum Event {
Book(BookMessage),
PriceChange(PriceChangeMessage),
/// Its wire shape was never observed: only the symbol is read, and it
/// resyncs, since the fresh book carries the new tick size.
/// Only the symbol and its market are read, and it resyncs, since the fresh book carries
/// the new tick size. The server sends each one twice: the second finds
/// the symbol resyncing and asks for nothing (docs/notes.md).
/// `market` was on every one seen; without it, only the load misses it.
TickSizeChange {
market: Option<String>,
asset_id: String,
},
LastTradePrice {
market: Option<String>,
},
/// Not counted in any load: the others come only with
/// `custom_feature_enabled`, which we don't ask for.
#[serde(other)]
Other,
}
@@ -560,6 +734,7 @@ struct LevelMessage {
#[derive(Deserialize)]
struct PriceChangeMessage {
market: String,
timestamp: String,
price_changes: Vec<ChangeMessage>,
}
@@ -600,7 +775,7 @@ fn next_minute(now: DateTime<Utc>) -> DateTime<Utc> {
fn count_resyncs(effects: &[Effect]) -> u64 {
effects
.iter()
.filter(|effect| matches!(effect, Effect::Resync(_)))
.filter(|effect| matches!(effect, Effect::Send(Command::Resync(_))))
.count() as u64
}
@@ -615,6 +790,17 @@ mod tests {
use super::*;
use crate::polymarket::fixture_markets;
/// A lane's feed holding `markets`, as the orchestrator's Adds leave it
/// at the start of the recording.
fn feed_of(markets: &[Market]) -> Feed {
let mut feed = Feed::new(0);
let start = fixture_frames()[0].0;
for market in markets {
feed.add_market(market, start);
}
feed
}
/// The recorded live feed for the fixture's markets (tests/fixtures/README.md).
fn fixture_frames() -> Vec<(DateTime<Utc>, String)> {
#[derive(Deserialize)]
@@ -653,8 +839,8 @@ mod tests {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Resync(token) => Some(token.as_str()),
Effect::Write(_) => None,
Effect::Send(Command::Resync(token)) => Some(token.as_str()),
_ => None,
})
.collect()
}
@@ -685,7 +871,7 @@ mod tests {
/// the recording must replay without a single loss.
#[test]
fn replay_has_no_loss() {
let mut feed = Feed::new(&fixture_markets());
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &fixture_frames());
assert_eq!(feed.losses, 0);
assert!(
@@ -712,7 +898,7 @@ mod tests {
let markets = fixture_markets();
// The full replay, noting the change frames whose symbols are all
// checked when they come.
let mut control = Feed::new(&markets);
let mut control = feed_of(&markets);
let mut candidates = Vec::new();
let control_effects: Vec<Vec<Effect>> = frames
.iter()
@@ -723,7 +909,7 @@ mod tests {
&& !tokens.is_empty()
&& tokens
.iter()
.all(|token| control.symbols[token].state == State::Live)
.all(|token| control.symbols[token].state == State::Verified)
{
candidates.push(index);
}
@@ -733,7 +919,7 @@ mod tests {
let lost = candidates[0];
let lost_tokens = changed_tokens(&frames[lost].1);
let mut feed = Feed::new(&markets);
let mut feed = feed_of(&markets);
replay(&mut feed, &frames[..lost]);
let (caught, token) = frames
.iter()
@@ -766,7 +952,7 @@ mod tests {
let effects = feed.on_frame(frames[caught].0, &book.to_string());
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!(feed.losses, losses + 1);
assert_eq!(feed.symbols[&token].state, State::Live);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
/// A feed replayed past a mid-stream book and its symbol's next change, and
@@ -788,7 +974,7 @@ mod tests {
let moved_on = (stale + 1..frames.len())
.find(|&index| changed_tokens(&frames[index].1).contains(&token))
.unwrap();
let mut feed = Feed::new(&fixture_markets());
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &frames[..=moved_on]);
(feed, token, frames[moved_on].0, frames[stale].1.clone())
}
@@ -839,25 +1025,26 @@ mod tests {
assert!(snapshot_symbols(&effects).is_empty());
assert_eq!(
(feed.books_behind, feed.resync_gaps, feed.losses),
(feed.books_behind, feed.subscription_gaps, feed.losses),
(1, 0, 0)
);
assert_eq!(feed.symbols[&token].book, ours);
assert_eq!(feed.symbols[&token].state, State::Live);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
/// Between our unsubscribe and subscribe the server sends no changes, so
/// the fresh book can hold some we never got: a resync gap, not a loss.
/// the fresh book can hold some we never got: a subscription gap, not a
/// loss.
#[test]
fn a_fresh_book_ahead_of_our_stream_is_a_resync_gap() {
fn a_fresh_book_ahead_of_our_stream_is_a_subscription_gap() {
let frames = fixture_frames();
let now = frames.last().unwrap().0;
let mut feed = Feed::new(&fixture_markets());
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &frames);
let (token, symbol) = feed
.symbols
.iter_mut()
.find(|(_, symbol)| symbol.state == State::Live)
.find(|(_, symbol)| symbol.state == State::Verified)
.unwrap();
let token = token.clone();
symbol.resync(&token, &mut Vec::new());
@@ -870,9 +1057,9 @@ mod tests {
let effects = feed.on_frame(now, &book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!((feed.resync_gaps, feed.losses), (1, 0));
assert_eq!((feed.subscription_gaps, feed.losses), (1, 0));
assert_eq!(feed.symbols[&token].book, ahead);
assert_eq!(feed.symbols[&token].state, State::Live);
assert_eq!(feed.symbols[&token].state, State::Verified);
}
/// The level test must not mistake our wrong book for a newer one: we
@@ -941,7 +1128,7 @@ mod tests {
for market in &mut markets {
market.min_order_size = "6".to_string();
}
let mut feed = Feed::new(&markets);
let mut feed = feed_of(&markets);
let frames = fixture_frames();
let opening = feed.on_frame(frames[0].0, &frames[0].1);
assert_eq!(snapshot_symbols(&opening).len(), 40);
@@ -963,7 +1150,7 @@ mod tests {
#[test]
fn frames_that_are_not_events_are_skipped() {
let mut feed = Feed::new(&fixture_markets());
let mut feed = feed_of(&fixture_markets());
let at = fixture_frames()[0].0;
for text in [
"PONG",
@@ -976,18 +1163,268 @@ mod tests {
}
}
/// A market's opening books, as a subscribe brings them.
fn opening_books(frames: &[(DateTime<Utc>, String)], market: &Market) -> String {
let opening: Vec<serde_json::Value> = serde_json::from_str(&frames[0].1).unwrap();
let books: Vec<_> = opening
.iter()
.filter(|book| {
market
.tokens
.iter()
.any(|token| book["asset_id"] == **token)
})
.collect();
serde_json::to_string(&books).unwrap()
}
#[test]
fn a_moved_market_logs_its_first_books_once() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut feed = feed_of(&markets[1..]);
replay(&mut feed, &frames[1..100]);
let effects = feed.move_in(&markets[0], frames[100].0);
assert!(matches!(
effects.as_slice(),
[Effect::Send(Command::Subscribe(tokens))] if *tokens == markets[0].tokens
));
let books = opening_books(&frames, &markets[0]);
let effects = feed.on_frame(frames[100].0, &books);
assert_eq!(snapshot_symbols(&effects).len(), 2);
assert_eq!(feed.books_after_moves, 2);
// Later books are the lane's own business.
feed.on_frame(frames[101].0, &books);
assert_eq!(feed.books_after_moves, 2);
assert_eq!((feed.subscription_gaps, feed.losses), (0, 0));
}
#[test]
fn a_market_added_mid_run_is_subscribed_and_its_first_book_written() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut feed = feed_of(&markets[1..]);
replay(&mut feed, &frames[1..100]);
let effects = feed.add_market(&markets[0], frames[100].0);
assert!(matches!(
effects.as_slice(),
[Effect::Send(Command::Subscribe(tokens))] if *tokens == markets[0].tokens
));
let effects = feed.on_frame(frames[100].0, &opening_books(&frames, &markets[0]));
let mut written = snapshot_symbols(&effects);
written.sort();
let mut tokens: Vec<&str> = markets[0].tokens.iter().map(String::as_str).collect();
tokens.sort();
assert_eq!(written, tokens);
assert!(
markets[0]
.tokens
.iter()
.all(|token| feed.symbols[token].state == State::Verified)
);
}
#[test]
fn a_removed_market_is_unsubscribed_and_written_no_more() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut feed = feed_of(&markets);
replay(&mut feed, &frames[..1]);
let removed = &markets[0].tokens;
let effects = feed.remove_market(&markets[0]);
assert!(matches!(
effects.as_slice(),
[Effect::Send(Command::Unsubscribe(tokens))] if tokens == removed
));
// The rest of the recording crosses two minutes: no Delta, Snapshot or
// resync names a removed token.
let effects = replay(&mut feed, &frames[1..]);
let mentions = |effect: &Effect| match effect {
Effect::Write(Output::Snapshot(snapshot)) => removed.contains(&snapshot.symbol),
Effect::Write(Output::Delta(delta)) => removed.contains(&delta.symbol),
Effect::Send(Command::Resync(token)) => removed.contains(token),
Effect::Send(_) => false,
Effect::Report(loads) => loads.contains_key(&markets[0].condition_id),
};
assert!(!effects.iter().any(mentions));
// The minute Snapshots of the 38 others, twice.
assert_eq!(snapshot_symbols(&effects).len(), 38 * 2);
}
/// Seen live: our subscribe hides changes made just before the opening
/// book reaches us, so the first fresh book differs at a level none of our
/// changes touched, before any change was verified.
#[test]
fn a_differing_book_before_any_verified_change_is_a_subscription_gap() {
let frames = fixture_frames();
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &frames[..1]);
let (token, symbol) = feed
.symbols
.iter()
.find(|(_, symbol)| symbol.state == State::Verified)
.unwrap();
let token = token.clone();
assert!(!symbol.verified_since_book);
let mut exchange = symbol.book.clone();
exchange.set(Side::Buy, 1_000, 123_000_000);
let book = fresh_book(&token, symbol, &exchange);
let effects = feed.on_frame(frames[1].0, &book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!((feed.subscription_gaps, feed.losses), (1, 0));
assert_eq!(feed.symbols[&token].book, exchange);
}
/// Once a change has been verified since the opening book, our book was
/// right then: a fresh book that differs is a loss.
#[test]
fn a_differing_book_after_a_verified_change_is_a_loss() {
let frames = fixture_frames();
let mut feed = feed_of(&fixture_markets());
replay(&mut feed, &frames[..frames.len() / 2]);
let (token, symbol) = feed
.symbols
.iter()
.find(|(_, symbol)| symbol.state == State::Verified && symbol.verified_since_book)
.unwrap();
let token = token.clone();
let mut exchange = symbol.book.clone();
exchange.set(Side::Buy, 1_000, 123_000_000);
let book = fresh_book(&token, symbol, &exchange);
let (gaps, losses) = (feed.subscription_gaps, feed.losses);
let effects = feed.on_frame(frames[frames.len() / 2].0, &book);
assert_eq!(snapshot_symbols(&effects), [token.as_str()]);
assert_eq!((feed.subscription_gaps, feed.losses), (gaps, losses + 1));
assert_eq!(feed.symbols[&token].book, exchange);
}
#[test]
fn each_minute_reports_the_bytes_per_second_of_every_market() {
let minute = "2026-11-10T16:00:00Z".parse::<DateTime<Utc>>().unwrap();
let at = |seconds| minute + TimeDelta::seconds(seconds);
let market = |id: &str| Market {
condition_id: id.to_string(),
tokens: vec![format!("{id}a"), format!("{id}b")],
min_order_size: "5".to_string(),
neg_risk: false,
};
let (busy, quiet, late) = (market("0xbusy"), market("0xquiet"), market("0xlate"));
let mut feed = Feed::new(0);
feed.add_market(&busy, at(0));
// Added halfway through the minute: measured over its 30 s.
feed.add_market(&quiet, at(30));
// Added 15 s before it: kept for the next report.
feed.add_market(&late, at(45));
let change = |id: &str| {
serde_json::json!({"event_type": "price_change", "market": id, "timestamp": "1",
"price_changes": []})
.to_string()
};
let trade = serde_json::json!({"event_type": "last_trade_price", "market": "0xbusy",
"asset_id": "0xbusya", "price": "0.5", "size": "1", "side": "BUY",
"timestamp": "1"})
.to_string();
let tick_size = serde_json::json!({"event_type": "tick_size_change", "market": "0xquiet",
"asset_id": "0xquieta", "old_tick_size": "0.01", "new_tick_size": "0.001",
"timestamp": "1"})
.to_string();
// One frame, two markets: each takes half.
let shared = format!("[{},{}]", change("0xbusy"), change("0xquiet"));
let mut reports = Vec::new();
let mut on_frame = |feed: &mut Feed, seconds, text: &str| {
for effect in feed.on_frame(at(seconds), text) {
if let Effect::Report(loads) = effect {
reports.push(loads);
}
}
};
on_frame(&mut feed, 10, &change("0xbusy"));
on_frame(&mut feed, 20, &trade);
on_frame(&mut feed, 40, &tick_size);
on_frame(&mut feed, 50, &change("0xlate"));
on_frame(&mut feed, 55, &change("0xunknown"));
// The next minute: the first report, then nothing more until the one
// after, which starts again from zero.
on_frame(&mut feed, 60, &shared);
on_frame(&mut feed, 120, &change("0xunknown"));
let bytes = |text: &String| text.len() as u64;
let half = bytes(&shared) / 2;
assert_eq!(
reports,
[
Loads::from([
(
"0xbusy".to_string(),
(bytes(&change("0xbusy")) + bytes(&trade)) / 60
),
("0xquiet".to_string(), bytes(&tick_size) / 30),
]),
Loads::from([
("0xbusy".to_string(), half / 60),
("0xquiet".to_string(), half / 60),
("0xlate".to_string(), bytes(&change("0xlate")) / 75),
]),
]
);
}
#[test]
fn the_recording_reports_every_market_once_measured_long_enough() {
let frames = fixture_frames();
let markets = fixture_markets();
let mut feed = feed_of(&markets);
let reports: Vec<Loads> = replay(&mut feed, &frames)
.into_iter()
.filter_map(|effect| match effect {
Effect::Report(loads) => Some(loads),
_ => None,
})
.collect();
// The recording crosses two minutes, the first too soon after its
// start for any market to be measured.
let start = frames[0].0;
assert!(next_minute(start) - start < MIN_MEASURED);
assert_eq!(reports.len(), 2);
assert!(reports[0].is_empty());
let loads = &reports[1];
assert_eq!(loads.len(), markets.len());
assert!(
markets
.iter()
.all(|market| loads.contains_key(&market.condition_id))
);
assert!(loads.values().sum::<u64>() > 0);
}
#[test]
fn the_output_reconciles() {
// The book and writer threads as main runs them, fed the recording.
let (frames_tx, frames_rx) = crossbeam::channel::unbounded();
let (outputs_tx, outputs_rx) = crossbeam::channel::unbounded();
let (resyncs_tx, _resyncs_rx) = crossbeam::channel::unbounded();
let (connection_tx, _connection_rx) = crossbeam::channel::unbounded();
let (_commands_tx, commands_rx) = crossbeam::channel::unbounded();
let (reports_tx, _reports_rx) = crossbeam::channel::unbounded();
for (received_at, text) in fixture_frames() {
frames_tx.send((received_at, text.into())).unwrap();
}
drop(frames_tx);
let feed = Feed::new(&fixture_markets());
let book = std::thread::spawn(move || run(feed, frames_rx, outputs_tx, resyncs_tx));
let feed = feed_of(&fixture_markets());
let book = std::thread::spawn(move || {
run(
feed,
frames_rx,
commands_rx,
outputs_tx,
connection_tx,
reports_tx,
)
});
let path = std::env::temp_dir().join(format!("reconcile-{}.sbe", std::process::id()));
let written = crate::write(path.to_str().unwrap(), outputs_rx);
book.join().unwrap();
+341
View File
@@ -0,0 +1,341 @@
//! Gamma, Polymarket's market listing: which markets are live. Two fetcher
//! threads page it on the orchestrator's request and stream what they read;
//! they hold no state and no timer (roadmap/03).
use std::thread;
use std::time::Duration;
use chrono::{DateTime, Utc};
use crossbeam::channel::{Receiver, Sender};
use serde::Deserialize;
use super::Market;
const KEYSET_URL: &str = "https://gamma-api.polymarket.com/markets/keyset";
/// Gamma's page size, whatever a larger `limit` asks for.
const PAGE_SIZE: &str = "100";
/// For every request, so SIGINT never waits on Gamma longer than this.
const TIMEOUT: Duration = Duration::from_secs(10);
/// A scan's tries at one page, and the wait between them.
const SCAN_TRIES: u32 = 5;
const SCAN_RETRY_AFTER: Duration = Duration::from_secs(5);
/// What REST's `/book` (and so the hash) says when Gamma's `orderMinSize` is
/// null (checked on 4 such tokens, 2026-11-04).
const MISSING_MIN_ORDER_SIZE: &str = "0";
pub enum Request {
/// Every live market.
Rescan,
/// The live markets created and the markets closed since then.
Recent { since: DateTime<Utc> },
}
#[derive(Debug)]
pub enum Finding {
/// A rescan began: once it is done, a market known now and not seen in it
/// is gone.
ScanStarted,
ScanPage(Vec<Market>),
ScanDone,
ScanFailed,
/// Live markets created since the request's time.
Created(Vec<Market>),
/// The `conditionId`s of markets closed since the request's time.
Closed(Vec<String>),
RecentDone,
RecentFailed,
}
/// A fetcher thread: answers each request until the orchestrator is gone.
pub fn run(requests: Receiver<Request>, findings: Sender<Finding>) {
let agent = ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.timeout_global(Some(TIMEOUT))
.build(),
);
for request in requests {
let sent = match request {
Request::Rescan => rescan(&agent, &findings),
Request::Recent { since } => recent(&agent, since, &findings),
};
if sent.is_err() {
return;
}
}
}
/// `Err` once the orchestrator is gone.
type Sent = Result<(), crossbeam::channel::SendError<Finding>>;
fn rescan(agent: &ureq::Agent, findings: &Sender<Finding>) -> Sent {
findings.send(Finding::ScanStarted)?;
let query = [("active", "true"), ("closed", "false")];
let mut cursor = None;
loop {
let page = match with_retries(|| page(agent, &query, cursor.as_deref())) {
Ok(page) => page,
Err(error) => {
log!("polymarket: rescan failed: {error}");
return findings.send(Finding::ScanFailed);
}
};
let last = page.markets.is_empty();
findings.send(Finding::ScanPage(live(page.markets)))?;
cursor = page.next_cursor;
if last || cursor.is_none() {
return findings.send(Finding::ScanDone);
}
}
}
fn with_retries(mut fetch: impl FnMut() -> anyhow::Result<Page>) -> anyhow::Result<Page> {
let mut tries = 1;
loop {
match fetch() {
Err(_) if tries < SCAN_TRIES => {
tries += 1;
thread::sleep(SCAN_RETRY_AFTER);
}
result => return result,
}
}
}
fn recent(agent: &ureq::Agent, since: DateTime<Utc>, findings: &Sender<Finding>) -> Sent {
let created = [
("active", "true"),
("closed", "false"),
("order", "createdAt"),
("ascending", "false"),
];
let closed = [
("closed", "true"),
("order", "closedTime"),
("ascending", "false"),
];
let result = since_then(agent, &created, since, created_at).and_then(|markets| {
findings.send(Finding::Created(live(markets)))?;
since_then(agent, &closed, since, closed_time)
});
match result {
Ok(markets) => {
let ids = markets.into_iter().map(|market| market.condition_id);
findings.send(Finding::Closed(ids.collect()))?;
findings.send(Finding::RecentDone)
}
Err(error) => {
log!("polymarket: recent check failed: {error}");
findings.send(Finding::RecentFailed)
}
}
}
fn created_at(market: &GammaMarket) -> &Option<String> {
&market.created_at
}
fn closed_time(market: &GammaMarket) -> &Option<String> {
&market.closed_time
}
/// The markets of `query`'s pages, sorted newest first by `time`, down to
/// `since`.
fn since_then(
agent: &ureq::Agent,
query: &[(&str, &str)],
since: DateTime<Utc>,
time: impl Fn(&GammaMarket) -> &Option<String>,
) -> anyhow::Result<Vec<GammaMarket>> {
let mut markets = Vec::new();
let mut cursor = None;
loop {
let page = page(agent, query, cursor.as_deref())?;
let (newer, reached) = newer_than(page.markets, since, &time);
markets.extend(newer);
cursor = page.next_cursor;
if reached || cursor.is_none() {
return Ok(markets);
}
}
}
/// The markets of a newest-first page at or after `since`, and whether the
/// page reached past it. A market with no readable time is kept.
fn newer_than(
markets: Vec<GammaMarket>,
since: DateTime<Utc>,
time: impl Fn(&GammaMarket) -> &Option<String>,
) -> (Vec<GammaMarket>, bool) {
let reached = markets.is_empty()
|| markets
.iter()
.any(|market| parse_time(time(market)).is_some_and(|time| time < since));
let newer = markets
.into_iter()
.filter(|market| parse_time(time(market)).is_none_or(|time| time >= since))
.collect();
(newer, reached)
}
/// `createdAt` is RFC 3339 (`2026-11-04T16:57:29.989119Z`), `closedTime` not
/// (`2026-11-04 16:57:50.384485+00`).
fn parse_time(text: &Option<String>) -> Option<DateTime<Utc>> {
let text = text.as_deref()?;
DateTime::parse_from_rfc3339(text)
.or_else(|_| DateTime::parse_from_str(text, "%Y-%m-%d %H:%M:%S%.f%#z"))
.ok()
.map(|time| time.to_utc())
}
#[derive(Deserialize)]
struct Page {
markets: Vec<GammaMarket>,
next_cursor: Option<String>,
}
/// Every field can be missing or null on some market (2026-11-04: 230 live
/// markets had no `negRisk`, 31 no `orderMinSize`).
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct GammaMarket {
#[serde(default)]
condition_id: String,
/// A JSON array inside a string.
clob_token_ids: Option<String>,
order_min_size: Option<serde_json::Number>,
neg_risk: Option<bool>,
active: Option<bool>,
closed: Option<bool>,
enable_order_book: Option<bool>,
accepting_orders: Option<bool>,
created_at: Option<String>,
closed_time: Option<String>,
}
fn page(agent: &ureq::Agent, query: &[(&str, &str)], cursor: Option<&str>) -> anyhow::Result<Page> {
let mut request = agent
.get(KEYSET_URL)
.query("limit", PAGE_SIZE)
.query_pairs(query.iter().copied());
if let Some(cursor) = cursor {
request = request.query("after_cursor", cursor);
}
let body = request.call()?.body_mut().read_to_string()?;
Ok(serde_json::from_str(&body)?)
}
/// The live markets among `markets`: active, not closed, with an order book
/// that accepts orders. The end date doesn't count (docs/notes.md).
fn live(markets: Vec<GammaMarket>) -> Vec<Market> {
markets
.into_iter()
.filter(|market| {
market.active == Some(true)
&& market.closed != Some(true)
&& market.enable_order_book == Some(true)
&& market.accepting_orders == Some(true)
&& !market.condition_id.is_empty()
})
.filter_map(|market| {
let tokens: Vec<String> =
serde_json::from_str(market.clob_token_ids.as_deref()?).ok()?;
if tokens.is_empty() {
return None;
}
Some(Market {
condition_id: market.condition_id,
tokens,
min_order_size: market
.order_min_size
.map_or(MISSING_MIN_ORDER_SIZE.to_string(), |size| size.to_string()),
neg_risk: market.neg_risk.unwrap_or(false),
})
})
.collect()
}
/// The live markets of a saved `/markets` response (a plain array).
#[cfg(test)]
pub(crate) fn parse_markets(body: &str) -> Vec<Market> {
live(serde_json::from_str(body).unwrap())
}
#[cfg(test)]
mod tests {
use super::*;
fn fixture(name: &str) -> Page {
let path = format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"));
serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()
}
fn time(text: &str) -> DateTime<Utc> {
parse_time(&Some(text.to_string())).unwrap()
}
#[test]
fn parses_gamma_markets() {
let markets = parse_markets(include_str!("../../tests/fixtures/gamma-markets.json"));
assert_eq!(markets.len(), 20);
assert!(markets.iter().all(|market| market.tokens.len() == 2));
assert!(markets[0].condition_id.starts_with("0x"));
assert_eq!(markets[0].min_order_size, "5");
}
#[test]
fn keyset_pages_parse_and_closed_markets_are_not_live() {
let created = fixture("gamma-created.json");
assert_eq!(created.markets.len(), 10);
assert!(created.next_cursor.is_some());
assert_eq!(live(created.markets).len(), 10);
let closed = fixture("gamma-closed.json");
assert_eq!(closed.markets.len(), 10);
assert!(live(closed.markets).is_empty());
}
#[test]
fn both_time_formats_parse() {
assert_eq!(
time("2026-11-04T16:57:29.989119Z"),
time("2026-11-04 16:57:29.989119+00")
);
}
#[test]
fn a_page_is_kept_down_to_since_and_reports_reaching_it() {
// The fixture's createdAt: 16:57:29.989, 16:54:27.166, .084, .011, then
// six from 16:54:26.943 down to 16:54:26.595.
let page = fixture("gamma-created.json");
let (newer, reached) = newer_than(page.markets, time("2026-11-04T16:54:27Z"), created_at);
assert_eq!(newer.len(), 4);
assert!(reached);
let page = fixture("gamma-created.json");
let (newer, reached) = newer_than(page.markets, time("2026-11-04T16:00:00Z"), created_at);
assert_eq!(newer.len(), 10);
assert!(!reached);
let (newer, reached) = newer_than(Vec::new(), time("2026-11-04T16:00:00Z"), created_at);
assert!(newer.is_empty() && reached);
}
#[test]
fn a_live_market_takes_what_the_hash_uses_and_needs_token_ids() {
let markets = live(
serde_json::from_str(
r#"[{"conditionId":"0xa","clobTokenIds":"[\"1\",\"2\"]","negRisk":null,
"active":true,"closed":false,"enableOrderBook":true,"acceptingOrders":true},
{"conditionId":"0xb","clobTokenIds":null,
"active":true,"closed":false,"enableOrderBook":true,"acceptingOrders":true},
{"conditionId":"0xd","clobTokenIds":"[]",
"active":true,"closed":false,"enableOrderBook":true,"acceptingOrders":true},
{"conditionId":"0xc","clobTokenIds":"[\"3\",\"4\"]",
"active":true,"closed":false,"enableOrderBook":true,"acceptingOrders":null}]"#,
)
.unwrap(),
);
assert_eq!(markets.len(), 1);
assert_eq!(markets[0].min_order_size, "0");
assert!(!markets[0].neg_risk);
}
}
+8 -59
View File
@@ -1,17 +1,17 @@
//! Polymarket: which markets to track (Gamma), the market-channel socket, and
//! the feed that turns its frames into books and output.
//! Polymarket: which markets are live (Gamma), the orchestrator that spreads
//! them over lanes, and each lane's connection and feed (roadmap/03).
pub mod feed;
pub mod gamma;
pub mod orchestrator;
pub mod socket;
use serde::Deserialize;
const GAMMA_MARKETS_URL: &str = "https://gamma-api.polymarket.com/markets";
/// A market's symbols and the two inputs of their book hash that the
/// A live market: its symbols and the two inputs of their book hash that the
/// websocket doesn't carry (docs/notes.md).
#[derive(Debug, Clone)]
pub struct Market {
/// Gamma's `conditionId`, the websocket's `market`.
pub condition_id: String,
/// Its token ids, one per outcome: each is a symbol.
pub tokens: Vec<String>,
/// Exactly as the hash expects it, e.g. `"5"`.
@@ -19,58 +19,7 @@ pub struct Market {
pub neg_risk: bool,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct GammaMarket {
/// A JSON array inside a string.
clob_token_ids: String,
order_min_size: serde_json::Number,
neg_risk: bool,
enable_order_book: bool,
}
/// The `limit` live markets with the most 24 h volume, keeping those with an
/// order book. Fixed for the run: discovery is M3.
pub fn top_markets(limit: u32) -> anyhow::Result<Vec<Market>> {
let body = ureq::get(GAMMA_MARKETS_URL)
.query("limit", limit.to_string())
.query("active", "true")
.query("closed", "false")
.query("order", "volume24hr")
.query("ascending", "false")
.call()?
.body_mut()
.read_to_string()?;
parse_markets(&body)
}
fn parse_markets(body: &str) -> anyhow::Result<Vec<Market>> {
let markets: Vec<GammaMarket> = serde_json::from_str(body)?;
markets
.into_iter()
.filter(|market| market.enable_order_book)
.map(|market| {
Ok(Market {
tokens: serde_json::from_str(&market.clob_token_ids)?,
min_order_size: market.order_min_size.to_string(),
neg_risk: market.neg_risk,
})
})
.collect()
}
#[cfg(test)]
pub(crate) fn fixture_markets() -> Vec<Market> {
parse_markets(include_str!("../../tests/fixtures/gamma-markets.json")).unwrap()
}
#[cfg(test)]
mod tests {
#[test]
fn parses_gamma_markets() {
let markets = super::fixture_markets();
assert_eq!(markets.len(), 20);
assert!(markets.iter().all(|market| market.tokens.len() == 2));
assert_eq!(markets[0].min_order_size, "5");
}
gamma::parse_markets(include_str!("../../tests/fixtures/gamma-markets.json"))
}
+678
View File
@@ -0,0 +1,678 @@
//! The orchestrator: the one owner of what is tracked. It keeps the tracked
//! markets equal to Gamma's live ones, spreads them over lanes and owns every
//! timer; the two fetchers only fetch (roadmap/03, ADR 0002).
use std::cmp::Reverse;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant};
use chrono::{DateTime, TimeDelta, Utc};
use crossbeam::channel::{Sender, select, unbounded};
use engineering_challenge::book::Output;
use super::feed::{self, Feed, LaneCommand, LoadReport, Loads};
use super::gamma::{self, Finding, Request};
use super::{Market, socket};
/// The most tokens a lane holds: a reconnect replays every one's book at once.
const LANE_TOKENS: usize = 5_000;
/// A lane's byte budget, under the lowest rate at which a connection was seen
/// to get a 1013 close (0.46 MB/s; docs/notes.md). Closes came at up to
/// 5.1 MB/s too: no rate is safe, only less likely to be closed.
const LANE_BYTES_PER_SECOND: u64 = 300_000;
/// Moves are limited so balancing doesn't churn. Rebalancing runs with each
/// minute check, every [`RECENT_EVERY`].
const MOVES_PER_MINUTE: usize = 50;
const MOVED_STAYS: Duration = Duration::from_secs(10 * 60);
const RECENT_EVERY: Duration = Duration::from_secs(60);
const RESCAN_EVERY: Duration = Duration::from_secs(60 * 60);
/// Gamma's indexing delay: each recent check reaches this far before the
/// last successful one. New markets were listed up to 5 minutes after their
/// `createdAt` (docs/notes.md); known markets are skipped, so the margin costs
/// only the pages it reads again.
const RECENT_MARGIN: TimeDelta = TimeDelta::minutes(15);
/// So SIGINT is seen while no timer is due.
const WAKE_EVERY: Duration = Duration::from_secs(1);
/// Runs until `stop` is set, then waits for every lane. Returns how many
/// threads failed: the lane threads that panicked, and one if a fetcher died.
pub fn run(outputs: Sender<Output>, stop: &Arc<AtomicBool>) -> usize {
let (findings_tx, findings) = unbounded();
let (rescan, rescan_requests) = unbounded();
let (recent, recent_requests) = unbounded();
// Two, so a full scan never holds up the minute check. Not joined: they
// hold no output, and each ends at its next send.
let fetchers: Vec<JoinHandle<()>> = [rescan_requests, recent_requests]
.into_iter()
.map(|requests| {
let findings = findings_tx.clone();
thread::spawn(move || gamma::run(requests, findings))
})
.collect();
drop(findings_tx);
let (reports_tx, reports) = unbounded::<LoadReport>();
let mut tracker = Tracker::default();
let mut lanes: Vec<Lane> = Vec::new();
let (mut added, mut removed, mut moved) = (0, 0, 0);
// Startup is the first rescan; the minute checks cover it from its start.
let mut recent_since = Utc::now() - RECENT_MARGIN;
let mut recent_asked_at: Option<DateTime<Utc>> = None;
let mut scanning = false;
let mut scanned = false;
let mut fetcher_died = false;
let mut next_rescan = Instant::now();
let mut next_recent = Instant::now() + RECENT_EVERY;
// Opens the lane on its first command.
let send = |lanes: &mut Vec<Lane>, lane: usize, command: LaneCommand| {
if lane == lanes.len() {
lanes.push(open_lane(lane, &outputs, &reports_tx, stop));
}
// A lane whose book thread died keeps its markets unserved: lane
// health is M5's.
_ = lanes[lane].commands.send(command);
};
while !stop.load(Ordering::Relaxed) {
// Only a panic ends a fetcher while this runs, even one in the middle
// of a request, whose answer would never come: nothing would keep the
// set current.
if fetchers.iter().any(JoinHandle::is_finished) {
log!("polymarket: orchestrator: a fetcher died, stopping");
stop.store(true, Ordering::Relaxed);
fetcher_died = true;
break;
}
let now = Instant::now();
if now >= next_rescan {
if !scanning {
_ = rescan.send(Request::Rescan);
scanning = true;
}
next_rescan = now + RESCAN_EVERY;
}
if now >= next_recent {
if recent_asked_at.is_none() {
_ = recent.send(Request::Recent {
since: recent_since,
});
recent_asked_at = Some(Utc::now());
}
next_recent = now + RECENT_EVERY;
// On the loads the lanes reported at the wall-clock minute.
let moves = tracker.rebalance(now);
moved += moves.len();
for (lane, command) in moves.into_iter().flat_map(Move::commands) {
send(&mut lanes, lane, command);
}
let busiest = tracker.lanes.iter().map(|lane| lane.load).max();
log!(
"polymarket: orchestrator: {} markets in {} lanes; {added} added, {removed} \
removed, {moved} moved this minute; busiest lane at {} bytes/s",
tracker.markets.len(),
lanes.len(),
busiest.unwrap_or(0)
);
(added, removed, moved) = (0, 0, 0);
}
let timeout = next_rescan
.min(next_recent)
.saturating_duration_since(Instant::now())
.min(WAKE_EVERY);
let finding = select! {
recv(findings) -> finding => match finding {
Ok(finding) => finding,
// Both fetchers are gone: the check above stops the run.
Err(_) => continue,
},
recv(reports) -> report => {
if let Ok(report) = report {
tracker.report(report.lane, &report.loads);
}
continue;
},
default(timeout) => continue,
};
let commands = match finding {
Finding::ScanStarted => {
tracker.scan_started();
Vec::new()
}
Finding::ScanPage(markets) | Finding::Created(markets) => markets
.into_iter()
.filter_map(|market| tracker.add(market))
.collect(),
Finding::ScanDone => {
scanning = false;
scanned = true;
tracker.scan_done()
}
Finding::ScanFailed => {
scanning = false;
tracker.scan_failed();
// Nothing can be tracked before a first scan: retry at once.
if !scanned {
next_rescan = Instant::now();
}
Vec::new()
}
Finding::Closed(ids) => ids.iter().filter_map(|id| tracker.remove(id)).collect(),
Finding::RecentDone => {
if let Some(asked_at) = recent_asked_at.take() {
recent_since = asked_at - RECENT_MARGIN;
}
Vec::new()
}
Finding::RecentFailed => {
recent_asked_at = None;
Vec::new()
}
};
for (lane, command) in commands {
match command {
LaneCommand::Add(_) => added += 1,
LaneCommand::Remove(_) => removed += 1,
// Only rebalancing moves, counted there.
LaneCommand::MoveIn(_) => {}
}
send(&mut lanes, lane, command);
}
}
drop(outputs);
let panicked = lanes
.into_iter()
.flat_map(|lane| lane.threads)
.map(JoinHandle::join)
.filter(Result::is_err)
.count();
panicked + usize::from(fetcher_died)
}
struct Lane {
commands: Sender<LaneCommand>,
threads: [JoinHandle<()>; 2],
}
/// Lane `number`'s connection and book thread, joined by [`run`].
fn open_lane(
number: usize,
outputs: &Sender<Output>,
reports: &Sender<LoadReport>,
stop: &Arc<AtomicBool>,
) -> Lane {
let (commands, commands_rx) = unbounded();
let (frames, frames_rx) = unbounded();
let (connection, connection_rx) = unbounded();
let stop = Arc::clone(stop);
let outputs = outputs.clone();
let reports = reports.clone();
let connection_thread = thread::spawn(move || {
socket::run(
socket::MARKET_CHANNEL_URL,
number,
&frames,
&connection_rx,
&stop,
)
});
let book_thread = thread::spawn(move || {
feed::run(
Feed::new(number),
frames_rx,
commands_rx,
outputs,
connection,
reports,
)
});
Lane {
commands,
threads: [connection_thread, book_thread],
}
}
/// A market leaving lane `from` for lane `to`.
#[derive(Debug)]
struct Move {
market: Market,
from: usize,
to: usize,
}
impl Move {
/// A Remove for the old lane, then a MoveIn for the new one, in the order
/// they are sent.
fn commands(self) -> [(usize, LaneCommand); 2] {
[
(self.from, LaneCommand::Remove(self.market.clone())),
(self.to, LaneCommand::MoveIn(self.market)),
]
}
}
/// A tracked market, where it is and what it weighs.
struct Tracked {
market: Market,
lane: usize,
/// Bytes per second over its lane's last report; 0 until one comes.
load: u64,
moved_at: Option<Instant>,
}
/// A lane's tokens and load, against its two limits.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
struct LaneLoad {
tokens: usize,
/// The sum of its markets' loads, bytes per second.
load: u64,
}
/// Which lane each market is in and what it weighs, and which markets a
/// rescan in progress hasn't seen yet.
#[derive(Default)]
struct Tracker {
/// By `conditionId`.
markets: HashMap<String, Tracked>,
/// By lane number.
lanes: Vec<LaneLoad>,
/// While a rescan runs: the markets known at its start, not seen since.
unseen: Option<HashSet<String>>,
}
impl Tracker {
/// A market it doesn't know goes to the least-loaded lane with token room,
/// a new lane if none has: the lane and its command.
fn add(&mut self, market: Market) -> Option<(usize, LaneCommand)> {
if let Some(unseen) = &mut self.unseen {
unseen.remove(&market.condition_id);
}
if self.markets.contains_key(&market.condition_id) {
return None;
}
let lane = self.lane_for(market.tokens.len(), None, None);
self.lanes[lane].tokens += market.tokens.len();
let tracked = Tracked {
market: market.clone(),
lane,
load: 0,
moved_at: None,
};
self.markets.insert(market.condition_id.clone(), tracked);
Some((lane, LaneCommand::Add(market)))
}
/// The least-loaded lane (then the one with fewest tokens), other than
/// `except`, with room for `tokens` more and, if given, `load` more
/// bytes/s; a new lane if none has.
fn lane_for(&mut self, tokens: usize, load: Option<u64>, except: Option<usize>) -> usize {
self.lanes
.iter()
.enumerate()
.filter(|&(lane, lane_load)| {
Some(lane) != except
&& lane_load.tokens + tokens <= LANE_TOKENS
&& load.is_none_or(|load| lane_load.load + load <= LANE_BYTES_PER_SECOND)
})
.min_by_key(|&(_, lane_load)| (lane_load.load, lane_load.tokens))
.map(|(lane, _)| lane)
.unwrap_or_else(|| {
self.lanes.push(LaneLoad::default());
self.lanes.len() - 1
})
}
fn remove(&mut self, condition_id: &str) -> Option<(usize, LaneCommand)> {
let tracked = self.markets.remove(condition_id)?;
let lane = &mut self.lanes[tracked.lane];
lane.tokens -= tracked.market.tokens.len();
lane.load -= tracked.load;
Some((tracked.lane, LaneCommand::Remove(tracked.market)))
}
/// A lane's loads per market. A market reported by a lane it has since
/// moved from keeps the load it has.
fn report(&mut self, lane: usize, loads: &Loads) {
for (condition_id, &load) in loads {
let Some(tracked) = self.markets.get_mut(condition_id) else {
continue;
};
if tracked.lane != lane {
continue;
}
let lane_load = &mut self.lanes[lane];
lane_load.load = lane_load.load - tracked.load + load;
tracked.load = load;
}
}
/// Moves the busiest markets off every lane over its byte budget, busiest
/// lane first, to the least-loaded lanes with room for them, until it is
/// under: at most [`MOVES_PER_MINUTE`], and none moved in the last
/// [`MOVED_STAYS`].
fn rebalance(&mut self, now: Instant) -> Vec<Move> {
let mut moves = Vec::new();
let mut over: Vec<usize> = (0..self.lanes.len())
.filter(|&lane| self.lanes[lane].load > LANE_BYTES_PER_SECOND)
.collect();
over.sort_by_key(|&lane| Reverse(self.lanes[lane].load));
for from in over {
// ponytail: a scan of every market per lane over budget, fine at a
// few hundred thousand once a minute; per-lane market sets if not.
let mut busiest: Vec<(String, u64)> = self
.markets
.iter()
.filter(|(_, tracked)| {
tracked.lane == from
&& tracked
.moved_at
.is_none_or(|at| now.duration_since(at) >= MOVED_STAYS)
})
.map(|(id, tracked)| (id.clone(), tracked.load))
.collect();
busiest.sort_by_key(|&(_, load)| Reverse(load));
for (id, load) in busiest {
let size = self.markets[&id].market.tokens.len();
// Alone on its lane, a market would only take its load along;
// one with no load would lighten nothing.
if self.lanes[from].load <= LANE_BYTES_PER_SECOND
|| moves.len() == MOVES_PER_MINUTE
|| self.lanes[from].tokens == size
|| load == 0
{
break;
}
let to = self.lane_for(size, Some(load), Some(from));
self.lanes[from].tokens -= size;
self.lanes[from].load -= load;
self.lanes[to].tokens += size;
self.lanes[to].load += load;
let tracked = self.markets.get_mut(&id).expect("listed above");
tracked.lane = to;
tracked.moved_at = Some(now);
moves.push(Move {
market: tracked.market.clone(),
from,
to,
});
}
}
moves
}
fn scan_started(&mut self) {
self.unseen = Some(self.markets.keys().cloned().collect());
}
/// Removes the markets known at the scan's start and not seen in it:
/// those added since were never at risk.
fn scan_done(&mut self) -> Vec<(usize, LaneCommand)> {
let unseen = self.unseen.take().unwrap_or_default();
unseen.iter().filter_map(|id| self.remove(id)).collect()
}
/// Half a listing says nothing of the other half: nothing is removed.
fn scan_failed(&mut self) {
self.unseen = None;
}
}
#[cfg(test)]
mod tests {
use super::*;
fn market(id: usize) -> Market {
Market {
condition_id: format!("0x{id}"),
tokens: vec![format!("{id}a"), format!("{id}b")],
min_order_size: "5".to_string(),
neg_risk: false,
}
}
fn lanes(commands: &[(usize, LaneCommand)]) -> Vec<usize> {
commands.iter().map(|(lane, _)| *lane).collect()
}
fn tokens(tracker: &Tracker) -> Vec<usize> {
tracker.lanes.iter().map(|lane| lane.tokens).collect()
}
/// Reports `bytes_per_second` for each market, as its lane would.
fn report(tracker: &mut Tracker, loads: &[(usize, u64)]) {
for &(id, bytes_per_second) in loads {
let lane = tracker.markets[&format!("0x{id}")].lane;
let loads = Loads::from([(format!("0x{id}"), bytes_per_second)]);
tracker.report(lane, &loads);
}
}
#[test]
fn a_market_joins_the_least_loaded_lane_with_room_and_lanes_open_when_full() {
let mut tracker = Tracker::default();
let per_lane = LANE_TOKENS / 2;
let first: Vec<_> = (0..per_lane + 1)
.filter_map(|id| tracker.add(market(id)))
.collect();
// Nothing measured yet: the first lane fills, then a second opens.
assert!(lanes(&first[..per_lane]).iter().all(|&lane| lane == 0));
assert_eq!(first[per_lane].0, 1);
// A hole in lane 0 draws the next market once lane 1 weighs more.
tracker.remove("0x0").unwrap();
report(&mut tracker, &[(per_lane, 1_000)]);
assert_eq!(tracker.add(market(per_lane + 1)).unwrap().0, 0);
// Both tokens of a market share its lane.
assert_eq!(tracker.markets["0x1"].lane, 0);
assert_eq!(tokens(&tracker), [LANE_TOKENS, 2]);
}
#[test]
fn between_lanes_of_equal_load_a_market_joins_the_one_with_fewest_tokens() {
let mut tracker = Tracker {
lanes: vec![
LaneLoad { tokens: 4, load: 0 },
LaneLoad { tokens: 2, load: 0 },
],
..Tracker::default()
};
assert_eq!(lanes(&[tracker.add(market(0)).unwrap()]), [1]);
assert_eq!(tokens(&tracker), [4, 4]);
}
#[test]
fn a_known_market_is_skipped() {
let mut tracker = Tracker::default();
assert!(tracker.add(market(1)).is_some());
assert!(tracker.add(market(1)).is_none());
assert_eq!(tokens(&tracker), [2]);
}
#[test]
fn a_report_sets_its_lanes_markets_and_no_others() {
let mut tracker = Tracker::default();
tracker.add(market(1));
tracker.add(market(2));
report(&mut tracker, &[(1, 300), (2, 100)]);
assert_eq!(tracker.lanes[0].load, 400);
// A lane the market has left reports it late: ignored.
tracker.lanes.push(LaneLoad::default());
let stale = Loads::from([("0x1".to_string(), 900)]);
tracker.report(1, &stale);
assert_eq!(tracker.markets["0x1"].load, 300);
assert_eq!(tracker.lanes[1].load, 0);
// A market gone takes its load along.
tracker.remove("0x1");
assert_eq!(tracker.lanes[0].load, 100);
}
#[test]
fn a_lane_over_budget_moves_its_busiest_markets_until_under() {
let mut tracker = Tracker::default();
for id in 0..10 {
tracker.add(market(id));
}
tracker.lanes.push(LaneLoad::default());
let budget = LANE_BYTES_PER_SECOND;
// Lane 0 at 1.5 budgets: its busiest market (half a budget) moving
// is enough.
let mut loads = vec![(0, budget / 2)];
loads.extend((1..10).map(|id| (id, budget / 9)));
report(&mut tracker, &loads);
let now = Instant::now();
let moves = tracker.rebalance(now);
assert!(matches!(
moves.as_slice(),
[Move { market, from: 0, to: 1 }] if market.condition_id == "0x0"
));
assert_eq!(tracker.markets["0x0"].lane, 1);
let [
(0, LaneCommand::Remove(removed)),
(1, LaneCommand::MoveIn(added)),
] = moves.into_iter().next().unwrap().commands()
else {
panic!("a move is a Remove for the old lane, then a MoveIn for the new");
};
assert_eq!(
(removed.condition_id, added.condition_id),
("0x0".into(), "0x0".into())
);
assert!(tracker.lanes[0].load <= budget);
assert_eq!(tokens(&tracker), [18, 2]);
assert_eq!(tracker.lanes[1].load, budget / 2);
// Under budget now: nothing more moves.
assert!(tracker.rebalance(now).is_empty());
}
#[test]
fn the_busiest_lane_over_budget_is_rebalanced_first() {
let mut tracker = Tracker::default();
tracker.add(market(0));
tracker.add(market(1));
tracker.lanes.push(LaneLoad::default());
// The empty lane is the least loaded: both join it.
tracker.add(market(2));
tracker.add(market(3));
assert_eq!(tokens(&tracker), [4, 4]);
let budget = LANE_BYTES_PER_SECOND;
report(&mut tracker, &[(0, budget), (1, budget / 10)]);
report(&mut tracker, &[(2, budget / 2), (3, budget)]);
let moves = tracker.rebalance(Instant::now());
assert!(matches!(
moves.first(),
Some(Move { market, from: 1, .. }) if market.condition_id == "0x3"
));
}
/// The markets that may move have no load: moving them would empty the
/// lane and lighten nothing.
#[test]
fn markets_with_no_load_are_not_moved() {
let mut tracker = Tracker::default();
for id in 0..4 {
tracker.add(market(id));
}
report(&mut tracker, &[(0, 2 * LANE_BYTES_PER_SECOND)]);
let now = Instant::now();
tracker.markets.get_mut("0x0").unwrap().moved_at = Some(now);
assert!(tracker.rebalance(now).is_empty());
}
#[test]
fn moves_are_limited_and_a_moved_market_stays() {
let mut tracker = Tracker::default();
let count = 2 * MOVES_PER_MINUTE;
for id in 0..count {
tracker.add(market(id));
}
// Every market alone near a budget: each needs a lane of its own.
let loads: Vec<_> = (0..count)
.map(|id| (id, LANE_BYTES_PER_SECOND * 9 / 10))
.collect();
report(&mut tracker, &loads);
let now = Instant::now();
let first = tracker.rebalance(now);
assert_eq!(first.len(), MOVES_PER_MINUTE);
// The rest move a minute later, but the last, alone and under budget.
let second = tracker.rebalance(now + Duration::from_secs(60));
assert_eq!(second.len(), MOVES_PER_MINUTE - 1);
assert!(second.iter().all(|moved| moved.from == 0));
// Ten minutes on, lane 0 holds one market: it is left alone.
assert!(tracker.rebalance(now + MOVED_STAYS).is_empty());
assert_eq!(tracker.lanes[0].tokens, 2);
}
/// Over budget on its own, a market would only carry its load from lane
/// to lane every ten minutes.
#[test]
fn a_market_alone_on_its_lane_is_not_moved() {
let mut tracker = Tracker::default();
tracker.add(market(0));
report(&mut tracker, &[(0, 2 * LANE_BYTES_PER_SECOND)]);
assert!(tracker.rebalance(Instant::now()).is_empty());
assert_eq!(tracker.lanes.len(), 1);
}
#[test]
fn a_market_moved_less_than_ten_minutes_ago_stays() {
let mut tracker = Tracker::default();
let budget = LANE_BYTES_PER_SECOND;
tracker.add(market(0));
tracker.add(market(1));
report(&mut tracker, &[(0, budget * 7 / 10), (1, budget / 2)]);
tracker.lanes.push(LaneLoad::default());
let now = Instant::now();
tracker.add(market(2));
assert_eq!(tracker.markets["0x2"].lane, 1);
report(&mut tracker, &[(2, budget / 10)]);
// Market 0 moves to lane 1, which then turns busy.
assert!(matches!(
tracker.rebalance(now).as_slice(),
[Move { from: 0, to: 1, .. }]
));
report(&mut tracker, &[(0, budget * 7 / 10), (2, budget * 6 / 10)]);
let moves = tracker.rebalance(now + Duration::from_secs(60));
assert!(matches!(
moves.as_slice(),
[Move { market, from: 1, to: 2 }] if market.condition_id == "0x2"
));
}
#[test]
fn a_completed_rescan_removes_only_the_known_markets_it_did_not_see() {
let mut tracker = Tracker::default();
for id in 0..3 {
tracker.add(market(id));
}
tracker.scan_started();
tracker.add(market(0));
// Added by a minute check during the scan, on a page already passed.
tracker.add(market(9));
// Closed during the scan.
tracker.remove("0x1");
let removed = tracker.scan_done();
assert!(matches!(
removed.as_slice(),
[(0, LaneCommand::Remove(market))] if market.condition_id == "0x2"
));
let mut tracked: Vec<_> = tracker.markets.keys().cloned().collect();
tracked.sort();
assert_eq!(tracked, ["0x0", "0x9"]);
assert_eq!(tokens(&tracker), [4]);
}
#[test]
fn a_failed_rescan_removes_nothing() {
let mut tracker = Tracker::default();
tracker.add(market(0));
tracker.scan_started();
tracker.scan_failed();
assert!(tracker.scan_done().is_empty());
assert_eq!(tracker.markets.len(), 1);
}
}
+204 -36
View File
@@ -1,6 +1,8 @@
//! The socket thread: reads the market channel and stamps each frame's receive
//! time, and nothing else. Parsing is the book thread's.
//! A lane's connection: reads the market channel and stamps each frame's
//! receive time, and sends what its book thread asks for (subscriptions,
//! resyncs). Parsing is the book thread's.
use std::collections::BTreeSet;
use std::io::ErrorKind;
use std::net::{TcpStream, ToSocketAddrs};
use std::sync::atomic::{AtomicBool, Ordering};
@@ -10,42 +12,66 @@ use std::time::{Duration, Instant};
use chrono::{DateTime, Utc};
use crossbeam::channel::{Receiver, Sender};
use serde_json::json;
use tungstenite::http::Uri;
use tungstenite::stream::MaybeTlsStream;
use tungstenite::{Message, Utf8Bytes, WebSocket};
const MARKET_CHANNEL_HOST: &str = "ws-subscriptions-clob.polymarket.com";
const MARKET_CHANNEL_URL: &str = "wss://ws-subscriptions-clob.polymarket.com/ws/market";
pub const MARKET_CHANNEL_URL: &str = "wss://ws-subscriptions-clob.polymarket.com/ws/market";
/// For the TCP connect and the TLS + websocket handshake, and for every send.
const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
/// How often the loop wakes when no frame comes: tungstenite has no `split`,
/// so the reads and the writes below (PING, resyncs, close) share this thread.
/// so the reads and the writes below (PING, commands, close) share this thread.
const READ_TIMEOUT: Duration = Duration::from_millis(200);
const PING_EVERY: Duration = Duration::from_secs(10);
const RECONNECT_AFTER: Duration = Duration::from_secs(1);
const CLOSE_TIMEOUT: Duration = Duration::from_secs(2);
/// Tokens per subscribe or unsubscribe message: a first message of more than
/// 500 gets no opening books (docs/notes.md).
const CHUNK: usize = 500;
/// A frame's text with its receive time.
pub type Frame = (DateTime<Utc>, Utf8Bytes);
/// What the book thread asks its connection for.
#[derive(Debug, PartialEq, Eq)]
pub enum Command {
Subscribe(Vec<String>),
Unsubscribe(Vec<String>),
/// Unsubscribe and subscribe again: the server then sends a fresh book.
Resync(String),
}
type Socket = WebSocket<MaybeTlsStream<TcpStream>>;
/// Runs until `stop` is set or the book thread is gone, reconnecting (and
/// subscribing every token again) whenever the connection drops. `resyncs`
/// carries the tokens to unsubscribe and subscribe again.
/// subscribing its tokens again) whenever the connection drops.
pub fn run(
tokens: &[String],
url: &str,
lane: usize,
frames: &Sender<Frame>,
resyncs: &Receiver<String>,
commands: &Receiver<Command>,
stop: &AtomicBool,
) {
let mut tokens = BTreeSet::new();
while !stop.load(Ordering::Relaxed) {
match connect(tokens) {
Ok(mut socket) => match read(&mut socket, frames, resyncs, stop) {
Ok(Ended::Stopped) => return close(&mut socket, frames),
Ok(Ended::BookThreadGone) => return,
Err(error) => log!("polymarket: connection lost: {error}"),
},
Err(error) => log!("polymarket: connect failed: {error}"),
// Asked for while disconnected: the subscribe on connect covers them,
// and its books are fresh, so resyncs have nothing left to do.
for command in commands.try_iter() {
_ = apply(&mut tokens, &command);
}
match connect(url, &tokens) {
Ok(mut socket) => {
log!(
"polymarket: lane {lane}: connected, {} symbols",
tokens.len()
);
match read(&mut socket, &mut tokens, frames, commands, stop) {
Ok(Ended::Stopped) => return close(&mut socket, lane, frames),
Ok(Ended::BookThreadGone) => return,
Err(error) => log!("polymarket: lane {lane}: connection lost: {error}"),
}
}
Err(error) => log!("polymarket: lane {lane}: connect failed: {error}"),
}
thread::sleep(RECONNECT_AFTER);
}
@@ -56,17 +82,57 @@ enum Ended {
BookThreadGone,
}
fn connect(tokens: &[String]) -> anyhow::Result<Socket> {
/// Updates the token set, and returns the messages that carry `command` out
/// on an open connection: none for a resync of a token not in the set.
#[must_use]
fn apply(tokens: &mut BTreeSet<String>, command: &Command) -> Vec<String> {
match command {
Command::Subscribe(added) => {
tokens.extend(added.iter().cloned());
operations("subscribe", added)
}
Command::Unsubscribe(removed) => {
removed.iter().for_each(|token| _ = tokens.remove(token));
operations("unsubscribe", removed)
}
// Subscribing a token already subscribed sends nothing back but
// `NO NEW ASSETS`: unsubscribing first makes the server send a fresh
// book.
Command::Resync(token) if tokens.contains(token) => [
operations("unsubscribe", [token]),
operations("subscribe", [token]),
]
.concat(),
Command::Resync(_) => Vec::new(),
}
}
/// The messages that subscribe or unsubscribe `tokens` on an open connection.
fn operations<'a>(operation: &str, tokens: impl IntoIterator<Item = &'a String>) -> Vec<String> {
let tokens: Vec<_> = tokens.into_iter().collect();
tokens
.chunks(CHUNK)
.map(|chunk| json!({"assets_ids": chunk, "operation": operation}).to_string())
.collect()
}
fn connect(url: &str, tokens: &BTreeSet<String>) -> anyhow::Result<Socket> {
let uri: Uri = url.parse()?;
let host = uri
.host()
.ok_or_else(|| anyhow::anyhow!("{url} has no host"))?;
// The market channel is `wss://` on TLS's port; the tests name theirs.
let port = uri.port_u16().unwrap_or(443);
// Timed, unlike `tungstenite::connect`, so a stalled connect or handshake
// can't keep SIGINT waiting. The name lookup itself has no timeout.
let address = (MARKET_CHANNEL_HOST, 443)
let address = (host, port)
.to_socket_addrs()?
.next()
.ok_or_else(|| anyhow::anyhow!("{MARKET_CHANNEL_HOST} has no address"))?;
.ok_or_else(|| anyhow::anyhow!("{host} has no address"))?;
let stream = TcpStream::connect_timeout(&address, CONNECT_TIMEOUT)?;
stream.set_read_timeout(Some(CONNECT_TIMEOUT))?;
stream.set_write_timeout(Some(CONNECT_TIMEOUT))?;
let (mut socket, _response) = tungstenite::client_tls(MARKET_CHANNEL_URL, stream)
let (mut socket, _response) = tungstenite::client_tls(url, stream)
.map_err(|error| anyhow::anyhow!("handshake failed: {error}"))?;
let stream = match socket.get_ref() {
MaybeTlsStream::Plain(stream) => stream,
@@ -74,17 +140,21 @@ fn connect(tokens: &[String]) -> anyhow::Result<Socket> {
_ => anyhow::bail!("unexpected TLS stream"),
};
stream.set_read_timeout(Some(READ_TIMEOUT))?;
// Opened empty, then subscribed in chunks: one rule for every token.
socket.send(Message::text(
json!({"type": "market", "assets_ids": tokens}).to_string(),
json!({"type": "market", "assets_ids": []}).to_string(),
))?;
log!("polymarket: connected, subscribed {} symbols", tokens.len());
for message in operations("subscribe", tokens) {
socket.send(Message::text(message))?;
}
Ok(socket)
}
fn read(
socket: &mut Socket,
tokens: &mut BTreeSet<String>,
frames: &Sender<Frame>,
resyncs: &Receiver<String>,
commands: &Receiver<Command>,
stop: &AtomicBool,
) -> anyhow::Result<Ended> {
let mut last_ping = Instant::now();
@@ -111,25 +181,19 @@ fn read(
socket.send(Message::text("PING"))?;
last_ping = Instant::now();
}
for token in resyncs.try_iter() {
// Subscribing a token already subscribed sends nothing back but
// `NO NEW ASSETS`: unsubscribing first makes the server send a
// fresh book.
socket.send(Message::text(
json!({"assets_ids": [&token], "operation": "unsubscribe"}).to_string(),
))?;
socket.send(Message::text(
json!({"assets_ids": [&token], "operation": "subscribe"}).to_string(),
))?;
for command in commands.try_iter() {
for message in apply(tokens, &command) {
socket.send(Message::text(message))?;
}
}
}
}
/// Closes the connection, forwarding what still arrives until the server
/// confirms or [`CLOSE_TIMEOUT`] passes.
fn close(socket: &mut Socket, frames: &Sender<Frame>) {
fn close(socket: &mut Socket, lane: usize, frames: &Sender<Frame>) {
if let Err(error) = socket.close(None) {
return log!("polymarket: close failed: {error}");
return log!("polymarket: lane {lane}: close failed: {error}");
}
let deadline = Instant::now() + CLOSE_TIMEOUT;
while Instant::now() < deadline {
@@ -143,7 +207,6 @@ fn close(socket: &mut Socket, frames: &Sender<Frame>) {
Err(_) => break,
}
}
log!("polymarket: socket closed");
}
/// A read timeout shows up as `WouldBlock` on macOS and `TimedOut` elsewhere;
@@ -151,3 +214,108 @@ fn close(socket: &mut Socket, frames: &Sender<Frame>) {
fn timed_out(error: &std::io::Error) -> bool {
matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut)
}
#[cfg(test)]
mod tests {
use std::net::TcpListener;
use std::sync::Arc;
use crossbeam::channel::unbounded;
use super::*;
/// The tokens each message of one connection subscribed, after the first.
fn subscribed(socket: &mut WebSocket<TcpStream>, tokens: usize) -> Vec<Vec<String>> {
let first = socket.read().unwrap().into_text().unwrap();
let first: serde_json::Value = serde_json::from_str(&first).unwrap();
assert_eq!(first, json!({"type": "market", "assets_ids": []}));
let mut chunks = Vec::new();
while chunks.iter().map(Vec::len).sum::<usize>() < tokens {
let text = socket.read().unwrap().into_text().unwrap();
let message: serde_json::Value = serde_json::from_str(&text).unwrap();
assert_eq!(message["operation"], "subscribe");
chunks.push(serde_json::from_value(message["assets_ids"].clone()).unwrap());
}
chunks
}
#[test]
fn a_reconnect_subscribes_every_token_again_in_chunks() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("ws://{}/ws/market", listener.local_addr().unwrap());
let tokens: Vec<String> = (0..1_200).map(|token| token.to_string()).collect();
let total = tokens.len();
let server = thread::spawn(move || {
let mut connections = Vec::new();
for _ in 0..2 {
let (stream, _) = listener.accept().unwrap();
// A client that never sends fails the test rather than hangs it.
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
let mut socket = tungstenite::accept(stream).unwrap();
connections.push(subscribed(&mut socket, total));
// The client reconnects after this close; after the second,
// it finds no server and stops.
socket.close(None).unwrap();
while socket.read().is_ok() {}
}
connections
});
let (frames_tx, _frames_rx) = unbounded();
let (commands_tx, commands_rx) = unbounded();
commands_tx
.send(Command::Subscribe(tokens.clone()))
.unwrap();
let stop = Arc::new(AtomicBool::new(false));
let client_stop = Arc::clone(&stop);
let client = thread::spawn(move || run(&url, 0, &frames_tx, &commands_rx, &client_stop));
let connections = server.join().unwrap();
stop.store(true, Ordering::Relaxed);
client.join().unwrap();
let mut tokens = tokens;
tokens.sort();
for chunks in connections {
assert!(chunks.iter().all(|chunk| chunk.len() <= CHUNK));
let mut subscribed = chunks.concat();
subscribed.sort();
assert_eq!(subscribed, tokens);
}
}
#[test]
fn commands_update_the_token_set_and_say_what_to_send() {
let mut tokens = BTreeSet::new();
let operation = |message: &String| -> (String, Vec<String>) {
let message: serde_json::Value = serde_json::from_str(message).unwrap();
let tokens = serde_json::from_value(message["assets_ids"].clone()).unwrap();
(message["operation"].as_str().unwrap().to_string(), tokens)
};
let sent = |messages: Vec<String>| messages.iter().map(operation).collect::<Vec<_>>();
let op = |name: &str, tokens: &[&str]| {
(
name.to_string(),
tokens.iter().map(|token| token.to_string()).collect(),
)
};
let subscribe = Command::Subscribe(vec!["a".into(), "b".into()]);
assert_eq!(
sent(apply(&mut tokens, &subscribe)),
[op("subscribe", &["a", "b"])]
);
assert_eq!(
sent(apply(&mut tokens, &Command::Resync("a".into()))),
[op("unsubscribe", &["a"]), op("subscribe", &["a"])]
);
let unsubscribe = Command::Unsubscribe(vec!["a".into()]);
assert_eq!(
sent(apply(&mut tokens, &unsubscribe)),
[op("unsubscribe", &["a"])]
);
assert!(apply(&mut tokens, &Command::Resync("a".into())).is_empty());
assert_eq!(tokens.into_iter().collect::<Vec<_>>(), ["b"]);
}
}