feat(polymarket): track every live market over load-balanced lanes

An orchestrator keeps the tracked set equal to Gamma's live markets (a scan at
startup and every hour, new and closed markets every minute, reaching back 15
minutes for Gamma's indexing delay) and spreads them over lanes, each a
connection and a book thread, all writing through the one writer. Lanes report
each market's load in bytes per second; the orchestrator puts new markets on
the least-loaded lane and moves the busiest markets off lanes over a 300 KB/s
budget, busiest lane first. Differing books our own subscribe explains are
subscription gaps, not losses, and every first book after a move is logged,
so reconcile's mismatches are all accounted for in the log. A 75-minute live
run tracked ~195k markets (~390k symbols) in 89 lanes.
This commit is contained in:
pascal committed 2026-11-11 06:28:25 +00:00
1 parent adb7fee8d8
commit 8114a51dc2
18 files changed
+2826 -320

No files matched your search

+678
View File
@@ -0,0 +1,678 @@
//! The orchestrator: the one owner of what is tracked. It keeps the tracked
//! markets equal to Gamma's live ones, spreads them over lanes and owns every
//! timer; the two fetchers only fetch (roadmap/03, ADR 0002).
use std::cmp::Reverse;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant};
use chrono::{DateTime, TimeDelta, Utc};
use crossbeam::channel::{Sender, select, unbounded};
use engineering_challenge::book::Output;
use super::feed::{self, Feed, LaneCommand, LoadReport, Loads};
use super::gamma::{self, Finding, Request};
use super::{Market, socket};
/// The most tokens a lane holds: a reconnect replays every one's book at once.
const LANE_TOKENS: usize = 5_000;
/// A lane's byte budget, under the lowest rate at which a connection was seen
/// to get a 1013 close (0.46 MB/s; docs/notes.md). Closes came at up to
/// 5.1 MB/s too: no rate is safe, only less likely to be closed.
const LANE_BYTES_PER_SECOND: u64 = 300_000;
/// Moves are limited so balancing doesn't churn. Rebalancing runs with each
/// minute check, every [`RECENT_EVERY`].
const MOVES_PER_MINUTE: usize = 50;
const MOVED_STAYS: Duration = Duration::from_secs(10 * 60);
const RECENT_EVERY: Duration = Duration::from_secs(60);
const RESCAN_EVERY: Duration = Duration::from_secs(60 * 60);
/// Gamma's indexing delay: each recent check reaches this far before the
/// last successful one. New markets were listed up to 5 minutes after their
/// `createdAt` (docs/notes.md); known markets are skipped, so the margin costs
/// only the pages it reads again.
const RECENT_MARGIN: TimeDelta = TimeDelta::minutes(15);
/// So SIGINT is seen while no timer is due.
const WAKE_EVERY: Duration = Duration::from_secs(1);
/// Runs until `stop` is set, then waits for every lane. Returns how many
/// threads failed: the lane threads that panicked, and one if a fetcher died.
pub fn run(outputs: Sender<Output>, stop: &Arc<AtomicBool>) -> usize {
let (findings_tx, findings) = unbounded();
let (rescan, rescan_requests) = unbounded();
let (recent, recent_requests) = unbounded();
// Two, so a full scan never holds up the minute check. Not joined: they
// hold no output, and each ends at its next send.
let fetchers: Vec<JoinHandle<()>> = [rescan_requests, recent_requests]
.into_iter()
.map(|requests| {
let findings = findings_tx.clone();
thread::spawn(move || gamma::run(requests, findings))
})
.collect();
drop(findings_tx);
let (reports_tx, reports) = unbounded::<LoadReport>();
let mut tracker = Tracker::default();
let mut lanes: Vec<Lane> = Vec::new();
let (mut added, mut removed, mut moved) = (0, 0, 0);
// Startup is the first rescan; the minute checks cover it from its start.
let mut recent_since = Utc::now() - RECENT_MARGIN;
let mut recent_asked_at: Option<DateTime<Utc>> = None;
let mut scanning = false;
let mut scanned = false;
let mut fetcher_died = false;
let mut next_rescan = Instant::now();
let mut next_recent = Instant::now() + RECENT_EVERY;
// Opens the lane on its first command.
let send = |lanes: &mut Vec<Lane>, lane: usize, command: LaneCommand| {
if lane == lanes.len() {
lanes.push(open_lane(lane, &outputs, &reports_tx, stop));
}
// A lane whose book thread died keeps its markets unserved: lane
// health is M5's.
_ = lanes[lane].commands.send(command);
};
while !stop.load(Ordering::Relaxed) {
// Only a panic ends a fetcher while this runs, even one in the middle
// of a request, whose answer would never come: nothing would keep the
// set current.
if fetchers.iter().any(JoinHandle::is_finished) {
log!("polymarket: orchestrator: a fetcher died, stopping");
stop.store(true, Ordering::Relaxed);
fetcher_died = true;
break;
}
let now = Instant::now();
if now >= next_rescan {
if !scanning {
_ = rescan.send(Request::Rescan);
scanning = true;
}
next_rescan = now + RESCAN_EVERY;
}
if now >= next_recent {
if recent_asked_at.is_none() {
_ = recent.send(Request::Recent {
since: recent_since,
});
recent_asked_at = Some(Utc::now());
}
next_recent = now + RECENT_EVERY;
// On the loads the lanes reported at the wall-clock minute.
let moves = tracker.rebalance(now);
moved += moves.len();
for (lane, command) in moves.into_iter().flat_map(Move::commands) {
send(&mut lanes, lane, command);
}
let busiest = tracker.lanes.iter().map(|lane| lane.load).max();
log!(
"polymarket: orchestrator: {} markets in {} lanes; {added} added, {removed} \
removed, {moved} moved this minute; busiest lane at {} bytes/s",
tracker.markets.len(),
lanes.len(),
busiest.unwrap_or(0)
);
(added, removed, moved) = (0, 0, 0);
}
let timeout = next_rescan
.min(next_recent)
.saturating_duration_since(Instant::now())
.min(WAKE_EVERY);
let finding = select! {
recv(findings) -> finding => match finding {
Ok(finding) => finding,
// Both fetchers are gone: the check above stops the run.
Err(_) => continue,
},
recv(reports) -> report => {
if let Ok(report) = report {
tracker.report(report.lane, &report.loads);
}
continue;
},
default(timeout) => continue,
};
let commands = match finding {
Finding::ScanStarted => {
tracker.scan_started();
Vec::new()
}
Finding::ScanPage(markets) | Finding::Created(markets) => markets
.into_iter()
.filter_map(|market| tracker.add(market))
.collect(),
Finding::ScanDone => {
scanning = false;
scanned = true;
tracker.scan_done()
}
Finding::ScanFailed => {
scanning = false;
tracker.scan_failed();
// Nothing can be tracked before a first scan: retry at once.
if !scanned {
next_rescan = Instant::now();
}
Vec::new()
}
Finding::Closed(ids) => ids.iter().filter_map(|id| tracker.remove(id)).collect(),
Finding::RecentDone => {
if let Some(asked_at) = recent_asked_at.take() {
recent_since = asked_at - RECENT_MARGIN;
}
Vec::new()
}
Finding::RecentFailed => {
recent_asked_at = None;
Vec::new()
}
};
for (lane, command) in commands {
match command {
LaneCommand::Add(_) => added += 1,
LaneCommand::Remove(_) => removed += 1,
// Only rebalancing moves, counted there.
LaneCommand::MoveIn(_) => {}
}
send(&mut lanes, lane, command);
}
}
drop(outputs);
let panicked = lanes
.into_iter()
.flat_map(|lane| lane.threads)
.map(JoinHandle::join)
.filter(Result::is_err)
.count();
panicked + usize::from(fetcher_died)
}
struct Lane {
commands: Sender<LaneCommand>,
threads: [JoinHandle<()>; 2],
}
/// Lane `number`'s connection and book thread, joined by [`run`].
fn open_lane(
number: usize,
outputs: &Sender<Output>,
reports: &Sender<LoadReport>,
stop: &Arc<AtomicBool>,
) -> Lane {
let (commands, commands_rx) = unbounded();
let (frames, frames_rx) = unbounded();
let (connection, connection_rx) = unbounded();
let stop = Arc::clone(stop);
let outputs = outputs.clone();
let reports = reports.clone();
let connection_thread = thread::spawn(move || {
socket::run(
socket::MARKET_CHANNEL_URL,
number,
&frames,
&connection_rx,
&stop,
)
});
let book_thread = thread::spawn(move || {
feed::run(
Feed::new(number),
frames_rx,
commands_rx,
outputs,
connection,
reports,
)
});
Lane {
commands,
threads: [connection_thread, book_thread],
}
}
/// A market leaving lane `from` for lane `to`.
#[derive(Debug)]
struct Move {
market: Market,
from: usize,
to: usize,
}
impl Move {
/// A Remove for the old lane, then a MoveIn for the new one, in the order
/// they are sent.
fn commands(self) -> [(usize, LaneCommand); 2] {
[
(self.from, LaneCommand::Remove(self.market.clone())),
(self.to, LaneCommand::MoveIn(self.market)),
]
}
}
/// A tracked market, where it is and what it weighs.
struct Tracked {
market: Market,
lane: usize,
/// Bytes per second over its lane's last report; 0 until one comes.
load: u64,
moved_at: Option<Instant>,
}
/// A lane's tokens and load, against its two limits.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
struct LaneLoad {
tokens: usize,
/// The sum of its markets' loads, bytes per second.
load: u64,
}
/// Which lane each market is in and what it weighs, and which markets a
/// rescan in progress hasn't seen yet.
#[derive(Default)]
struct Tracker {
/// By `conditionId`.
markets: HashMap<String, Tracked>,
/// By lane number.
lanes: Vec<LaneLoad>,
/// While a rescan runs: the markets known at its start, not seen since.
unseen: Option<HashSet<String>>,
}
impl Tracker {
/// A market it doesn't know goes to the least-loaded lane with token room,
/// a new lane if none has: the lane and its command.
fn add(&mut self, market: Market) -> Option<(usize, LaneCommand)> {
if let Some(unseen) = &mut self.unseen {
unseen.remove(&market.condition_id);
}
if self.markets.contains_key(&market.condition_id) {
return None;
}
let lane = self.lane_for(market.tokens.len(), None, None);
self.lanes[lane].tokens += market.tokens.len();
let tracked = Tracked {
market: market.clone(),
lane,
load: 0,
moved_at: None,
};
self.markets.insert(market.condition_id.clone(), tracked);
Some((lane, LaneCommand::Add(market)))
}
/// The least-loaded lane (then the one with fewest tokens), other than
/// `except`, with room for `tokens` more and, if given, `load` more
/// bytes/s; a new lane if none has.
fn lane_for(&mut self, tokens: usize, load: Option<u64>, except: Option<usize>) -> usize {
self.lanes
.iter()
.enumerate()
.filter(|&(lane, lane_load)| {
Some(lane) != except
&& lane_load.tokens + tokens <= LANE_TOKENS
&& load.is_none_or(|load| lane_load.load + load <= LANE_BYTES_PER_SECOND)
})
.min_by_key(|&(_, lane_load)| (lane_load.load, lane_load.tokens))
.map(|(lane, _)| lane)
.unwrap_or_else(|| {
self.lanes.push(LaneLoad::default());
self.lanes.len() - 1
})
}
fn remove(&mut self, condition_id: &str) -> Option<(usize, LaneCommand)> {
let tracked = self.markets.remove(condition_id)?;
let lane = &mut self.lanes[tracked.lane];
lane.tokens -= tracked.market.tokens.len();
lane.load -= tracked.load;
Some((tracked.lane, LaneCommand::Remove(tracked.market)))
}
/// A lane's loads per market. A market reported by a lane it has since
/// moved from keeps the load it has.
fn report(&mut self, lane: usize, loads: &Loads) {
for (condition_id, &load) in loads {
let Some(tracked) = self.markets.get_mut(condition_id) else {
continue;
};
if tracked.lane != lane {
continue;
}
let lane_load = &mut self.lanes[lane];
lane_load.load = lane_load.load - tracked.load + load;
tracked.load = load;
}
}
/// Moves the busiest markets off every lane over its byte budget, busiest
/// lane first, to the least-loaded lanes with room for them, until it is
/// under: at most [`MOVES_PER_MINUTE`], and none moved in the last
/// [`MOVED_STAYS`].
fn rebalance(&mut self, now: Instant) -> Vec<Move> {
let mut moves = Vec::new();
let mut over: Vec<usize> = (0..self.lanes.len())
.filter(|&lane| self.lanes[lane].load > LANE_BYTES_PER_SECOND)
.collect();
over.sort_by_key(|&lane| Reverse(self.lanes[lane].load));
for from in over {
// ponytail: a scan of every market per lane over budget, fine at a
// few hundred thousand once a minute; per-lane market sets if not.
let mut busiest: Vec<(String, u64)> = self
.markets
.iter()
.filter(|(_, tracked)| {
tracked.lane == from
&& tracked
.moved_at
.is_none_or(|at| now.duration_since(at) >= MOVED_STAYS)
})
.map(|(id, tracked)| (id.clone(), tracked.load))
.collect();
busiest.sort_by_key(|&(_, load)| Reverse(load));
for (id, load) in busiest {
let size = self.markets[&id].market.tokens.len();
// Alone on its lane, a market would only take its load along;
// one with no load would lighten nothing.
if self.lanes[from].load <= LANE_BYTES_PER_SECOND
|| moves.len() == MOVES_PER_MINUTE
|| self.lanes[from].tokens == size
|| load == 0
{
break;
}
let to = self.lane_for(size, Some(load), Some(from));
self.lanes[from].tokens -= size;
self.lanes[from].load -= load;
self.lanes[to].tokens += size;
self.lanes[to].load += load;
let tracked = self.markets.get_mut(&id).expect("listed above");
tracked.lane = to;
tracked.moved_at = Some(now);
moves.push(Move {
market: tracked.market.clone(),
from,
to,
});
}
}
moves
}
fn scan_started(&mut self) {
self.unseen = Some(self.markets.keys().cloned().collect());
}
/// Removes the markets known at the scan's start and not seen in it:
/// those added since were never at risk.
fn scan_done(&mut self) -> Vec<(usize, LaneCommand)> {
let unseen = self.unseen.take().unwrap_or_default();
unseen.iter().filter_map(|id| self.remove(id)).collect()
}
/// Half a listing says nothing of the other half: nothing is removed.
fn scan_failed(&mut self) {
self.unseen = None;
}
}
#[cfg(test)]
mod tests {
use super::*;
fn market(id: usize) -> Market {
Market {
condition_id: format!("0x{id}"),
tokens: vec![format!("{id}a"), format!("{id}b")],
min_order_size: "5".to_string(),
neg_risk: false,
}
}
fn lanes(commands: &[(usize, LaneCommand)]) -> Vec<usize> {
commands.iter().map(|(lane, _)| *lane).collect()
}
fn tokens(tracker: &Tracker) -> Vec<usize> {
tracker.lanes.iter().map(|lane| lane.tokens).collect()
}
/// Reports `bytes_per_second` for each market, as its lane would.
fn report(tracker: &mut Tracker, loads: &[(usize, u64)]) {
for &(id, bytes_per_second) in loads {
let lane = tracker.markets[&format!("0x{id}")].lane;
let loads = Loads::from([(format!("0x{id}"), bytes_per_second)]);
tracker.report(lane, &loads);
}
}
#[test]
fn a_market_joins_the_least_loaded_lane_with_room_and_lanes_open_when_full() {
let mut tracker = Tracker::default();
let per_lane = LANE_TOKENS / 2;
let first: Vec<_> = (0..per_lane + 1)
.filter_map(|id| tracker.add(market(id)))
.collect();
// Nothing measured yet: the first lane fills, then a second opens.
assert!(lanes(&first[..per_lane]).iter().all(|&lane| lane == 0));
assert_eq!(first[per_lane].0, 1);
// A hole in lane 0 draws the next market once lane 1 weighs more.
tracker.remove("0x0").unwrap();
report(&mut tracker, &[(per_lane, 1_000)]);
assert_eq!(tracker.add(market(per_lane + 1)).unwrap().0, 0);
// Both tokens of a market share its lane.
assert_eq!(tracker.markets["0x1"].lane, 0);
assert_eq!(tokens(&tracker), [LANE_TOKENS, 2]);
}
#[test]
fn between_lanes_of_equal_load_a_market_joins_the_one_with_fewest_tokens() {
let mut tracker = Tracker {
lanes: vec![
LaneLoad { tokens: 4, load: 0 },
LaneLoad { tokens: 2, load: 0 },
],
..Tracker::default()
};
assert_eq!(lanes(&[tracker.add(market(0)).unwrap()]), [1]);
assert_eq!(tokens(&tracker), [4, 4]);
}
#[test]
fn a_known_market_is_skipped() {
let mut tracker = Tracker::default();
assert!(tracker.add(market(1)).is_some());
assert!(tracker.add(market(1)).is_none());
assert_eq!(tokens(&tracker), [2]);
}
#[test]
fn a_report_sets_its_lanes_markets_and_no_others() {
let mut tracker = Tracker::default();
tracker.add(market(1));
tracker.add(market(2));
report(&mut tracker, &[(1, 300), (2, 100)]);
assert_eq!(tracker.lanes[0].load, 400);
// A lane the market has left reports it late: ignored.
tracker.lanes.push(LaneLoad::default());
let stale = Loads::from([("0x1".to_string(), 900)]);
tracker.report(1, &stale);
assert_eq!(tracker.markets["0x1"].load, 300);
assert_eq!(tracker.lanes[1].load, 0);
// A market gone takes its load along.
tracker.remove("0x1");
assert_eq!(tracker.lanes[0].load, 100);
}
#[test]
fn a_lane_over_budget_moves_its_busiest_markets_until_under() {
let mut tracker = Tracker::default();
for id in 0..10 {
tracker.add(market(id));
}
tracker.lanes.push(LaneLoad::default());
let budget = LANE_BYTES_PER_SECOND;
// Lane 0 at 1.5 budgets: its busiest market (half a budget) moving
// is enough.
let mut loads = vec![(0, budget / 2)];
loads.extend((1..10).map(|id| (id, budget / 9)));
report(&mut tracker, &loads);
let now = Instant::now();
let moves = tracker.rebalance(now);
assert!(matches!(
moves.as_slice(),
[Move { market, from: 0, to: 1 }] if market.condition_id == "0x0"
));
assert_eq!(tracker.markets["0x0"].lane, 1);
let [
(0, LaneCommand::Remove(removed)),
(1, LaneCommand::MoveIn(added)),
] = moves.into_iter().next().unwrap().commands()
else {
panic!("a move is a Remove for the old lane, then a MoveIn for the new");
};
assert_eq!(
(removed.condition_id, added.condition_id),
("0x0".into(), "0x0".into())
);
assert!(tracker.lanes[0].load <= budget);
assert_eq!(tokens(&tracker), [18, 2]);
assert_eq!(tracker.lanes[1].load, budget / 2);
// Under budget now: nothing more moves.
assert!(tracker.rebalance(now).is_empty());
}
#[test]
fn the_busiest_lane_over_budget_is_rebalanced_first() {
let mut tracker = Tracker::default();
tracker.add(market(0));
tracker.add(market(1));
tracker.lanes.push(LaneLoad::default());
// The empty lane is the least loaded: both join it.
tracker.add(market(2));
tracker.add(market(3));
assert_eq!(tokens(&tracker), [4, 4]);
let budget = LANE_BYTES_PER_SECOND;
report(&mut tracker, &[(0, budget), (1, budget / 10)]);
report(&mut tracker, &[(2, budget / 2), (3, budget)]);
let moves = tracker.rebalance(Instant::now());
assert!(matches!(
moves.first(),
Some(Move { market, from: 1, .. }) if market.condition_id == "0x3"
));
}
/// The markets that may move have no load: moving them would empty the
/// lane and lighten nothing.
#[test]
fn markets_with_no_load_are_not_moved() {
let mut tracker = Tracker::default();
for id in 0..4 {
tracker.add(market(id));
}
report(&mut tracker, &[(0, 2 * LANE_BYTES_PER_SECOND)]);
let now = Instant::now();
tracker.markets.get_mut("0x0").unwrap().moved_at = Some(now);
assert!(tracker.rebalance(now).is_empty());
}
#[test]
fn moves_are_limited_and_a_moved_market_stays() {
let mut tracker = Tracker::default();
let count = 2 * MOVES_PER_MINUTE;
for id in 0..count {
tracker.add(market(id));
}
// Every market alone near a budget: each needs a lane of its own.
let loads: Vec<_> = (0..count)
.map(|id| (id, LANE_BYTES_PER_SECOND * 9 / 10))
.collect();
report(&mut tracker, &loads);
let now = Instant::now();
let first = tracker.rebalance(now);
assert_eq!(first.len(), MOVES_PER_MINUTE);
// The rest move a minute later, but the last, alone and under budget.
let second = tracker.rebalance(now + Duration::from_secs(60));
assert_eq!(second.len(), MOVES_PER_MINUTE - 1);
assert!(second.iter().all(|moved| moved.from == 0));
// Ten minutes on, lane 0 holds one market: it is left alone.
assert!(tracker.rebalance(now + MOVED_STAYS).is_empty());
assert_eq!(tracker.lanes[0].tokens, 2);
}
/// Over budget on its own, a market would only carry its load from lane
/// to lane every ten minutes.
#[test]
fn a_market_alone_on_its_lane_is_not_moved() {
let mut tracker = Tracker::default();
tracker.add(market(0));
report(&mut tracker, &[(0, 2 * LANE_BYTES_PER_SECOND)]);
assert!(tracker.rebalance(Instant::now()).is_empty());
assert_eq!(tracker.lanes.len(), 1);
}
#[test]
fn a_market_moved_less_than_ten_minutes_ago_stays() {
let mut tracker = Tracker::default();
let budget = LANE_BYTES_PER_SECOND;
tracker.add(market(0));
tracker.add(market(1));
report(&mut tracker, &[(0, budget * 7 / 10), (1, budget / 2)]);
tracker.lanes.push(LaneLoad::default());
let now = Instant::now();
tracker.add(market(2));
assert_eq!(tracker.markets["0x2"].lane, 1);
report(&mut tracker, &[(2, budget / 10)]);
// Market 0 moves to lane 1, which then turns busy.
assert!(matches!(
tracker.rebalance(now).as_slice(),
[Move { from: 0, to: 1, .. }]
));
report(&mut tracker, &[(0, budget * 7 / 10), (2, budget * 6 / 10)]);
let moves = tracker.rebalance(now + Duration::from_secs(60));
assert!(matches!(
moves.as_slice(),
[Move { market, from: 1, to: 2 }] if market.condition_id == "0x2"
));
}
#[test]
fn a_completed_rescan_removes_only_the_known_markets_it_did_not_see() {
let mut tracker = Tracker::default();
for id in 0..3 {
tracker.add(market(id));
}
tracker.scan_started();
tracker.add(market(0));
// Added by a minute check during the scan, on a page already passed.
tracker.add(market(9));
// Closed during the scan.
tracker.remove("0x1");
let removed = tracker.scan_done();
assert!(matches!(
removed.as_slice(),
[(0, LaneCommand::Remove(market))] if market.condition_id == "0x2"
));
let mut tracked: Vec<_> = tracker.markets.keys().cloned().collect();
tracked.sort();
assert_eq!(tracked, ["0x0", "0x9"]);
assert_eq!(tokens(&tracker), [4]);
}
#[test]
fn a_failed_rescan_removes_nothing() {
let mut tracker = Tracker::default();
tracker.add(market(0));
tracker.scan_started();
tracker.scan_failed();
assert!(tracker.scan_done().is_empty());
assert_eq!(tracker.markets.len(), 1);
}
}