This repository has been archived on 2026-11-16. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
orderbook-collector/src/polymarket/orchestrator.rs
T
pascal 02747b882f feat(polymarket)!: two connections per market, first one wins
Every live market is subscribed on two connections chosen at random, and its
book takes each change from whichever connection delivers it first: the book
hash says whether a change is a duplicate, the next one, or one to hold until
the change before it arrives. A change one connection misses costs nothing,
each Delta carries the earliest receive time, and a connection that drops is
covered by the other. When a change fits on no connection in time, the book
follows one connection unchecked and asks the other for a fresh book.
Connections now decode and route events to 16 book threads, independent of
them; a move leaves its old connection only once the new one delivers, so it
costs no change; a connection closed with 1013 twice within ten minutes is
split. reconcile reads the collector's log and fails only on a mismatch no
cause line explains. SIGTERM and SIGHUP stop the collector cleanly.

Breaking: lanes are gone (connections and book threads are separate), and
reconcile takes the log as a second argument.
2026-11-11 15:18:48 +00:00

1018 lines
40 KiB
Rust

//! The orchestrator: the one owner of what is tracked. It keeps the tracked
//! markets equal to Gamma's live ones, decides which connections carry each
//! market and owns every timer; the two fetchers only fetch (ADR 0002, ADR
//! 0003, roadmap/04).
use std::cmp::Reverse;
use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use chrono::{DateTime, TimeDelta, Utc};
use crossbeam::channel::{Sender, select, unbounded};
use engineering_challenge::book::Output;
use super::feed::{self, BookCommand, Feed, Loads};
use super::gamma::{self, Finding, Request};
use super::wire::Received;
use super::{BOOK_THREADS, Market, book_thread, socket};
/// How many connections carry each market, first one wins (ADR 0003). 1 for
/// a run to compare with.
const SUBSCRIPTIONS: usize = 2;
/// The most tokens a connection holds: a reconnect replays every one's book
/// at once.
const CONNECTION_TOKENS: usize = 5_000;
/// What the startup deal fills a connection to, leaving room for new markets.
const STARTUP_FILL: usize = 4_500;
/// A connection's byte budget, under the lowest rate at which one was seen to
/// get a 1013 close (0.46 MB/s; docs/notes.md). Closes came at up to
/// 5.1 MB/s too: no rate is safe, only less likely to be closed.
const CONNECTION_BYTES_PER_SECOND: u64 = 300_000;
/// The startup deal's pace, about 5,000 markets a second: every connection
/// subscribing at once would send more opening books than the link carries.
const DEAL_AT_ONCE: usize = 500;
const DEAL_EVERY: Duration = Duration::from_millis(100);
/// Moves are limited so balancing doesn't churn. Rebalancing runs with each
/// minute check, every [`RECENT_EVERY`].
const MOVES_PER_MINUTE: usize = 50;
const MOVED_STAYS: Duration = Duration::from_secs(10 * 60);
/// A connection closed with 1013 twice within this is split: one close can be
/// the server's bad moment (several connections within a second), a second
/// one says it carries too much. It is then left alone for as long.
const SPLIT_WITHIN: Duration = Duration::from_secs(10 * 60);
const RECENT_EVERY: Duration = Duration::from_secs(60);
const RESCAN_EVERY: Duration = Duration::from_secs(60 * 60);
/// Gamma's indexing delay: each recent check reaches this far before the
/// last successful one. New markets were listed up to 5 minutes after their
/// `createdAt` (docs/notes.md); known markets are skipped, so the margin costs
/// only the pages it reads again.
const RECENT_MARGIN: TimeDelta = TimeDelta::minutes(15);
/// So SIGINT is seen while no timer is due.
const WAKE_EVERY: Duration = Duration::from_secs(1);
/// Runs until `stop` is set, then waits for every connection and book thread.
/// Returns how many threads failed: those that panicked, and one if a fetcher
/// died.
pub fn run(outputs: Sender<Output>, stop: &Arc<AtomicBool>) -> usize {
let (findings_tx, findings) = unbounded();
let (rescan, rescan_requests) = unbounded();
let (recent, recent_requests) = unbounded();
// Two, so a full scan never holds up the minute check. Not joined: they
// hold no output, and each ends at its next send.
let fetchers: Vec<JoinHandle<()>> = [rescan_requests, recent_requests]
.into_iter()
.map(|requests| {
let findings = findings_tx.clone();
thread::spawn(move || gamma::run(requests, findings))
})
.collect();
drop(findings_tx);
let (reports_tx, reports) = unbounded::<Loads>();
let (closes_tx, closes) = unbounded::<usize>();
let mut threads = Threads::start(&outputs, &reports_tx, &closes_tx, stop);
drop(closes_tx);
// The book threads hold the only senders the writer waits for.
drop(outputs);
drop(reports_tx);
let seed = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(1, |since| since.as_nanos() as u64);
let mut tracker = Tracker::new(SUBSCRIPTIONS, seed);
// Until the first scan is done: the live markets it found, waiting to be
// dealt at random. Dealing page by page would keep markets created
// together on the same connections.
let mut first_scan: Option<Vec<Market>> = Some(Vec::new());
// The startup deal's commands still to send, and whatever came behind
// them: a market's commands keep their order.
let mut queue: VecDeque<BookCommand> = VecDeque::new();
let mut next_deal = Instant::now();
let (mut added, mut removed, mut moved, mut splits) = (0, 0, 0, 0);
// The minute checks cover the first scan from its start.
let mut recent_since = Utc::now() - RECENT_MARGIN;
let mut recent_asked_at: Option<DateTime<Utc>> = None;
let mut scanning = false;
let mut scanned = false;
let mut fetcher_died = false;
let mut next_rescan = Instant::now();
let mut next_recent = Instant::now() + RECENT_EVERY;
while !stop.load(Ordering::Relaxed) {
// Only a panic ends a fetcher while this runs, even one in the middle
// of a request, whose answer would never come: nothing would keep the
// set current.
if fetchers.iter().any(JoinHandle::is_finished) {
log!("polymarket: orchestrator: a fetcher died, stopping");
stop.store(true, Ordering::Relaxed);
fetcher_died = true;
break;
}
let now = Instant::now();
let mut commands = Vec::new();
if now >= next_rescan {
if !scanning {
_ = rescan.send(Request::Rescan);
scanning = true;
}
next_rescan = now + RESCAN_EVERY;
}
if now >= next_recent {
if recent_asked_at.is_none() {
_ = recent.send(Request::Recent {
since: recent_since,
});
recent_asked_at = Some(Utc::now());
}
next_recent = now + RECENT_EVERY;
// On the loads the book threads reported at the wall-clock minute.
let moves = tracker.rebalance(now);
moved += moves.len();
commands.extend(moves);
let busiest = tracker.connections.iter().map(|one| one.load).max();
log!(
"polymarket: orchestrator: {} markets on {} connections; {added} added, {removed} \
removed, {moved} moved, {splits} splits this minute; busiest connection at {} bytes/s",
tracker.markets.len(),
tracker.connections.len(),
busiest.unwrap_or(0)
);
(added, removed, moved, splits) = (0, 0, 0, 0);
}
if !queue.is_empty() && now >= next_deal {
for command in queue.drain(..DEAL_AT_ONCE.min(queue.len())) {
threads.send(command);
}
next_deal = now + DEAL_EVERY;
}
let mut timeout = next_rescan
.min(next_recent)
.saturating_duration_since(Instant::now())
.min(WAKE_EVERY);
if !queue.is_empty() {
timeout = timeout.min(next_deal.saturating_duration_since(Instant::now()));
}
select! {
recv(findings) -> finding => match finding {
Ok(Finding::ScanStarted) => {
tracker.scan_started();
// A first scan that failed starts over.
if let Some(waiting) = &mut first_scan {
waiting.clear();
}
}
Ok(Finding::ScanPage(markets) | Finding::Created(markets)) => match &mut first_scan {
Some(waiting) => waiting.extend(markets),
None => {
let adds: Vec<_> = markets
.into_iter()
.filter_map(|market| tracker.add(market))
.collect();
added += adds.len();
commands.extend(adds);
}
},
Ok(Finding::ScanDone) => {
scanning = false;
scanned = true;
match first_scan.take() {
Some(markets) => {
let adds = tracker.deal(markets);
added += adds.len();
commands.extend(adds);
}
None => {
let gone = tracker.scan_done();
removed += gone.len();
commands.extend(gone);
}
}
}
Ok(Finding::ScanFailed) => {
scanning = false;
tracker.scan_failed();
// Nothing can be tracked before a first scan: retry at once.
if !scanned {
next_rescan = Instant::now();
}
}
Ok(Finding::Closed(ids)) => {
// Closed while the first scan runs: never dealt.
if let Some(waiting) = &mut first_scan {
waiting.retain(|market| !ids.contains(&market.condition_id));
}
let gone: Vec<_> = ids.iter().filter_map(|id| tracker.remove(id)).collect();
removed += gone.len();
commands.extend(gone);
}
Ok(Finding::RecentDone) => {
if let Some(asked_at) = recent_asked_at.take() {
recent_since = asked_at - RECENT_MARGIN;
}
}
Ok(Finding::RecentFailed) => recent_asked_at = None,
// Both fetchers are gone: the check above stops the run.
Err(_) => {}
},
recv(reports) -> loads => {
if let Ok(loads) = loads {
tracker.report(&loads);
}
},
recv(closes) -> closed => {
if let Ok(connection) = closed {
let moves = tracker.closed(connection, Instant::now());
if !moves.is_empty() {
log!(
"polymarket: orchestrator: connection {connection} closed with 1013 \
twice, split: {} markets moved off",
moves.len()
);
splits += 1;
moved += moves.len();
}
commands.extend(moves);
}
},
default(timeout) => {}
}
// A connection is open before any command names it.
threads.open(tracker.connections.len());
// The startup deal is paced; what comes while it lasts waits behind it.
if commands.len() > DEAL_AT_ONCE || !queue.is_empty() {
queue.extend(commands);
} else {
for command in commands {
threads.send(command);
}
}
}
usize::from(fetcher_died) + threads.join()
}
/// The threads the orchestrator feeds: the book threads, there from the
/// start, and the connections it opens.
struct Threads {
book_commands: Vec<Sender<BookCommand>>,
/// Every book thread's events channel, by number: each connection gets a
/// clone of them all.
events: Vec<Sender<Received>>,
closes: Sender<usize>,
book_threads: Vec<JoinHandle<()>>,
connections: Vec<JoinHandle<()>>,
stop: Arc<AtomicBool>,
}
impl Threads {
fn start(
outputs: &Sender<Output>,
reports: &Sender<Loads>,
closes: &Sender<usize>,
stop: &Arc<AtomicBool>,
) -> Self {
let mut threads = Self {
book_commands: Vec::new(),
events: Vec::new(),
closes: closes.clone(),
book_threads: Vec::new(),
connections: Vec::new(),
stop: Arc::clone(stop),
};
for number in 0..BOOK_THREADS {
let (commands, commands_rx) = unbounded();
let (events, events_rx) = unbounded();
let (outputs, reports) = (outputs.clone(), reports.clone());
threads.book_commands.push(commands);
threads.events.push(events);
threads.book_threads.push(thread::spawn(move || {
feed::run(Feed::new(number), events_rx, commands_rx, outputs, reports)
}));
}
threads
}
/// Opens connections until there are `count`, telling every book thread
/// how to reach each.
fn open(&mut self, count: usize) {
while self.connections.len() < count {
let number = self.connections.len();
let (commands, commands_rx) = unbounded();
let (events, stop) = (self.events.clone(), Arc::clone(&self.stop));
let closes = self.closes.clone();
self.connections.push(thread::spawn(move || {
socket::run(
socket::MARKET_CHANNEL_URL,
number,
&events,
&commands_rx,
&closes,
&stop,
)
}));
for book_thread in &self.book_commands {
_ = book_thread.send(BookCommand::Connected(number, commands.clone()));
}
}
}
/// To the market's book thread. One that died keeps its markets unserved,
/// and its connections stop the run.
fn send(&self, command: BookCommand) {
let market = match &command {
BookCommand::Add(market, _)
| BookCommand::Move { market, .. }
| BookCommand::Remove(market) => market,
BookCommand::Connected(..) => unreachable!("sent by open() to every book thread"),
};
_ = self.book_commands[book_thread(&market.condition_id)].send(command);
}
/// Waits for the connections, which stop on the flag, then for the book
/// threads, which stop once no connection can send them an event: how
/// many panicked.
fn join(self) -> usize {
let panicked = |threads: Vec<JoinHandle<()>>| {
threads
.into_iter()
.map(JoinHandle::join)
.filter(Result::is_err)
.count()
};
let connections = panicked(self.connections);
drop(self.events);
drop(self.book_commands);
connections + panicked(self.book_threads)
}
}
/// A tracked market, where it is and what it weighs.
struct Tracked {
market: Market,
/// The connections that carry it, all different.
connections: Vec<usize>,
/// Bytes per second on one connection, over its book thread's last
/// report; 0 until one comes.
load: u64,
moved_at: Option<Instant>,
}
/// A connection's tokens and load, against its two limits, and its 1013
/// closes.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
struct ConnectionLoad {
tokens: usize,
/// The sum of its markets' loads, bytes per second.
load: u64,
/// When it was last closed with 1013.
closed_at: Option<Instant>,
split_at: Option<Instant>,
}
/// Which connections carry each market and what it weighs, and which markets
/// a rescan in progress hasn't seen yet.
struct Tracker {
subscriptions: usize,
/// By `conditionId`.
markets: HashMap<String, Tracked>,
/// By connection number.
connections: Vec<ConnectionLoad>,
/// While a rescan runs: the markets known at its start, not seen since.
unseen: Option<HashSet<String>>,
random: Random,
}
impl Tracker {
fn new(subscriptions: usize, seed: u64) -> Self {
Self {
subscriptions,
markets: HashMap::new(),
connections: Vec::new(),
unseen: None,
random: Random(seed | 1),
}
}
/// The startup deal: every market on `subscriptions` different
/// connections, each subscription dealt in an order of its own, so a
/// market's connections hold different neighbours and no connection gets
/// the markets created together.
fn deal(&mut self, markets: Vec<Market>) -> Vec<BookCommand> {
let mut known = HashSet::new();
let markets: Vec<Market> = markets
.into_iter()
.filter(|market| known.insert(market.condition_id.clone()))
.collect();
let tokens: usize = markets.iter().map(|market| market.tokens.len()).sum();
let count = (self.subscriptions * tokens)
.div_ceil(STARTUP_FILL)
.max(self.subscriptions);
self.connections = vec![ConnectionLoad::default(); count];
let mut placed = vec![Vec::new(); markets.len()];
for _ in 0..self.subscriptions {
let mut order: Vec<usize> = (0..markets.len()).collect();
self.random.shuffle(&mut order);
for (position, market) in order.into_iter().enumerate() {
let mut connection = position % count;
while placed[market].contains(&connection) {
connection = (connection + 1) % count;
}
placed[market].push(connection);
self.connections[connection].tokens += markets[market].tokens.len();
}
}
markets
.into_iter()
.zip(placed)
.map(|(market, connections)| self.track(market, connections))
.collect()
}
/// A market it doesn't know goes on different random connections with
/// room: its Add.
fn add(&mut self, market: Market) -> Option<BookCommand> {
if let Some(unseen) = &mut self.unseen {
unseen.remove(&market.condition_id);
}
if self.markets.contains_key(&market.condition_id) {
return None;
}
let mut connections = Vec::new();
for _ in 0..self.subscriptions {
let connection = self.pick(market.tokens.len(), 0, &connections);
self.connections[connection].tokens += market.tokens.len();
connections.push(connection);
}
Some(self.track(market, connections))
}
fn track(&mut self, market: Market, connections: Vec<usize>) -> BookCommand {
let tracked = Tracked {
market: market.clone(),
connections: connections.clone(),
load: 0,
moved_at: None,
};
self.markets.insert(market.condition_id.clone(), tracked);
BookCommand::Add(market, connections)
}
/// A random connection with room for `tokens` more and `load` more
/// bytes/s, none of `except`; a new connection if none has. Random, since
/// the least loaded one would take every market of a batch created
/// together: the cluster a load average can't see.
fn pick(&mut self, tokens: usize, load: u64, except: &[usize]) -> usize {
let with_room: Vec<usize> = (0..self.connections.len())
.filter(|number| {
let connection = &self.connections[*number];
!except.contains(number)
&& connection.tokens + tokens <= CONNECTION_TOKENS
&& connection.load + load <= CONNECTION_BYTES_PER_SECOND
})
.collect();
if with_room.is_empty() {
self.connections.push(ConnectionLoad::default());
return self.connections.len() - 1;
}
with_room[self.random.below(with_room.len())]
}
fn remove(&mut self, condition_id: &str) -> Option<BookCommand> {
let tracked = self.markets.remove(condition_id)?;
for &connection in &tracked.connections {
let connection = &mut self.connections[connection];
connection.tokens -= tracked.market.tokens.len();
connection.load -= tracked.load;
}
Some(BookCommand::Remove(tracked.market))
}
/// A book thread's loads per market, each weighing on every connection
/// that carries the market.
fn report(&mut self, loads: &Loads) {
for (condition_id, &load) in loads {
let Some(tracked) = self.markets.get_mut(condition_id) else {
continue;
};
for &connection in &tracked.connections {
let connection = &mut self.connections[connection];
connection.load = connection.load - tracked.load + load;
}
tracked.load = load;
}
}
/// Moves the busiest markets' subscriptions off every connection over its
/// byte budget, busiest connection first, each to a random connection with
/// room that doesn't carry the market, until it is under: at most
/// [`MOVES_PER_MINUTE`], and none moved in the last [`MOVED_STAYS`].
fn rebalance(&mut self, now: Instant) -> Vec<BookCommand> {
let mut moves = Vec::new();
let mut over: Vec<usize> = (0..self.connections.len())
.filter(|&number| self.connections[number].load > CONNECTION_BYTES_PER_SECOND)
.collect();
over.sort_by_key(|&number| Reverse(self.connections[number].load));
for from in over {
// ponytail: a scan of every market per connection over budget,
// fine at a few hundred thousand once a minute; per-connection
// market sets if not.
let mut busiest: Vec<(String, u64)> = self
.markets
.iter()
.filter(|(_, tracked)| {
tracked.connections.contains(&from)
&& tracked
.moved_at
.is_none_or(|at| now.duration_since(at) >= MOVED_STAYS)
})
.map(|(id, tracked)| (id.clone(), tracked.load))
.collect();
busiest.sort_by_key(|&(_, load)| Reverse(load));
for (id, load) in busiest {
let size = self.markets[&id].market.tokens.len();
// Alone on its connection, a market would only take its load
// along; one with no load would lighten nothing.
if self.connections[from].load <= CONNECTION_BYTES_PER_SECOND
|| moves.len() == MOVES_PER_MINUTE
|| self.connections[from].tokens == size
|| load == 0
{
break;
}
moves.push(self.move_off(&id, from, now));
}
}
moves
}
/// Moves the market's subscription on `from` to a random connection with
/// room that doesn't carry it yet: its Move.
fn move_off(&mut self, id: &str, from: usize, now: Instant) -> BookCommand {
let tracked = &self.markets[id];
let (size, load) = (tracked.market.tokens.len(), tracked.load);
let carrying = tracked.connections.clone();
let to = self.pick(size, load, &carrying);
self.connections[from].tokens -= size;
self.connections[from].load -= load;
self.connections[to].tokens += size;
self.connections[to].load += load;
let tracked = self.markets.get_mut(id).expect("tracked");
for connection in &mut tracked.connections {
if *connection == from {
*connection = to;
}
}
tracked.moved_at = Some(now);
BookCommand::Move {
market: tracked.market.clone(),
from,
to,
}
}
/// A connection closed with 1013: on its second close within
/// [`SPLIT_WITHIN`], every second of its markets, in load order, moves off
/// at once, beyond rebalancing's limits.
fn closed(&mut self, connection: usize, now: Instant) -> Vec<BookCommand> {
let Some(closed) = self.connections.get_mut(connection) else {
return Vec::new();
};
let recent =
|at: Option<Instant>| at.is_some_and(|at| now.duration_since(at) < SPLIT_WITHIN);
let again = recent(closed.closed_at);
closed.closed_at = Some(now);
if !again || recent(closed.split_at) {
return Vec::new();
}
closed.split_at = Some(now);
let mut carried: Vec<(String, u64)> = self
.markets
.iter()
.filter(|(_, tracked)| tracked.connections.contains(&connection))
.map(|(id, tracked)| (id.clone(), tracked.load))
.collect();
// Busiest first, ties by id so the split is the same each time.
carried.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
carried
.into_iter()
.skip(1)
.step_by(2)
.map(|(id, _)| self.move_off(&id, connection, now))
.collect()
}
fn scan_started(&mut self) {
self.unseen = Some(self.markets.keys().cloned().collect());
}
/// Removes the markets known at the scan's start and not seen in it:
/// those added since were never at risk.
fn scan_done(&mut self) -> Vec<BookCommand> {
let unseen = self.unseen.take().unwrap_or_default();
unseen.iter().filter_map(|id| self.remove(id)).collect()
}
/// Half a listing says nothing of the other half: nothing is removed.
fn scan_failed(&mut self) {
self.unseen = None;
}
}
/// Placement needs choices that differ, not good randomness: xorshift64.
struct Random(u64);
impl Random {
fn next(&mut self) -> u64 {
self.0 ^= self.0 << 13;
self.0 ^= self.0 >> 7;
self.0 ^= self.0 << 17;
self.0
}
/// A number under `bound`, which is not 0.
fn below(&mut self, bound: usize) -> usize {
(self.next() % bound as u64) as usize
}
fn shuffle<T>(&mut self, items: &mut [T]) {
for last in (1..items.len()).rev() {
items.swap(last, self.below(last + 1));
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn market(id: usize) -> Market {
Market {
condition_id: format!("0x{id}"),
tokens: vec![format!("{id}a"), format!("{id}b")],
min_order_size: "5".to_string(),
neg_risk: false,
}
}
/// A tracker with two subscriptions per market, as first one wins has.
fn tracker() -> Tracker {
Tracker::new(2, 7)
}
fn tokens(tracker: &Tracker) -> Vec<usize> {
tracker.connections.iter().map(|one| one.tokens).collect()
}
fn connections(tracker: &Tracker, id: usize) -> Vec<usize> {
tracker.markets[&format!("0x{id}")].connections.clone()
}
/// Reports `bytes_per_second` for each market, as its book thread would.
fn report(tracker: &mut Tracker, loads: &[(usize, u64)]) {
let loads = loads
.iter()
.map(|&(id, bytes_per_second)| (format!("0x{id}"), bytes_per_second))
.collect();
tracker.report(&loads);
}
/// The connection each of `commands` moves a market from and to.
fn moves(commands: &[BookCommand]) -> Vec<(&str, usize, usize)> {
commands
.iter()
.map(|command| match command {
BookCommand::Move { market, from, to } => {
(market.condition_id.as_str(), *from, *to)
}
other => panic!("not a move: {other:?}"),
})
.collect()
}
#[test]
fn the_startup_deal_is_even_and_gives_each_market_different_connections() {
let mut tracker = tracker();
// 10,000 markets, 20,000 tokens, twice: 9 connections of 4,500.
let adds = tracker.deal((0..10_000).map(market).collect());
assert_eq!(adds.len(), 10_000);
assert_eq!(tracker.connections.len(), 9);
let tokens = tokens(&tracker);
assert_eq!(tokens.iter().sum::<usize>(), 40_000);
assert!(tokens.iter().all(|&held| held <= CONNECTION_TOKENS));
// Even, but for the markets whose second connection had to be the next.
assert!(tokens.iter().max().unwrap() - tokens.iter().min().unwrap() <= 100);
for command in &adds {
let BookCommand::Add(market, on) = command else {
panic!("not an add: {command:?}");
};
assert_eq!(on.len(), 2);
assert_ne!(on[0], on[1], "{}", market.condition_id);
}
// Markets created together (neighbours in the list) are spread: the
// first 100 don't share one pair of connections.
let pairs: HashSet<Vec<usize>> = (0..100).map(|id| connections(&tracker, id)).collect();
assert!(pairs.len() > 30, "{}", pairs.len());
}
#[test]
fn a_market_listed_twice_is_dealt_once() {
let mut tracker = tracker();
let adds = tracker.deal(vec![market(0), market(1), market(0)]);
assert_eq!(adds.len(), 2);
assert_eq!(tokens(&tracker).iter().sum::<usize>(), 8);
}
#[test]
fn a_new_market_goes_on_different_connections_and_one_opens_when_too_few_have_room() {
let mut tracker = tracker();
// Nothing open: its two connections open.
tracker.add(market(0)).unwrap();
assert_eq!(connections(&tracker, 0).len(), 2);
assert_eq!(tokens(&tracker), [2, 2]);
// Both fill together; then a third and a fourth open.
for id in 1..CONNECTION_TOKENS / 2 {
tracker.add(market(id)).unwrap();
}
assert_eq!(tokens(&tracker), [CONNECTION_TOKENS, CONNECTION_TOKENS]);
tracker.add(market(9_999)).unwrap();
assert_eq!(
tokens(&tracker),
[CONNECTION_TOKENS, CONNECTION_TOKENS, 2, 2]
);
// Room on one connection only: the other subscription opens a new one.
tracker.remove("0x0").unwrap();
tracker.connections[1].tokens = CONNECTION_TOKENS;
tracker.connections[2].tokens = CONNECTION_TOKENS;
tracker.connections[3].tokens = CONNECTION_TOKENS;
tracker.add(market(10_000)).unwrap();
let mut on = connections(&tracker, 10_000);
on.sort();
assert_eq!(on, [0, 4]);
}
#[test]
fn a_connection_over_budget_takes_no_new_market() {
let mut tracker = tracker();
tracker.add(market(0)).unwrap();
report(&mut tracker, &[(0, CONNECTION_BYTES_PER_SECOND + 1)]);
tracker.add(market(1)).unwrap();
let mut on = connections(&tracker, 1);
on.sort();
assert_eq!(on, [2, 3]);
}
#[test]
fn a_known_market_is_skipped() {
let mut tracker = tracker();
assert!(tracker.add(market(1)).is_some());
assert!(tracker.add(market(1)).is_none());
assert_eq!(tokens(&tracker), [2, 2]);
}
#[test]
fn a_report_weighs_on_each_of_the_markets_connections() {
let mut tracker = tracker();
tracker.add(market(1));
tracker.add(market(2));
report(&mut tracker, &[(1, 300), (2, 100)]);
let loads: Vec<u64> = tracker.connections.iter().map(|one| one.load).collect();
assert_eq!(loads, [400, 400]);
// A later report replaces the load; an unknown market's is ignored.
report(&mut tracker, &[(1, 50), (9, 900)]);
assert_eq!(tracker.connections[0].load, 150);
// A market gone takes its load along.
tracker.remove("0x1");
assert_eq!(tracker.connections[1].load, 100);
assert_eq!(tokens(&tracker), [2, 2]);
}
#[test]
fn a_connection_over_budget_moves_its_busiest_markets_until_under() {
let mut tracker = tracker();
for id in 0..10 {
tracker.add(market(id));
}
tracker.connections.push(ConnectionLoad::default());
let budget = CONNECTION_BYTES_PER_SECOND;
// Both connections at 1.5 budgets, market 0 weighing half a budget.
let mut loads = vec![(0, budget / 2)];
loads.extend((1..10).map(|id| (id, budget / 9)));
report(&mut tracker, &loads);
let now = Instant::now();
let commands = tracker.rebalance(now);
// Connection 0 sheds its busiest market, which is enough. Market 0
// then stays put, its other subscription included, so connection 1
// sheds smaller ones: a market's two subscriptions never move in the
// same minute.
let moved = moves(&commands);
assert_eq!(moved[0], ("0x0", 0, 2));
assert_eq!(connections(&tracker, 0), [2, 1]);
assert!(
moved[1..]
.iter()
.all(|&(id, from, _)| id != "0x0" && from == 1)
);
assert_eq!(moved.len(), 1 + 5);
assert!(tracker.connections.iter().all(|one| one.load <= budget));
assert_eq!(tokens(&tracker).iter().sum::<usize>(), 40);
// Under budget now: nothing more moves.
assert!(tracker.rebalance(now).is_empty());
}
#[test]
fn a_move_never_targets_a_connection_that_carries_the_market() {
for seed in 1..50 {
let mut tracker = Tracker::new(2, seed);
for id in 0..6 {
tracker.add(market(id));
}
// Room elsewhere, on several connections.
tracker.connections.resize(6, ConnectionLoad::default());
report(
&mut tracker,
&[(0, CONNECTION_BYTES_PER_SECOND), (1, 1_000)],
);
let before = connections(&tracker, 0);
for (_, from, to) in moves(&tracker.rebalance(Instant::now())) {
assert!(before.contains(&from));
assert!(!before.contains(&to), "seed {seed}");
}
let after = connections(&tracker, 0);
assert_ne!(after[0], after[1], "seed {seed}");
}
}
#[test]
fn the_busiest_connection_over_budget_is_rebalanced_first() {
let mut tracker = Tracker::new(1, 7);
tracker.deal(vec![market(0), market(1)]);
tracker.connections.push(ConnectionLoad::default());
tracker.connections.push(ConnectionLoad::default());
// One connection so far: put markets 2 and 3 on another.
for id in [2, 3] {
tracker.connections[0].tokens = CONNECTION_TOKENS;
tracker.connections[2].tokens = CONNECTION_TOKENS;
tracker.add(market(id));
}
tracker.connections[0].tokens = 4;
tracker.connections[2].tokens = 0;
assert_eq!(tokens(&tracker), [4, 4, 0]);
let budget = CONNECTION_BYTES_PER_SECOND;
report(&mut tracker, &[(0, budget), (1, budget / 10)]);
report(&mut tracker, &[(2, budget / 2), (3, budget)]);
let commands = tracker.rebalance(Instant::now());
assert_eq!(moves(&commands)[0], ("0x3", 1, 2));
}
/// The markets that may move have no load: moving them would empty the
/// connection and lighten nothing.
#[test]
fn markets_with_no_load_are_not_moved() {
let mut tracker = Tracker::new(1, 7);
tracker.deal((0..4).map(market).collect());
report(&mut tracker, &[(0, 2 * CONNECTION_BYTES_PER_SECOND)]);
let now = Instant::now();
tracker.markets.get_mut("0x0").unwrap().moved_at = Some(now);
assert!(tracker.rebalance(now).is_empty());
}
#[test]
fn moves_are_limited_and_a_moved_market_stays() {
let mut tracker = Tracker::new(1, 7);
let count = 2 * MOVES_PER_MINUTE;
tracker.deal((0..count).map(market).collect());
assert_eq!(tracker.connections.len(), 1);
// Every market alone near a budget: each needs a connection of its own.
let loads: Vec<_> = (0..count)
.map(|id| (id, CONNECTION_BYTES_PER_SECOND * 9 / 10))
.collect();
report(&mut tracker, &loads);
let now = Instant::now();
let first = tracker.rebalance(now);
assert_eq!(first.len(), MOVES_PER_MINUTE);
// The rest move a minute later, but the last, alone and under budget.
let second = tracker.rebalance(now + Duration::from_secs(60));
assert_eq!(second.len(), MOVES_PER_MINUTE - 1);
assert!(moves(&second).iter().all(|&(_, from, _)| from == 0));
// Ten minutes on, connection 0 holds one market: it is left alone.
assert!(tracker.rebalance(now + MOVED_STAYS).is_empty());
assert_eq!(tracker.connections[0].tokens, 2);
}
/// Over budget on its own, a market would only carry its load from
/// connection to connection every ten minutes.
#[test]
fn a_market_alone_on_its_connection_is_not_moved() {
let mut tracker = Tracker::new(1, 7);
tracker.add(market(0));
report(&mut tracker, &[(0, 2 * CONNECTION_BYTES_PER_SECOND)]);
assert!(tracker.rebalance(Instant::now()).is_empty());
assert_eq!(tracker.connections.len(), 1);
}
#[test]
fn a_market_moved_less_than_ten_minutes_ago_stays() {
let mut tracker = Tracker::new(1, 7);
let budget = CONNECTION_BYTES_PER_SECOND;
tracker.deal(vec![market(0), market(1)]);
report(&mut tracker, &[(0, budget * 7 / 10), (1, budget / 2)]);
let now = Instant::now();
// Market 0 moves to a new connection, which then turns busy.
assert_eq!(moves(&tracker.rebalance(now)), [("0x0", 0, 1)]);
tracker.connections[0].tokens = CONNECTION_TOKENS;
tracker.add(market(2));
assert_eq!(connections(&tracker, 2), [1]);
report(&mut tracker, &[(0, budget * 7 / 10), (2, budget * 6 / 10)]);
let commands = tracker.rebalance(now + Duration::from_secs(60));
assert_eq!(moves(&commands), [("0x2", 1, 2)]);
}
#[test]
fn a_connection_closed_twice_within_ten_minutes_is_split() {
let mut tracker = Tracker::new(1, 7);
tracker.deal((0..10).map(market).collect());
assert_eq!(tracker.connections.len(), 1);
let loads: Vec<_> = (0..10).map(|id| (id, 1_000 * (id as u64 + 1))).collect();
report(&mut tracker, &loads);
let now = Instant::now();
// One close: the server's moment, perhaps.
assert!(tracker.closed(0, now).is_empty());
// A second within ten minutes: every second market, busiest first.
let moved = tracker.closed(0, now + Duration::from_secs(5 * 60));
let ids: Vec<&str> = moves(&moved).into_iter().map(|(id, _, _)| id).collect();
assert_eq!(ids, ["0x8", "0x6", "0x4", "0x2", "0x0"]);
assert!(
moves(&moved)
.iter()
.all(|&(_, from, to)| from == 0 && to != 0)
);
assert_eq!(tracker.connections[0].tokens, 10);
// The split connection is left alone for ten minutes.
let later = now + Duration::from_secs(6 * 60);
assert!(tracker.closed(0, later).is_empty());
assert!(
tracker
.closed(0, later + Duration::from_secs(60))
.is_empty()
);
// Ten minutes after both, two closes split it again.
let much_later = now + Duration::from_secs(30 * 60);
assert!(tracker.closed(0, much_later).is_empty());
assert_eq!(
tracker
.closed(0, much_later + Duration::from_secs(60))
.len(),
2
);
}
#[test]
fn closes_ten_minutes_apart_split_nothing() {
let mut tracker = Tracker::new(2, 7);
tracker.deal((0..10).map(market).collect());
let now = Instant::now();
assert!(tracker.closed(0, now).is_empty());
assert!(tracker.closed(0, now + SPLIT_WITHIN).is_empty());
// An unknown connection is ignored.
assert!(tracker.closed(99, now).is_empty());
}
#[test]
fn a_completed_rescan_removes_only_the_known_markets_it_did_not_see() {
let mut tracker = tracker();
for id in 0..3 {
tracker.add(market(id));
}
tracker.scan_started();
tracker.add(market(0));
// Added by a minute check during the scan, on a page already passed.
tracker.add(market(9));
// Closed during the scan.
tracker.remove("0x1");
let removed = tracker.scan_done();
assert!(matches!(
removed.as_slice(),
[BookCommand::Remove(market)] if market.condition_id == "0x2"
));
let mut tracked: Vec<_> = tracker.markets.keys().cloned().collect();
tracked.sort();
assert_eq!(tracked, ["0x0", "0x9"]);
assert_eq!(tokens(&tracker), [4, 4]);
}
#[test]
fn a_failed_rescan_removes_nothing() {
let mut tracker = tracker();
tracker.add(market(0));
tracker.scan_started();
tracker.scan_failed();
assert!(tracker.scan_done().is_empty());
assert_eq!(tracker.markets.len(), 1);
}
}