//! The orchestrator: the one owner of what is tracked. It keeps the tracked //! markets equal to Gamma's live ones, decides which connections carry each //! market and owns every timer; the two fetchers only fetch (ADR 0002, ADR //! 0003, roadmap/04). use std::cmp::Reverse; use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use chrono::{DateTime, TimeDelta, Utc}; use crossbeam::channel::{Sender, select, unbounded}; use engineering_challenge::book::Output; use super::feed::{self, BookCommand, Feed, Loads}; use super::gamma::{self, Finding, Request}; use super::wire::Received; use super::{BOOK_THREADS, Market, book_thread, socket}; /// How many connections carry each market, first one wins (ADR 0003). 1 for /// a run to compare with. const SUBSCRIPTIONS: usize = 2; /// The most tokens a connection holds: a reconnect replays every one's book /// at once. const CONNECTION_TOKENS: usize = 5_000; /// What the startup deal fills a connection to, leaving room for new markets. const STARTUP_FILL: usize = 4_500; /// A connection's byte budget, under the lowest rate at which one was seen to /// get a 1013 close (0.46 MB/s; docs/notes.md). Closes came at up to /// 5.1 MB/s too: no rate is safe, only less likely to be closed. const CONNECTION_BYTES_PER_SECOND: u64 = 300_000; /// The startup deal's pace, about 5,000 markets a second: every connection /// subscribing at once would send more opening books than the link carries. const DEAL_AT_ONCE: usize = 500; const DEAL_EVERY: Duration = Duration::from_millis(100); /// Moves are limited so balancing doesn't churn. Rebalancing runs with each /// minute check, every [`RECENT_EVERY`]. const MOVES_PER_MINUTE: usize = 50; const MOVED_STAYS: Duration = Duration::from_secs(10 * 60); /// A connection closed with 1013 twice within this is split: one close can be /// the server's bad moment (several connections within a second), a second /// one says it carries too much. It is then left alone for as long. const SPLIT_WITHIN: Duration = Duration::from_secs(10 * 60); const RECENT_EVERY: Duration = Duration::from_secs(60); const RESCAN_EVERY: Duration = Duration::from_secs(60 * 60); /// Gamma's indexing delay: each recent check reaches this far before the /// last successful one. New markets were listed up to 5 minutes after their /// `createdAt` (docs/notes.md); known markets are skipped, so the margin costs /// only the pages it reads again. const RECENT_MARGIN: TimeDelta = TimeDelta::minutes(15); /// So SIGINT is seen while no timer is due. const WAKE_EVERY: Duration = Duration::from_secs(1); /// Runs until `stop` is set, then waits for every connection and book thread. /// Returns how many threads failed: those that panicked, and one if a fetcher /// died. pub fn run(outputs: Sender, stop: &Arc) -> usize { let (findings_tx, findings) = unbounded(); let (rescan, rescan_requests) = unbounded(); let (recent, recent_requests) = unbounded(); // Two, so a full scan never holds up the minute check. Not joined: they // hold no output, and each ends at its next send. let fetchers: Vec> = [rescan_requests, recent_requests] .into_iter() .map(|requests| { let findings = findings_tx.clone(); thread::spawn(move || gamma::run(requests, findings)) }) .collect(); drop(findings_tx); let (reports_tx, reports) = unbounded::(); let (closes_tx, closes) = unbounded::(); let mut threads = Threads::start(&outputs, &reports_tx, &closes_tx, stop); drop(closes_tx); // The book threads hold the only senders the writer waits for. drop(outputs); drop(reports_tx); let seed = SystemTime::now() .duration_since(UNIX_EPOCH) .map_or(1, |since| since.as_nanos() as u64); let mut tracker = Tracker::new(SUBSCRIPTIONS, seed); // Until the first scan is done: the live markets it found, waiting to be // dealt at random. Dealing page by page would keep markets created // together on the same connections. let mut first_scan: Option> = Some(Vec::new()); // The startup deal's commands still to send, and whatever came behind // them: a market's commands keep their order. let mut queue: VecDeque = VecDeque::new(); let mut next_deal = Instant::now(); let (mut added, mut removed, mut moved, mut splits) = (0, 0, 0, 0); // The minute checks cover the first scan from its start. let mut recent_since = Utc::now() - RECENT_MARGIN; let mut recent_asked_at: Option> = None; let mut scanning = false; let mut scanned = false; let mut fetcher_died = false; let mut next_rescan = Instant::now(); let mut next_recent = Instant::now() + RECENT_EVERY; while !stop.load(Ordering::Relaxed) { // Only a panic ends a fetcher while this runs, even one in the middle // of a request, whose answer would never come: nothing would keep the // set current. if fetchers.iter().any(JoinHandle::is_finished) { log!("polymarket: orchestrator: a fetcher died, stopping"); stop.store(true, Ordering::Relaxed); fetcher_died = true; break; } let now = Instant::now(); let mut commands = Vec::new(); if now >= next_rescan { if !scanning { _ = rescan.send(Request::Rescan); scanning = true; } next_rescan = now + RESCAN_EVERY; } if now >= next_recent { if recent_asked_at.is_none() { _ = recent.send(Request::Recent { since: recent_since, }); recent_asked_at = Some(Utc::now()); } next_recent = now + RECENT_EVERY; // On the loads the book threads reported at the wall-clock minute. let moves = tracker.rebalance(now); moved += moves.len(); commands.extend(moves); let busiest = tracker.connections.iter().map(|one| one.load).max(); log!( "polymarket: orchestrator: {} markets on {} connections; {added} added, {removed} \ removed, {moved} moved, {splits} splits this minute; busiest connection at {} bytes/s", tracker.markets.len(), tracker.connections.len(), busiest.unwrap_or(0) ); (added, removed, moved, splits) = (0, 0, 0, 0); } if !queue.is_empty() && now >= next_deal { for command in queue.drain(..DEAL_AT_ONCE.min(queue.len())) { threads.send(command); } next_deal = now + DEAL_EVERY; } let mut timeout = next_rescan .min(next_recent) .saturating_duration_since(Instant::now()) .min(WAKE_EVERY); if !queue.is_empty() { timeout = timeout.min(next_deal.saturating_duration_since(Instant::now())); } select! { recv(findings) -> finding => match finding { Ok(Finding::ScanStarted) => { tracker.scan_started(); // A first scan that failed starts over. if let Some(waiting) = &mut first_scan { waiting.clear(); } } Ok(Finding::ScanPage(markets) | Finding::Created(markets)) => match &mut first_scan { Some(waiting) => waiting.extend(markets), None => { let adds: Vec<_> = markets .into_iter() .filter_map(|market| tracker.add(market)) .collect(); added += adds.len(); commands.extend(adds); } }, Ok(Finding::ScanDone) => { scanning = false; scanned = true; match first_scan.take() { Some(markets) => { let adds = tracker.deal(markets); added += adds.len(); commands.extend(adds); } None => { let gone = tracker.scan_done(); removed += gone.len(); commands.extend(gone); } } } Ok(Finding::ScanFailed) => { scanning = false; tracker.scan_failed(); // Nothing can be tracked before a first scan: retry at once. if !scanned { next_rescan = Instant::now(); } } Ok(Finding::Closed(ids)) => { // Closed while the first scan runs: never dealt. if let Some(waiting) = &mut first_scan { waiting.retain(|market| !ids.contains(&market.condition_id)); } let gone: Vec<_> = ids.iter().filter_map(|id| tracker.remove(id)).collect(); removed += gone.len(); commands.extend(gone); } Ok(Finding::RecentDone) => { if let Some(asked_at) = recent_asked_at.take() { recent_since = asked_at - RECENT_MARGIN; } } Ok(Finding::RecentFailed) => recent_asked_at = None, // Both fetchers are gone: the check above stops the run. Err(_) => {} }, recv(reports) -> loads => { if let Ok(loads) = loads { tracker.report(&loads); } }, recv(closes) -> closed => { if let Ok(connection) = closed { let moves = tracker.closed(connection, Instant::now()); if !moves.is_empty() { log!( "polymarket: orchestrator: connection {connection} closed with 1013 \ twice, split: {} markets moved off", moves.len() ); splits += 1; moved += moves.len(); } commands.extend(moves); } }, default(timeout) => {} } // A connection is open before any command names it. threads.open(tracker.connections.len()); // The startup deal is paced; what comes while it lasts waits behind it. if commands.len() > DEAL_AT_ONCE || !queue.is_empty() { queue.extend(commands); } else { for command in commands { threads.send(command); } } } usize::from(fetcher_died) + threads.join() } /// The threads the orchestrator feeds: the book threads, there from the /// start, and the connections it opens. struct Threads { book_commands: Vec>, /// Every book thread's events channel, by number: each connection gets a /// clone of them all. events: Vec>, closes: Sender, book_threads: Vec>, connections: Vec>, stop: Arc, } impl Threads { fn start( outputs: &Sender, reports: &Sender, closes: &Sender, stop: &Arc, ) -> Self { let mut threads = Self { book_commands: Vec::new(), events: Vec::new(), closes: closes.clone(), book_threads: Vec::new(), connections: Vec::new(), stop: Arc::clone(stop), }; for number in 0..BOOK_THREADS { let (commands, commands_rx) = unbounded(); let (events, events_rx) = unbounded(); let (outputs, reports) = (outputs.clone(), reports.clone()); threads.book_commands.push(commands); threads.events.push(events); threads.book_threads.push(thread::spawn(move || { feed::run(Feed::new(number), events_rx, commands_rx, outputs, reports) })); } threads } /// Opens connections until there are `count`, telling every book thread /// how to reach each. fn open(&mut self, count: usize) { while self.connections.len() < count { let number = self.connections.len(); let (commands, commands_rx) = unbounded(); let (events, stop) = (self.events.clone(), Arc::clone(&self.stop)); let closes = self.closes.clone(); self.connections.push(thread::spawn(move || { socket::run( socket::MARKET_CHANNEL_URL, number, &events, &commands_rx, &closes, &stop, ) })); for book_thread in &self.book_commands { _ = book_thread.send(BookCommand::Connected(number, commands.clone())); } } } /// To the market's book thread. One that died keeps its markets unserved, /// and its connections stop the run. fn send(&self, command: BookCommand) { let market = match &command { BookCommand::Add(market, _) | BookCommand::Move { market, .. } | BookCommand::Remove(market) => market, BookCommand::Connected(..) => unreachable!("sent by open() to every book thread"), }; _ = self.book_commands[book_thread(&market.condition_id)].send(command); } /// Waits for the connections, which stop on the flag, then for the book /// threads, which stop once no connection can send them an event: how /// many panicked. fn join(self) -> usize { let panicked = |threads: Vec>| { threads .into_iter() .map(JoinHandle::join) .filter(Result::is_err) .count() }; let connections = panicked(self.connections); drop(self.events); drop(self.book_commands); connections + panicked(self.book_threads) } } /// A tracked market, where it is and what it weighs. struct Tracked { market: Market, /// The connections that carry it, all different. connections: Vec, /// Bytes per second on one connection, over its book thread's last /// report; 0 until one comes. load: u64, moved_at: Option, } /// A connection's tokens and load, against its two limits, and its 1013 /// closes. #[derive(Debug, Default, Clone, PartialEq, Eq)] struct ConnectionLoad { tokens: usize, /// The sum of its markets' loads, bytes per second. load: u64, /// When it was last closed with 1013. closed_at: Option, split_at: Option, } /// Which connections carry each market and what it weighs, and which markets /// a rescan in progress hasn't seen yet. struct Tracker { subscriptions: usize, /// By `conditionId`. markets: HashMap, /// By connection number. connections: Vec, /// While a rescan runs: the markets known at its start, not seen since. unseen: Option>, random: Random, } impl Tracker { fn new(subscriptions: usize, seed: u64) -> Self { Self { subscriptions, markets: HashMap::new(), connections: Vec::new(), unseen: None, random: Random(seed | 1), } } /// The startup deal: every market on `subscriptions` different /// connections, each subscription dealt in an order of its own, so a /// market's connections hold different neighbours and no connection gets /// the markets created together. fn deal(&mut self, markets: Vec) -> Vec { let mut known = HashSet::new(); let markets: Vec = markets .into_iter() .filter(|market| known.insert(market.condition_id.clone())) .collect(); let tokens: usize = markets.iter().map(|market| market.tokens.len()).sum(); let count = (self.subscriptions * tokens) .div_ceil(STARTUP_FILL) .max(self.subscriptions); self.connections = vec![ConnectionLoad::default(); count]; let mut placed = vec![Vec::new(); markets.len()]; for _ in 0..self.subscriptions { let mut order: Vec = (0..markets.len()).collect(); self.random.shuffle(&mut order); for (position, market) in order.into_iter().enumerate() { let mut connection = position % count; while placed[market].contains(&connection) { connection = (connection + 1) % count; } placed[market].push(connection); self.connections[connection].tokens += markets[market].tokens.len(); } } markets .into_iter() .zip(placed) .map(|(market, connections)| self.track(market, connections)) .collect() } /// A market it doesn't know goes on different random connections with /// room: its Add. fn add(&mut self, market: Market) -> Option { if let Some(unseen) = &mut self.unseen { unseen.remove(&market.condition_id); } if self.markets.contains_key(&market.condition_id) { return None; } let mut connections = Vec::new(); for _ in 0..self.subscriptions { let connection = self.pick(market.tokens.len(), 0, &connections); self.connections[connection].tokens += market.tokens.len(); connections.push(connection); } Some(self.track(market, connections)) } fn track(&mut self, market: Market, connections: Vec) -> BookCommand { let tracked = Tracked { market: market.clone(), connections: connections.clone(), load: 0, moved_at: None, }; self.markets.insert(market.condition_id.clone(), tracked); BookCommand::Add(market, connections) } /// A random connection with room for `tokens` more and `load` more /// bytes/s, none of `except`; a new connection if none has. Random, since /// the least loaded one would take every market of a batch created /// together: the cluster a load average can't see. fn pick(&mut self, tokens: usize, load: u64, except: &[usize]) -> usize { let with_room: Vec = (0..self.connections.len()) .filter(|number| { let connection = &self.connections[*number]; !except.contains(number) && connection.tokens + tokens <= CONNECTION_TOKENS && connection.load + load <= CONNECTION_BYTES_PER_SECOND }) .collect(); if with_room.is_empty() { self.connections.push(ConnectionLoad::default()); return self.connections.len() - 1; } with_room[self.random.below(with_room.len())] } fn remove(&mut self, condition_id: &str) -> Option { let tracked = self.markets.remove(condition_id)?; for &connection in &tracked.connections { let connection = &mut self.connections[connection]; connection.tokens -= tracked.market.tokens.len(); connection.load -= tracked.load; } Some(BookCommand::Remove(tracked.market)) } /// A book thread's loads per market, each weighing on every connection /// that carries the market. fn report(&mut self, loads: &Loads) { for (condition_id, &load) in loads { let Some(tracked) = self.markets.get_mut(condition_id) else { continue; }; for &connection in &tracked.connections { let connection = &mut self.connections[connection]; connection.load = connection.load - tracked.load + load; } tracked.load = load; } } /// Moves the busiest markets' subscriptions off every connection over its /// byte budget, busiest connection first, each to a random connection with /// room that doesn't carry the market, until it is under: at most /// [`MOVES_PER_MINUTE`], and none moved in the last [`MOVED_STAYS`]. fn rebalance(&mut self, now: Instant) -> Vec { let mut moves = Vec::new(); let mut over: Vec = (0..self.connections.len()) .filter(|&number| self.connections[number].load > CONNECTION_BYTES_PER_SECOND) .collect(); over.sort_by_key(|&number| Reverse(self.connections[number].load)); for from in over { // ponytail: a scan of every market per connection over budget, // fine at a few hundred thousand once a minute; per-connection // market sets if not. let mut busiest: Vec<(String, u64)> = self .markets .iter() .filter(|(_, tracked)| { tracked.connections.contains(&from) && tracked .moved_at .is_none_or(|at| now.duration_since(at) >= MOVED_STAYS) }) .map(|(id, tracked)| (id.clone(), tracked.load)) .collect(); busiest.sort_by_key(|&(_, load)| Reverse(load)); for (id, load) in busiest { let size = self.markets[&id].market.tokens.len(); // Alone on its connection, a market would only take its load // along; one with no load would lighten nothing. if self.connections[from].load <= CONNECTION_BYTES_PER_SECOND || moves.len() == MOVES_PER_MINUTE || self.connections[from].tokens == size || load == 0 { break; } moves.push(self.move_off(&id, from, now)); } } moves } /// Moves the market's subscription on `from` to a random connection with /// room that doesn't carry it yet: its Move. fn move_off(&mut self, id: &str, from: usize, now: Instant) -> BookCommand { let tracked = &self.markets[id]; let (size, load) = (tracked.market.tokens.len(), tracked.load); let carrying = tracked.connections.clone(); let to = self.pick(size, load, &carrying); self.connections[from].tokens -= size; self.connections[from].load -= load; self.connections[to].tokens += size; self.connections[to].load += load; let tracked = self.markets.get_mut(id).expect("tracked"); for connection in &mut tracked.connections { if *connection == from { *connection = to; } } tracked.moved_at = Some(now); BookCommand::Move { market: tracked.market.clone(), from, to, } } /// A connection closed with 1013: on its second close within /// [`SPLIT_WITHIN`], every second of its markets, in load order, moves off /// at once, beyond rebalancing's limits. fn closed(&mut self, connection: usize, now: Instant) -> Vec { let Some(closed) = self.connections.get_mut(connection) else { return Vec::new(); }; let recent = |at: Option| at.is_some_and(|at| now.duration_since(at) < SPLIT_WITHIN); let again = recent(closed.closed_at); closed.closed_at = Some(now); if !again || recent(closed.split_at) { return Vec::new(); } closed.split_at = Some(now); let mut carried: Vec<(String, u64)> = self .markets .iter() .filter(|(_, tracked)| tracked.connections.contains(&connection)) .map(|(id, tracked)| (id.clone(), tracked.load)) .collect(); // Busiest first, ties by id so the split is the same each time. carried.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); carried .into_iter() .skip(1) .step_by(2) .map(|(id, _)| self.move_off(&id, connection, now)) .collect() } fn scan_started(&mut self) { self.unseen = Some(self.markets.keys().cloned().collect()); } /// Removes the markets known at the scan's start and not seen in it: /// those added since were never at risk. fn scan_done(&mut self) -> Vec { let unseen = self.unseen.take().unwrap_or_default(); unseen.iter().filter_map(|id| self.remove(id)).collect() } /// Half a listing says nothing of the other half: nothing is removed. fn scan_failed(&mut self) { self.unseen = None; } } /// Placement needs choices that differ, not good randomness: xorshift64. struct Random(u64); impl Random { fn next(&mut self) -> u64 { self.0 ^= self.0 << 13; self.0 ^= self.0 >> 7; self.0 ^= self.0 << 17; self.0 } /// A number under `bound`, which is not 0. fn below(&mut self, bound: usize) -> usize { (self.next() % bound as u64) as usize } fn shuffle(&mut self, items: &mut [T]) { for last in (1..items.len()).rev() { items.swap(last, self.below(last + 1)); } } } #[cfg(test)] mod tests { use super::*; fn market(id: usize) -> Market { Market { condition_id: format!("0x{id}"), tokens: vec![format!("{id}a"), format!("{id}b")], min_order_size: "5".to_string(), neg_risk: false, } } /// A tracker with two subscriptions per market, as first one wins has. fn tracker() -> Tracker { Tracker::new(2, 7) } fn tokens(tracker: &Tracker) -> Vec { tracker.connections.iter().map(|one| one.tokens).collect() } fn connections(tracker: &Tracker, id: usize) -> Vec { tracker.markets[&format!("0x{id}")].connections.clone() } /// Reports `bytes_per_second` for each market, as its book thread would. fn report(tracker: &mut Tracker, loads: &[(usize, u64)]) { let loads = loads .iter() .map(|&(id, bytes_per_second)| (format!("0x{id}"), bytes_per_second)) .collect(); tracker.report(&loads); } /// The connection each of `commands` moves a market from and to. fn moves(commands: &[BookCommand]) -> Vec<(&str, usize, usize)> { commands .iter() .map(|command| match command { BookCommand::Move { market, from, to } => { (market.condition_id.as_str(), *from, *to) } other => panic!("not a move: {other:?}"), }) .collect() } #[test] fn the_startup_deal_is_even_and_gives_each_market_different_connections() { let mut tracker = tracker(); // 10,000 markets, 20,000 tokens, twice: 9 connections of 4,500. let adds = tracker.deal((0..10_000).map(market).collect()); assert_eq!(adds.len(), 10_000); assert_eq!(tracker.connections.len(), 9); let tokens = tokens(&tracker); assert_eq!(tokens.iter().sum::(), 40_000); assert!(tokens.iter().all(|&held| held <= CONNECTION_TOKENS)); // Even, but for the markets whose second connection had to be the next. assert!(tokens.iter().max().unwrap() - tokens.iter().min().unwrap() <= 100); for command in &adds { let BookCommand::Add(market, on) = command else { panic!("not an add: {command:?}"); }; assert_eq!(on.len(), 2); assert_ne!(on[0], on[1], "{}", market.condition_id); } // Markets created together (neighbours in the list) are spread: the // first 100 don't share one pair of connections. let pairs: HashSet> = (0..100).map(|id| connections(&tracker, id)).collect(); assert!(pairs.len() > 30, "{}", pairs.len()); } #[test] fn a_market_listed_twice_is_dealt_once() { let mut tracker = tracker(); let adds = tracker.deal(vec![market(0), market(1), market(0)]); assert_eq!(adds.len(), 2); assert_eq!(tokens(&tracker).iter().sum::(), 8); } #[test] fn a_new_market_goes_on_different_connections_and_one_opens_when_too_few_have_room() { let mut tracker = tracker(); // Nothing open: its two connections open. tracker.add(market(0)).unwrap(); assert_eq!(connections(&tracker, 0).len(), 2); assert_eq!(tokens(&tracker), [2, 2]); // Both fill together; then a third and a fourth open. for id in 1..CONNECTION_TOKENS / 2 { tracker.add(market(id)).unwrap(); } assert_eq!(tokens(&tracker), [CONNECTION_TOKENS, CONNECTION_TOKENS]); tracker.add(market(9_999)).unwrap(); assert_eq!( tokens(&tracker), [CONNECTION_TOKENS, CONNECTION_TOKENS, 2, 2] ); // Room on one connection only: the other subscription opens a new one. tracker.remove("0x0").unwrap(); tracker.connections[1].tokens = CONNECTION_TOKENS; tracker.connections[2].tokens = CONNECTION_TOKENS; tracker.connections[3].tokens = CONNECTION_TOKENS; tracker.add(market(10_000)).unwrap(); let mut on = connections(&tracker, 10_000); on.sort(); assert_eq!(on, [0, 4]); } #[test] fn a_connection_over_budget_takes_no_new_market() { let mut tracker = tracker(); tracker.add(market(0)).unwrap(); report(&mut tracker, &[(0, CONNECTION_BYTES_PER_SECOND + 1)]); tracker.add(market(1)).unwrap(); let mut on = connections(&tracker, 1); on.sort(); assert_eq!(on, [2, 3]); } #[test] fn a_known_market_is_skipped() { let mut tracker = tracker(); assert!(tracker.add(market(1)).is_some()); assert!(tracker.add(market(1)).is_none()); assert_eq!(tokens(&tracker), [2, 2]); } #[test] fn a_report_weighs_on_each_of_the_markets_connections() { let mut tracker = tracker(); tracker.add(market(1)); tracker.add(market(2)); report(&mut tracker, &[(1, 300), (2, 100)]); let loads: Vec = tracker.connections.iter().map(|one| one.load).collect(); assert_eq!(loads, [400, 400]); // A later report replaces the load; an unknown market's is ignored. report(&mut tracker, &[(1, 50), (9, 900)]); assert_eq!(tracker.connections[0].load, 150); // A market gone takes its load along. tracker.remove("0x1"); assert_eq!(tracker.connections[1].load, 100); assert_eq!(tokens(&tracker), [2, 2]); } #[test] fn a_connection_over_budget_moves_its_busiest_markets_until_under() { let mut tracker = tracker(); for id in 0..10 { tracker.add(market(id)); } tracker.connections.push(ConnectionLoad::default()); let budget = CONNECTION_BYTES_PER_SECOND; // Both connections at 1.5 budgets, market 0 weighing half a budget. let mut loads = vec![(0, budget / 2)]; loads.extend((1..10).map(|id| (id, budget / 9))); report(&mut tracker, &loads); let now = Instant::now(); let commands = tracker.rebalance(now); // Connection 0 sheds its busiest market, which is enough. Market 0 // then stays put, its other subscription included, so connection 1 // sheds smaller ones: a market's two subscriptions never move in the // same minute. let moved = moves(&commands); assert_eq!(moved[0], ("0x0", 0, 2)); assert_eq!(connections(&tracker, 0), [2, 1]); assert!( moved[1..] .iter() .all(|&(id, from, _)| id != "0x0" && from == 1) ); assert_eq!(moved.len(), 1 + 5); assert!(tracker.connections.iter().all(|one| one.load <= budget)); assert_eq!(tokens(&tracker).iter().sum::(), 40); // Under budget now: nothing more moves. assert!(tracker.rebalance(now).is_empty()); } #[test] fn a_move_never_targets_a_connection_that_carries_the_market() { for seed in 1..50 { let mut tracker = Tracker::new(2, seed); for id in 0..6 { tracker.add(market(id)); } // Room elsewhere, on several connections. tracker.connections.resize(6, ConnectionLoad::default()); report( &mut tracker, &[(0, CONNECTION_BYTES_PER_SECOND), (1, 1_000)], ); let before = connections(&tracker, 0); for (_, from, to) in moves(&tracker.rebalance(Instant::now())) { assert!(before.contains(&from)); assert!(!before.contains(&to), "seed {seed}"); } let after = connections(&tracker, 0); assert_ne!(after[0], after[1], "seed {seed}"); } } #[test] fn the_busiest_connection_over_budget_is_rebalanced_first() { let mut tracker = Tracker::new(1, 7); tracker.deal(vec![market(0), market(1)]); tracker.connections.push(ConnectionLoad::default()); tracker.connections.push(ConnectionLoad::default()); // One connection so far: put markets 2 and 3 on another. for id in [2, 3] { tracker.connections[0].tokens = CONNECTION_TOKENS; tracker.connections[2].tokens = CONNECTION_TOKENS; tracker.add(market(id)); } tracker.connections[0].tokens = 4; tracker.connections[2].tokens = 0; assert_eq!(tokens(&tracker), [4, 4, 0]); let budget = CONNECTION_BYTES_PER_SECOND; report(&mut tracker, &[(0, budget), (1, budget / 10)]); report(&mut tracker, &[(2, budget / 2), (3, budget)]); let commands = tracker.rebalance(Instant::now()); assert_eq!(moves(&commands)[0], ("0x3", 1, 2)); } /// The markets that may move have no load: moving them would empty the /// connection and lighten nothing. #[test] fn markets_with_no_load_are_not_moved() { let mut tracker = Tracker::new(1, 7); tracker.deal((0..4).map(market).collect()); report(&mut tracker, &[(0, 2 * CONNECTION_BYTES_PER_SECOND)]); let now = Instant::now(); tracker.markets.get_mut("0x0").unwrap().moved_at = Some(now); assert!(tracker.rebalance(now).is_empty()); } #[test] fn moves_are_limited_and_a_moved_market_stays() { let mut tracker = Tracker::new(1, 7); let count = 2 * MOVES_PER_MINUTE; tracker.deal((0..count).map(market).collect()); assert_eq!(tracker.connections.len(), 1); // Every market alone near a budget: each needs a connection of its own. let loads: Vec<_> = (0..count) .map(|id| (id, CONNECTION_BYTES_PER_SECOND * 9 / 10)) .collect(); report(&mut tracker, &loads); let now = Instant::now(); let first = tracker.rebalance(now); assert_eq!(first.len(), MOVES_PER_MINUTE); // The rest move a minute later, but the last, alone and under budget. let second = tracker.rebalance(now + Duration::from_secs(60)); assert_eq!(second.len(), MOVES_PER_MINUTE - 1); assert!(moves(&second).iter().all(|&(_, from, _)| from == 0)); // Ten minutes on, connection 0 holds one market: it is left alone. assert!(tracker.rebalance(now + MOVED_STAYS).is_empty()); assert_eq!(tracker.connections[0].tokens, 2); } /// Over budget on its own, a market would only carry its load from /// connection to connection every ten minutes. #[test] fn a_market_alone_on_its_connection_is_not_moved() { let mut tracker = Tracker::new(1, 7); tracker.add(market(0)); report(&mut tracker, &[(0, 2 * CONNECTION_BYTES_PER_SECOND)]); assert!(tracker.rebalance(Instant::now()).is_empty()); assert_eq!(tracker.connections.len(), 1); } #[test] fn a_market_moved_less_than_ten_minutes_ago_stays() { let mut tracker = Tracker::new(1, 7); let budget = CONNECTION_BYTES_PER_SECOND; tracker.deal(vec![market(0), market(1)]); report(&mut tracker, &[(0, budget * 7 / 10), (1, budget / 2)]); let now = Instant::now(); // Market 0 moves to a new connection, which then turns busy. assert_eq!(moves(&tracker.rebalance(now)), [("0x0", 0, 1)]); tracker.connections[0].tokens = CONNECTION_TOKENS; tracker.add(market(2)); assert_eq!(connections(&tracker, 2), [1]); report(&mut tracker, &[(0, budget * 7 / 10), (2, budget * 6 / 10)]); let commands = tracker.rebalance(now + Duration::from_secs(60)); assert_eq!(moves(&commands), [("0x2", 1, 2)]); } #[test] fn a_connection_closed_twice_within_ten_minutes_is_split() { let mut tracker = Tracker::new(1, 7); tracker.deal((0..10).map(market).collect()); assert_eq!(tracker.connections.len(), 1); let loads: Vec<_> = (0..10).map(|id| (id, 1_000 * (id as u64 + 1))).collect(); report(&mut tracker, &loads); let now = Instant::now(); // One close: the server's moment, perhaps. assert!(tracker.closed(0, now).is_empty()); // A second within ten minutes: every second market, busiest first. let moved = tracker.closed(0, now + Duration::from_secs(5 * 60)); let ids: Vec<&str> = moves(&moved).into_iter().map(|(id, _, _)| id).collect(); assert_eq!(ids, ["0x8", "0x6", "0x4", "0x2", "0x0"]); assert!( moves(&moved) .iter() .all(|&(_, from, to)| from == 0 && to != 0) ); assert_eq!(tracker.connections[0].tokens, 10); // The split connection is left alone for ten minutes. let later = now + Duration::from_secs(6 * 60); assert!(tracker.closed(0, later).is_empty()); assert!( tracker .closed(0, later + Duration::from_secs(60)) .is_empty() ); // Ten minutes after both, two closes split it again. let much_later = now + Duration::from_secs(30 * 60); assert!(tracker.closed(0, much_later).is_empty()); assert_eq!( tracker .closed(0, much_later + Duration::from_secs(60)) .len(), 2 ); } #[test] fn closes_ten_minutes_apart_split_nothing() { let mut tracker = Tracker::new(2, 7); tracker.deal((0..10).map(market).collect()); let now = Instant::now(); assert!(tracker.closed(0, now).is_empty()); assert!(tracker.closed(0, now + SPLIT_WITHIN).is_empty()); // An unknown connection is ignored. assert!(tracker.closed(99, now).is_empty()); } #[test] fn a_completed_rescan_removes_only_the_known_markets_it_did_not_see() { let mut tracker = tracker(); for id in 0..3 { tracker.add(market(id)); } tracker.scan_started(); tracker.add(market(0)); // Added by a minute check during the scan, on a page already passed. tracker.add(market(9)); // Closed during the scan. tracker.remove("0x1"); let removed = tracker.scan_done(); assert!(matches!( removed.as_slice(), [BookCommand::Remove(market)] if market.condition_id == "0x2" )); let mut tracked: Vec<_> = tracker.markets.keys().cloned().collect(); tracked.sort(); assert_eq!(tracked, ["0x0", "0x9"]); assert_eq!(tokens(&tracker), [4, 4]); } #[test] fn a_failed_rescan_removes_nothing() { let mut tracker = tracker(); tracker.add(market(0)); tracker.scan_started(); tracker.scan_failed(); assert!(tracker.scan_done().is_empty()); assert_eq!(tracker.markets.len(), 1); } }