Archived
Every live market is subscribed on two connections chosen at random, and its book takes each change from whichever connection delivers it first: the book hash says whether a change is a duplicate, the next one, or one to hold until the change before it arrives. A change one connection misses costs nothing, each Delta carries the earliest receive time, and a connection that drops is covered by the other. When a change fits on no connection in time, the book follows one connection unchecked and asks the other for a fresh book. Connections now decode and route events to 16 book threads, independent of them; a move leaves its old connection only once the new one delivers, so it costs no change; a connection closed with 1013 twice within ten minutes is split. reconcile reads the collector's log and fails only on a mismatch no cause line explains. SIGTERM and SIGHUP stop the collector cleanly. Breaking: lanes are gone (connections and book threads are separate), and reconcile takes the log as a second argument.
1018 lines
40 KiB
Rust
1018 lines
40 KiB
Rust
//! The orchestrator: the one owner of what is tracked. It keeps the tracked
|
|
//! markets equal to Gamma's live ones, decides which connections carry each
|
|
//! market and owns every timer; the two fetchers only fetch (ADR 0002, ADR
|
|
//! 0003, roadmap/04).
|
|
|
|
use std::cmp::Reverse;
|
|
use std::collections::{HashMap, HashSet, VecDeque};
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
use std::thread::{self, JoinHandle};
|
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
|
|
|
use chrono::{DateTime, TimeDelta, Utc};
|
|
use crossbeam::channel::{Sender, select, unbounded};
|
|
use engineering_challenge::book::Output;
|
|
|
|
use super::feed::{self, BookCommand, Feed, Loads};
|
|
use super::gamma::{self, Finding, Request};
|
|
use super::wire::Received;
|
|
use super::{BOOK_THREADS, Market, book_thread, socket};
|
|
|
|
/// How many connections carry each market, first one wins (ADR 0003). 1 for
|
|
/// a run to compare with.
|
|
const SUBSCRIPTIONS: usize = 2;
|
|
/// The most tokens a connection holds: a reconnect replays every one's book
|
|
/// at once.
|
|
const CONNECTION_TOKENS: usize = 5_000;
|
|
/// What the startup deal fills a connection to, leaving room for new markets.
|
|
const STARTUP_FILL: usize = 4_500;
|
|
/// A connection's byte budget, under the lowest rate at which one was seen to
|
|
/// get a 1013 close (0.46 MB/s; docs/notes.md). Closes came at up to
|
|
/// 5.1 MB/s too: no rate is safe, only less likely to be closed.
|
|
const CONNECTION_BYTES_PER_SECOND: u64 = 300_000;
|
|
/// The startup deal's pace, about 5,000 markets a second: every connection
|
|
/// subscribing at once would send more opening books than the link carries.
|
|
const DEAL_AT_ONCE: usize = 500;
|
|
const DEAL_EVERY: Duration = Duration::from_millis(100);
|
|
/// Moves are limited so balancing doesn't churn. Rebalancing runs with each
|
|
/// minute check, every [`RECENT_EVERY`].
|
|
const MOVES_PER_MINUTE: usize = 50;
|
|
const MOVED_STAYS: Duration = Duration::from_secs(10 * 60);
|
|
/// A connection closed with 1013 twice within this is split: one close can be
|
|
/// the server's bad moment (several connections within a second), a second
|
|
/// one says it carries too much. It is then left alone for as long.
|
|
const SPLIT_WITHIN: Duration = Duration::from_secs(10 * 60);
|
|
const RECENT_EVERY: Duration = Duration::from_secs(60);
|
|
const RESCAN_EVERY: Duration = Duration::from_secs(60 * 60);
|
|
/// Gamma's indexing delay: each recent check reaches this far before the
|
|
/// last successful one. New markets were listed up to 5 minutes after their
|
|
/// `createdAt` (docs/notes.md); known markets are skipped, so the margin costs
|
|
/// only the pages it reads again.
|
|
const RECENT_MARGIN: TimeDelta = TimeDelta::minutes(15);
|
|
/// So SIGINT is seen while no timer is due.
|
|
const WAKE_EVERY: Duration = Duration::from_secs(1);
|
|
|
|
/// Runs until `stop` is set, then waits for every connection and book thread.
|
|
/// Returns how many threads failed: those that panicked, and one if a fetcher
|
|
/// died.
|
|
pub fn run(outputs: Sender<Output>, stop: &Arc<AtomicBool>) -> usize {
|
|
let (findings_tx, findings) = unbounded();
|
|
let (rescan, rescan_requests) = unbounded();
|
|
let (recent, recent_requests) = unbounded();
|
|
// Two, so a full scan never holds up the minute check. Not joined: they
|
|
// hold no output, and each ends at its next send.
|
|
let fetchers: Vec<JoinHandle<()>> = [rescan_requests, recent_requests]
|
|
.into_iter()
|
|
.map(|requests| {
|
|
let findings = findings_tx.clone();
|
|
thread::spawn(move || gamma::run(requests, findings))
|
|
})
|
|
.collect();
|
|
drop(findings_tx);
|
|
let (reports_tx, reports) = unbounded::<Loads>();
|
|
let (closes_tx, closes) = unbounded::<usize>();
|
|
let mut threads = Threads::start(&outputs, &reports_tx, &closes_tx, stop);
|
|
drop(closes_tx);
|
|
// The book threads hold the only senders the writer waits for.
|
|
drop(outputs);
|
|
drop(reports_tx);
|
|
|
|
let seed = SystemTime::now()
|
|
.duration_since(UNIX_EPOCH)
|
|
.map_or(1, |since| since.as_nanos() as u64);
|
|
let mut tracker = Tracker::new(SUBSCRIPTIONS, seed);
|
|
// Until the first scan is done: the live markets it found, waiting to be
|
|
// dealt at random. Dealing page by page would keep markets created
|
|
// together on the same connections.
|
|
let mut first_scan: Option<Vec<Market>> = Some(Vec::new());
|
|
// The startup deal's commands still to send, and whatever came behind
|
|
// them: a market's commands keep their order.
|
|
let mut queue: VecDeque<BookCommand> = VecDeque::new();
|
|
let mut next_deal = Instant::now();
|
|
let (mut added, mut removed, mut moved, mut splits) = (0, 0, 0, 0);
|
|
// The minute checks cover the first scan from its start.
|
|
let mut recent_since = Utc::now() - RECENT_MARGIN;
|
|
let mut recent_asked_at: Option<DateTime<Utc>> = None;
|
|
let mut scanning = false;
|
|
let mut scanned = false;
|
|
let mut fetcher_died = false;
|
|
let mut next_rescan = Instant::now();
|
|
let mut next_recent = Instant::now() + RECENT_EVERY;
|
|
|
|
while !stop.load(Ordering::Relaxed) {
|
|
// Only a panic ends a fetcher while this runs, even one in the middle
|
|
// of a request, whose answer would never come: nothing would keep the
|
|
// set current.
|
|
if fetchers.iter().any(JoinHandle::is_finished) {
|
|
log!("polymarket: orchestrator: a fetcher died, stopping");
|
|
stop.store(true, Ordering::Relaxed);
|
|
fetcher_died = true;
|
|
break;
|
|
}
|
|
let now = Instant::now();
|
|
let mut commands = Vec::new();
|
|
if now >= next_rescan {
|
|
if !scanning {
|
|
_ = rescan.send(Request::Rescan);
|
|
scanning = true;
|
|
}
|
|
next_rescan = now + RESCAN_EVERY;
|
|
}
|
|
if now >= next_recent {
|
|
if recent_asked_at.is_none() {
|
|
_ = recent.send(Request::Recent {
|
|
since: recent_since,
|
|
});
|
|
recent_asked_at = Some(Utc::now());
|
|
}
|
|
next_recent = now + RECENT_EVERY;
|
|
// On the loads the book threads reported at the wall-clock minute.
|
|
let moves = tracker.rebalance(now);
|
|
moved += moves.len();
|
|
commands.extend(moves);
|
|
let busiest = tracker.connections.iter().map(|one| one.load).max();
|
|
log!(
|
|
"polymarket: orchestrator: {} markets on {} connections; {added} added, {removed} \
|
|
removed, {moved} moved, {splits} splits this minute; busiest connection at {} bytes/s",
|
|
tracker.markets.len(),
|
|
tracker.connections.len(),
|
|
busiest.unwrap_or(0)
|
|
);
|
|
(added, removed, moved, splits) = (0, 0, 0, 0);
|
|
}
|
|
if !queue.is_empty() && now >= next_deal {
|
|
for command in queue.drain(..DEAL_AT_ONCE.min(queue.len())) {
|
|
threads.send(command);
|
|
}
|
|
next_deal = now + DEAL_EVERY;
|
|
}
|
|
|
|
let mut timeout = next_rescan
|
|
.min(next_recent)
|
|
.saturating_duration_since(Instant::now())
|
|
.min(WAKE_EVERY);
|
|
if !queue.is_empty() {
|
|
timeout = timeout.min(next_deal.saturating_duration_since(Instant::now()));
|
|
}
|
|
select! {
|
|
recv(findings) -> finding => match finding {
|
|
Ok(Finding::ScanStarted) => {
|
|
tracker.scan_started();
|
|
// A first scan that failed starts over.
|
|
if let Some(waiting) = &mut first_scan {
|
|
waiting.clear();
|
|
}
|
|
}
|
|
Ok(Finding::ScanPage(markets) | Finding::Created(markets)) => match &mut first_scan {
|
|
Some(waiting) => waiting.extend(markets),
|
|
None => {
|
|
let adds: Vec<_> = markets
|
|
.into_iter()
|
|
.filter_map(|market| tracker.add(market))
|
|
.collect();
|
|
added += adds.len();
|
|
commands.extend(adds);
|
|
}
|
|
},
|
|
Ok(Finding::ScanDone) => {
|
|
scanning = false;
|
|
scanned = true;
|
|
match first_scan.take() {
|
|
Some(markets) => {
|
|
let adds = tracker.deal(markets);
|
|
added += adds.len();
|
|
commands.extend(adds);
|
|
}
|
|
None => {
|
|
let gone = tracker.scan_done();
|
|
removed += gone.len();
|
|
commands.extend(gone);
|
|
}
|
|
}
|
|
}
|
|
Ok(Finding::ScanFailed) => {
|
|
scanning = false;
|
|
tracker.scan_failed();
|
|
// Nothing can be tracked before a first scan: retry at once.
|
|
if !scanned {
|
|
next_rescan = Instant::now();
|
|
}
|
|
}
|
|
Ok(Finding::Closed(ids)) => {
|
|
// Closed while the first scan runs: never dealt.
|
|
if let Some(waiting) = &mut first_scan {
|
|
waiting.retain(|market| !ids.contains(&market.condition_id));
|
|
}
|
|
let gone: Vec<_> = ids.iter().filter_map(|id| tracker.remove(id)).collect();
|
|
removed += gone.len();
|
|
commands.extend(gone);
|
|
}
|
|
Ok(Finding::RecentDone) => {
|
|
if let Some(asked_at) = recent_asked_at.take() {
|
|
recent_since = asked_at - RECENT_MARGIN;
|
|
}
|
|
}
|
|
Ok(Finding::RecentFailed) => recent_asked_at = None,
|
|
// Both fetchers are gone: the check above stops the run.
|
|
Err(_) => {}
|
|
},
|
|
recv(reports) -> loads => {
|
|
if let Ok(loads) = loads {
|
|
tracker.report(&loads);
|
|
}
|
|
},
|
|
recv(closes) -> closed => {
|
|
if let Ok(connection) = closed {
|
|
let moves = tracker.closed(connection, Instant::now());
|
|
if !moves.is_empty() {
|
|
log!(
|
|
"polymarket: orchestrator: connection {connection} closed with 1013 \
|
|
twice, split: {} markets moved off",
|
|
moves.len()
|
|
);
|
|
splits += 1;
|
|
moved += moves.len();
|
|
}
|
|
commands.extend(moves);
|
|
}
|
|
},
|
|
default(timeout) => {}
|
|
}
|
|
// A connection is open before any command names it.
|
|
threads.open(tracker.connections.len());
|
|
// The startup deal is paced; what comes while it lasts waits behind it.
|
|
if commands.len() > DEAL_AT_ONCE || !queue.is_empty() {
|
|
queue.extend(commands);
|
|
} else {
|
|
for command in commands {
|
|
threads.send(command);
|
|
}
|
|
}
|
|
}
|
|
|
|
usize::from(fetcher_died) + threads.join()
|
|
}
|
|
|
|
/// The threads the orchestrator feeds: the book threads, there from the
|
|
/// start, and the connections it opens.
|
|
struct Threads {
|
|
book_commands: Vec<Sender<BookCommand>>,
|
|
/// Every book thread's events channel, by number: each connection gets a
|
|
/// clone of them all.
|
|
events: Vec<Sender<Received>>,
|
|
closes: Sender<usize>,
|
|
book_threads: Vec<JoinHandle<()>>,
|
|
connections: Vec<JoinHandle<()>>,
|
|
stop: Arc<AtomicBool>,
|
|
}
|
|
|
|
impl Threads {
|
|
fn start(
|
|
outputs: &Sender<Output>,
|
|
reports: &Sender<Loads>,
|
|
closes: &Sender<usize>,
|
|
stop: &Arc<AtomicBool>,
|
|
) -> Self {
|
|
let mut threads = Self {
|
|
book_commands: Vec::new(),
|
|
events: Vec::new(),
|
|
closes: closes.clone(),
|
|
book_threads: Vec::new(),
|
|
connections: Vec::new(),
|
|
stop: Arc::clone(stop),
|
|
};
|
|
for number in 0..BOOK_THREADS {
|
|
let (commands, commands_rx) = unbounded();
|
|
let (events, events_rx) = unbounded();
|
|
let (outputs, reports) = (outputs.clone(), reports.clone());
|
|
threads.book_commands.push(commands);
|
|
threads.events.push(events);
|
|
threads.book_threads.push(thread::spawn(move || {
|
|
feed::run(Feed::new(number), events_rx, commands_rx, outputs, reports)
|
|
}));
|
|
}
|
|
threads
|
|
}
|
|
|
|
/// Opens connections until there are `count`, telling every book thread
|
|
/// how to reach each.
|
|
fn open(&mut self, count: usize) {
|
|
while self.connections.len() < count {
|
|
let number = self.connections.len();
|
|
let (commands, commands_rx) = unbounded();
|
|
let (events, stop) = (self.events.clone(), Arc::clone(&self.stop));
|
|
let closes = self.closes.clone();
|
|
self.connections.push(thread::spawn(move || {
|
|
socket::run(
|
|
socket::MARKET_CHANNEL_URL,
|
|
number,
|
|
&events,
|
|
&commands_rx,
|
|
&closes,
|
|
&stop,
|
|
)
|
|
}));
|
|
for book_thread in &self.book_commands {
|
|
_ = book_thread.send(BookCommand::Connected(number, commands.clone()));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// To the market's book thread. One that died keeps its markets unserved,
|
|
/// and its connections stop the run.
|
|
fn send(&self, command: BookCommand) {
|
|
let market = match &command {
|
|
BookCommand::Add(market, _)
|
|
| BookCommand::Move { market, .. }
|
|
| BookCommand::Remove(market) => market,
|
|
BookCommand::Connected(..) => unreachable!("sent by open() to every book thread"),
|
|
};
|
|
_ = self.book_commands[book_thread(&market.condition_id)].send(command);
|
|
}
|
|
|
|
/// Waits for the connections, which stop on the flag, then for the book
|
|
/// threads, which stop once no connection can send them an event: how
|
|
/// many panicked.
|
|
fn join(self) -> usize {
|
|
let panicked = |threads: Vec<JoinHandle<()>>| {
|
|
threads
|
|
.into_iter()
|
|
.map(JoinHandle::join)
|
|
.filter(Result::is_err)
|
|
.count()
|
|
};
|
|
let connections = panicked(self.connections);
|
|
drop(self.events);
|
|
drop(self.book_commands);
|
|
connections + panicked(self.book_threads)
|
|
}
|
|
}
|
|
|
|
/// A tracked market, where it is and what it weighs.
|
|
struct Tracked {
|
|
market: Market,
|
|
/// The connections that carry it, all different.
|
|
connections: Vec<usize>,
|
|
/// Bytes per second on one connection, over its book thread's last
|
|
/// report; 0 until one comes.
|
|
load: u64,
|
|
moved_at: Option<Instant>,
|
|
}
|
|
|
|
/// A connection's tokens and load, against its two limits, and its 1013
|
|
/// closes.
|
|
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
|
struct ConnectionLoad {
|
|
tokens: usize,
|
|
/// The sum of its markets' loads, bytes per second.
|
|
load: u64,
|
|
/// When it was last closed with 1013.
|
|
closed_at: Option<Instant>,
|
|
split_at: Option<Instant>,
|
|
}
|
|
|
|
/// Which connections carry each market and what it weighs, and which markets
|
|
/// a rescan in progress hasn't seen yet.
|
|
struct Tracker {
|
|
subscriptions: usize,
|
|
/// By `conditionId`.
|
|
markets: HashMap<String, Tracked>,
|
|
/// By connection number.
|
|
connections: Vec<ConnectionLoad>,
|
|
/// While a rescan runs: the markets known at its start, not seen since.
|
|
unseen: Option<HashSet<String>>,
|
|
random: Random,
|
|
}
|
|
|
|
impl Tracker {
|
|
fn new(subscriptions: usize, seed: u64) -> Self {
|
|
Self {
|
|
subscriptions,
|
|
markets: HashMap::new(),
|
|
connections: Vec::new(),
|
|
unseen: None,
|
|
random: Random(seed | 1),
|
|
}
|
|
}
|
|
|
|
/// The startup deal: every market on `subscriptions` different
|
|
/// connections, each subscription dealt in an order of its own, so a
|
|
/// market's connections hold different neighbours and no connection gets
|
|
/// the markets created together.
|
|
fn deal(&mut self, markets: Vec<Market>) -> Vec<BookCommand> {
|
|
let mut known = HashSet::new();
|
|
let markets: Vec<Market> = markets
|
|
.into_iter()
|
|
.filter(|market| known.insert(market.condition_id.clone()))
|
|
.collect();
|
|
let tokens: usize = markets.iter().map(|market| market.tokens.len()).sum();
|
|
let count = (self.subscriptions * tokens)
|
|
.div_ceil(STARTUP_FILL)
|
|
.max(self.subscriptions);
|
|
self.connections = vec![ConnectionLoad::default(); count];
|
|
let mut placed = vec![Vec::new(); markets.len()];
|
|
for _ in 0..self.subscriptions {
|
|
let mut order: Vec<usize> = (0..markets.len()).collect();
|
|
self.random.shuffle(&mut order);
|
|
for (position, market) in order.into_iter().enumerate() {
|
|
let mut connection = position % count;
|
|
while placed[market].contains(&connection) {
|
|
connection = (connection + 1) % count;
|
|
}
|
|
placed[market].push(connection);
|
|
self.connections[connection].tokens += markets[market].tokens.len();
|
|
}
|
|
}
|
|
markets
|
|
.into_iter()
|
|
.zip(placed)
|
|
.map(|(market, connections)| self.track(market, connections))
|
|
.collect()
|
|
}
|
|
|
|
/// A market it doesn't know goes on different random connections with
|
|
/// room: its Add.
|
|
fn add(&mut self, market: Market) -> Option<BookCommand> {
|
|
if let Some(unseen) = &mut self.unseen {
|
|
unseen.remove(&market.condition_id);
|
|
}
|
|
if self.markets.contains_key(&market.condition_id) {
|
|
return None;
|
|
}
|
|
let mut connections = Vec::new();
|
|
for _ in 0..self.subscriptions {
|
|
let connection = self.pick(market.tokens.len(), 0, &connections);
|
|
self.connections[connection].tokens += market.tokens.len();
|
|
connections.push(connection);
|
|
}
|
|
Some(self.track(market, connections))
|
|
}
|
|
|
|
fn track(&mut self, market: Market, connections: Vec<usize>) -> BookCommand {
|
|
let tracked = Tracked {
|
|
market: market.clone(),
|
|
connections: connections.clone(),
|
|
load: 0,
|
|
moved_at: None,
|
|
};
|
|
self.markets.insert(market.condition_id.clone(), tracked);
|
|
BookCommand::Add(market, connections)
|
|
}
|
|
|
|
/// A random connection with room for `tokens` more and `load` more
|
|
/// bytes/s, none of `except`; a new connection if none has. Random, since
|
|
/// the least loaded one would take every market of a batch created
|
|
/// together: the cluster a load average can't see.
|
|
fn pick(&mut self, tokens: usize, load: u64, except: &[usize]) -> usize {
|
|
let with_room: Vec<usize> = (0..self.connections.len())
|
|
.filter(|number| {
|
|
let connection = &self.connections[*number];
|
|
!except.contains(number)
|
|
&& connection.tokens + tokens <= CONNECTION_TOKENS
|
|
&& connection.load + load <= CONNECTION_BYTES_PER_SECOND
|
|
})
|
|
.collect();
|
|
if with_room.is_empty() {
|
|
self.connections.push(ConnectionLoad::default());
|
|
return self.connections.len() - 1;
|
|
}
|
|
with_room[self.random.below(with_room.len())]
|
|
}
|
|
|
|
fn remove(&mut self, condition_id: &str) -> Option<BookCommand> {
|
|
let tracked = self.markets.remove(condition_id)?;
|
|
for &connection in &tracked.connections {
|
|
let connection = &mut self.connections[connection];
|
|
connection.tokens -= tracked.market.tokens.len();
|
|
connection.load -= tracked.load;
|
|
}
|
|
Some(BookCommand::Remove(tracked.market))
|
|
}
|
|
|
|
/// A book thread's loads per market, each weighing on every connection
|
|
/// that carries the market.
|
|
fn report(&mut self, loads: &Loads) {
|
|
for (condition_id, &load) in loads {
|
|
let Some(tracked) = self.markets.get_mut(condition_id) else {
|
|
continue;
|
|
};
|
|
for &connection in &tracked.connections {
|
|
let connection = &mut self.connections[connection];
|
|
connection.load = connection.load - tracked.load + load;
|
|
}
|
|
tracked.load = load;
|
|
}
|
|
}
|
|
|
|
/// Moves the busiest markets' subscriptions off every connection over its
|
|
/// byte budget, busiest connection first, each to a random connection with
|
|
/// room that doesn't carry the market, until it is under: at most
|
|
/// [`MOVES_PER_MINUTE`], and none moved in the last [`MOVED_STAYS`].
|
|
fn rebalance(&mut self, now: Instant) -> Vec<BookCommand> {
|
|
let mut moves = Vec::new();
|
|
let mut over: Vec<usize> = (0..self.connections.len())
|
|
.filter(|&number| self.connections[number].load > CONNECTION_BYTES_PER_SECOND)
|
|
.collect();
|
|
over.sort_by_key(|&number| Reverse(self.connections[number].load));
|
|
for from in over {
|
|
// ponytail: a scan of every market per connection over budget,
|
|
// fine at a few hundred thousand once a minute; per-connection
|
|
// market sets if not.
|
|
let mut busiest: Vec<(String, u64)> = self
|
|
.markets
|
|
.iter()
|
|
.filter(|(_, tracked)| {
|
|
tracked.connections.contains(&from)
|
|
&& tracked
|
|
.moved_at
|
|
.is_none_or(|at| now.duration_since(at) >= MOVED_STAYS)
|
|
})
|
|
.map(|(id, tracked)| (id.clone(), tracked.load))
|
|
.collect();
|
|
busiest.sort_by_key(|&(_, load)| Reverse(load));
|
|
for (id, load) in busiest {
|
|
let size = self.markets[&id].market.tokens.len();
|
|
// Alone on its connection, a market would only take its load
|
|
// along; one with no load would lighten nothing.
|
|
if self.connections[from].load <= CONNECTION_BYTES_PER_SECOND
|
|
|| moves.len() == MOVES_PER_MINUTE
|
|
|| self.connections[from].tokens == size
|
|
|| load == 0
|
|
{
|
|
break;
|
|
}
|
|
moves.push(self.move_off(&id, from, now));
|
|
}
|
|
}
|
|
moves
|
|
}
|
|
|
|
/// Moves the market's subscription on `from` to a random connection with
|
|
/// room that doesn't carry it yet: its Move.
|
|
fn move_off(&mut self, id: &str, from: usize, now: Instant) -> BookCommand {
|
|
let tracked = &self.markets[id];
|
|
let (size, load) = (tracked.market.tokens.len(), tracked.load);
|
|
let carrying = tracked.connections.clone();
|
|
let to = self.pick(size, load, &carrying);
|
|
self.connections[from].tokens -= size;
|
|
self.connections[from].load -= load;
|
|
self.connections[to].tokens += size;
|
|
self.connections[to].load += load;
|
|
let tracked = self.markets.get_mut(id).expect("tracked");
|
|
for connection in &mut tracked.connections {
|
|
if *connection == from {
|
|
*connection = to;
|
|
}
|
|
}
|
|
tracked.moved_at = Some(now);
|
|
BookCommand::Move {
|
|
market: tracked.market.clone(),
|
|
from,
|
|
to,
|
|
}
|
|
}
|
|
|
|
/// A connection closed with 1013: on its second close within
|
|
/// [`SPLIT_WITHIN`], every second of its markets, in load order, moves off
|
|
/// at once, beyond rebalancing's limits.
|
|
fn closed(&mut self, connection: usize, now: Instant) -> Vec<BookCommand> {
|
|
let Some(closed) = self.connections.get_mut(connection) else {
|
|
return Vec::new();
|
|
};
|
|
let recent =
|
|
|at: Option<Instant>| at.is_some_and(|at| now.duration_since(at) < SPLIT_WITHIN);
|
|
let again = recent(closed.closed_at);
|
|
closed.closed_at = Some(now);
|
|
if !again || recent(closed.split_at) {
|
|
return Vec::new();
|
|
}
|
|
closed.split_at = Some(now);
|
|
let mut carried: Vec<(String, u64)> = self
|
|
.markets
|
|
.iter()
|
|
.filter(|(_, tracked)| tracked.connections.contains(&connection))
|
|
.map(|(id, tracked)| (id.clone(), tracked.load))
|
|
.collect();
|
|
// Busiest first, ties by id so the split is the same each time.
|
|
carried.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
|
|
carried
|
|
.into_iter()
|
|
.skip(1)
|
|
.step_by(2)
|
|
.map(|(id, _)| self.move_off(&id, connection, now))
|
|
.collect()
|
|
}
|
|
|
|
fn scan_started(&mut self) {
|
|
self.unseen = Some(self.markets.keys().cloned().collect());
|
|
}
|
|
|
|
/// Removes the markets known at the scan's start and not seen in it:
|
|
/// those added since were never at risk.
|
|
fn scan_done(&mut self) -> Vec<BookCommand> {
|
|
let unseen = self.unseen.take().unwrap_or_default();
|
|
unseen.iter().filter_map(|id| self.remove(id)).collect()
|
|
}
|
|
|
|
/// Half a listing says nothing of the other half: nothing is removed.
|
|
fn scan_failed(&mut self) {
|
|
self.unseen = None;
|
|
}
|
|
}
|
|
|
|
/// Placement needs choices that differ, not good randomness: xorshift64.
|
|
struct Random(u64);
|
|
|
|
impl Random {
|
|
fn next(&mut self) -> u64 {
|
|
self.0 ^= self.0 << 13;
|
|
self.0 ^= self.0 >> 7;
|
|
self.0 ^= self.0 << 17;
|
|
self.0
|
|
}
|
|
|
|
/// A number under `bound`, which is not 0.
|
|
fn below(&mut self, bound: usize) -> usize {
|
|
(self.next() % bound as u64) as usize
|
|
}
|
|
|
|
fn shuffle<T>(&mut self, items: &mut [T]) {
|
|
for last in (1..items.len()).rev() {
|
|
items.swap(last, self.below(last + 1));
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn market(id: usize) -> Market {
|
|
Market {
|
|
condition_id: format!("0x{id}"),
|
|
tokens: vec![format!("{id}a"), format!("{id}b")],
|
|
min_order_size: "5".to_string(),
|
|
neg_risk: false,
|
|
}
|
|
}
|
|
|
|
/// A tracker with two subscriptions per market, as first one wins has.
|
|
fn tracker() -> Tracker {
|
|
Tracker::new(2, 7)
|
|
}
|
|
|
|
fn tokens(tracker: &Tracker) -> Vec<usize> {
|
|
tracker.connections.iter().map(|one| one.tokens).collect()
|
|
}
|
|
|
|
fn connections(tracker: &Tracker, id: usize) -> Vec<usize> {
|
|
tracker.markets[&format!("0x{id}")].connections.clone()
|
|
}
|
|
|
|
/// Reports `bytes_per_second` for each market, as its book thread would.
|
|
fn report(tracker: &mut Tracker, loads: &[(usize, u64)]) {
|
|
let loads = loads
|
|
.iter()
|
|
.map(|&(id, bytes_per_second)| (format!("0x{id}"), bytes_per_second))
|
|
.collect();
|
|
tracker.report(&loads);
|
|
}
|
|
|
|
/// The connection each of `commands` moves a market from and to.
|
|
fn moves(commands: &[BookCommand]) -> Vec<(&str, usize, usize)> {
|
|
commands
|
|
.iter()
|
|
.map(|command| match command {
|
|
BookCommand::Move { market, from, to } => {
|
|
(market.condition_id.as_str(), *from, *to)
|
|
}
|
|
other => panic!("not a move: {other:?}"),
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
fn the_startup_deal_is_even_and_gives_each_market_different_connections() {
|
|
let mut tracker = tracker();
|
|
// 10,000 markets, 20,000 tokens, twice: 9 connections of 4,500.
|
|
let adds = tracker.deal((0..10_000).map(market).collect());
|
|
assert_eq!(adds.len(), 10_000);
|
|
assert_eq!(tracker.connections.len(), 9);
|
|
let tokens = tokens(&tracker);
|
|
assert_eq!(tokens.iter().sum::<usize>(), 40_000);
|
|
assert!(tokens.iter().all(|&held| held <= CONNECTION_TOKENS));
|
|
// Even, but for the markets whose second connection had to be the next.
|
|
assert!(tokens.iter().max().unwrap() - tokens.iter().min().unwrap() <= 100);
|
|
for command in &adds {
|
|
let BookCommand::Add(market, on) = command else {
|
|
panic!("not an add: {command:?}");
|
|
};
|
|
assert_eq!(on.len(), 2);
|
|
assert_ne!(on[0], on[1], "{}", market.condition_id);
|
|
}
|
|
// Markets created together (neighbours in the list) are spread: the
|
|
// first 100 don't share one pair of connections.
|
|
let pairs: HashSet<Vec<usize>> = (0..100).map(|id| connections(&tracker, id)).collect();
|
|
assert!(pairs.len() > 30, "{}", pairs.len());
|
|
}
|
|
|
|
#[test]
|
|
fn a_market_listed_twice_is_dealt_once() {
|
|
let mut tracker = tracker();
|
|
let adds = tracker.deal(vec![market(0), market(1), market(0)]);
|
|
assert_eq!(adds.len(), 2);
|
|
assert_eq!(tokens(&tracker).iter().sum::<usize>(), 8);
|
|
}
|
|
|
|
#[test]
|
|
fn a_new_market_goes_on_different_connections_and_one_opens_when_too_few_have_room() {
|
|
let mut tracker = tracker();
|
|
// Nothing open: its two connections open.
|
|
tracker.add(market(0)).unwrap();
|
|
assert_eq!(connections(&tracker, 0).len(), 2);
|
|
assert_eq!(tokens(&tracker), [2, 2]);
|
|
// Both fill together; then a third and a fourth open.
|
|
for id in 1..CONNECTION_TOKENS / 2 {
|
|
tracker.add(market(id)).unwrap();
|
|
}
|
|
assert_eq!(tokens(&tracker), [CONNECTION_TOKENS, CONNECTION_TOKENS]);
|
|
tracker.add(market(9_999)).unwrap();
|
|
assert_eq!(
|
|
tokens(&tracker),
|
|
[CONNECTION_TOKENS, CONNECTION_TOKENS, 2, 2]
|
|
);
|
|
// Room on one connection only: the other subscription opens a new one.
|
|
tracker.remove("0x0").unwrap();
|
|
tracker.connections[1].tokens = CONNECTION_TOKENS;
|
|
tracker.connections[2].tokens = CONNECTION_TOKENS;
|
|
tracker.connections[3].tokens = CONNECTION_TOKENS;
|
|
tracker.add(market(10_000)).unwrap();
|
|
let mut on = connections(&tracker, 10_000);
|
|
on.sort();
|
|
assert_eq!(on, [0, 4]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_connection_over_budget_takes_no_new_market() {
|
|
let mut tracker = tracker();
|
|
tracker.add(market(0)).unwrap();
|
|
report(&mut tracker, &[(0, CONNECTION_BYTES_PER_SECOND + 1)]);
|
|
tracker.add(market(1)).unwrap();
|
|
let mut on = connections(&tracker, 1);
|
|
on.sort();
|
|
assert_eq!(on, [2, 3]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_known_market_is_skipped() {
|
|
let mut tracker = tracker();
|
|
assert!(tracker.add(market(1)).is_some());
|
|
assert!(tracker.add(market(1)).is_none());
|
|
assert_eq!(tokens(&tracker), [2, 2]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_report_weighs_on_each_of_the_markets_connections() {
|
|
let mut tracker = tracker();
|
|
tracker.add(market(1));
|
|
tracker.add(market(2));
|
|
report(&mut tracker, &[(1, 300), (2, 100)]);
|
|
let loads: Vec<u64> = tracker.connections.iter().map(|one| one.load).collect();
|
|
assert_eq!(loads, [400, 400]);
|
|
// A later report replaces the load; an unknown market's is ignored.
|
|
report(&mut tracker, &[(1, 50), (9, 900)]);
|
|
assert_eq!(tracker.connections[0].load, 150);
|
|
// A market gone takes its load along.
|
|
tracker.remove("0x1");
|
|
assert_eq!(tracker.connections[1].load, 100);
|
|
assert_eq!(tokens(&tracker), [2, 2]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_connection_over_budget_moves_its_busiest_markets_until_under() {
|
|
let mut tracker = tracker();
|
|
for id in 0..10 {
|
|
tracker.add(market(id));
|
|
}
|
|
tracker.connections.push(ConnectionLoad::default());
|
|
let budget = CONNECTION_BYTES_PER_SECOND;
|
|
// Both connections at 1.5 budgets, market 0 weighing half a budget.
|
|
let mut loads = vec![(0, budget / 2)];
|
|
loads.extend((1..10).map(|id| (id, budget / 9)));
|
|
report(&mut tracker, &loads);
|
|
let now = Instant::now();
|
|
let commands = tracker.rebalance(now);
|
|
|
|
// Connection 0 sheds its busiest market, which is enough. Market 0
|
|
// then stays put, its other subscription included, so connection 1
|
|
// sheds smaller ones: a market's two subscriptions never move in the
|
|
// same minute.
|
|
let moved = moves(&commands);
|
|
assert_eq!(moved[0], ("0x0", 0, 2));
|
|
assert_eq!(connections(&tracker, 0), [2, 1]);
|
|
assert!(
|
|
moved[1..]
|
|
.iter()
|
|
.all(|&(id, from, _)| id != "0x0" && from == 1)
|
|
);
|
|
assert_eq!(moved.len(), 1 + 5);
|
|
assert!(tracker.connections.iter().all(|one| one.load <= budget));
|
|
assert_eq!(tokens(&tracker).iter().sum::<usize>(), 40);
|
|
// Under budget now: nothing more moves.
|
|
assert!(tracker.rebalance(now).is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn a_move_never_targets_a_connection_that_carries_the_market() {
|
|
for seed in 1..50 {
|
|
let mut tracker = Tracker::new(2, seed);
|
|
for id in 0..6 {
|
|
tracker.add(market(id));
|
|
}
|
|
// Room elsewhere, on several connections.
|
|
tracker.connections.resize(6, ConnectionLoad::default());
|
|
report(
|
|
&mut tracker,
|
|
&[(0, CONNECTION_BYTES_PER_SECOND), (1, 1_000)],
|
|
);
|
|
let before = connections(&tracker, 0);
|
|
for (_, from, to) in moves(&tracker.rebalance(Instant::now())) {
|
|
assert!(before.contains(&from));
|
|
assert!(!before.contains(&to), "seed {seed}");
|
|
}
|
|
let after = connections(&tracker, 0);
|
|
assert_ne!(after[0], after[1], "seed {seed}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn the_busiest_connection_over_budget_is_rebalanced_first() {
|
|
let mut tracker = Tracker::new(1, 7);
|
|
tracker.deal(vec![market(0), market(1)]);
|
|
tracker.connections.push(ConnectionLoad::default());
|
|
tracker.connections.push(ConnectionLoad::default());
|
|
// One connection so far: put markets 2 and 3 on another.
|
|
for id in [2, 3] {
|
|
tracker.connections[0].tokens = CONNECTION_TOKENS;
|
|
tracker.connections[2].tokens = CONNECTION_TOKENS;
|
|
tracker.add(market(id));
|
|
}
|
|
tracker.connections[0].tokens = 4;
|
|
tracker.connections[2].tokens = 0;
|
|
assert_eq!(tokens(&tracker), [4, 4, 0]);
|
|
let budget = CONNECTION_BYTES_PER_SECOND;
|
|
report(&mut tracker, &[(0, budget), (1, budget / 10)]);
|
|
report(&mut tracker, &[(2, budget / 2), (3, budget)]);
|
|
let commands = tracker.rebalance(Instant::now());
|
|
assert_eq!(moves(&commands)[0], ("0x3", 1, 2));
|
|
}
|
|
|
|
/// The markets that may move have no load: moving them would empty the
|
|
/// connection and lighten nothing.
|
|
#[test]
|
|
fn markets_with_no_load_are_not_moved() {
|
|
let mut tracker = Tracker::new(1, 7);
|
|
tracker.deal((0..4).map(market).collect());
|
|
report(&mut tracker, &[(0, 2 * CONNECTION_BYTES_PER_SECOND)]);
|
|
let now = Instant::now();
|
|
tracker.markets.get_mut("0x0").unwrap().moved_at = Some(now);
|
|
assert!(tracker.rebalance(now).is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn moves_are_limited_and_a_moved_market_stays() {
|
|
let mut tracker = Tracker::new(1, 7);
|
|
let count = 2 * MOVES_PER_MINUTE;
|
|
tracker.deal((0..count).map(market).collect());
|
|
assert_eq!(tracker.connections.len(), 1);
|
|
// Every market alone near a budget: each needs a connection of its own.
|
|
let loads: Vec<_> = (0..count)
|
|
.map(|id| (id, CONNECTION_BYTES_PER_SECOND * 9 / 10))
|
|
.collect();
|
|
report(&mut tracker, &loads);
|
|
let now = Instant::now();
|
|
let first = tracker.rebalance(now);
|
|
assert_eq!(first.len(), MOVES_PER_MINUTE);
|
|
// The rest move a minute later, but the last, alone and under budget.
|
|
let second = tracker.rebalance(now + Duration::from_secs(60));
|
|
assert_eq!(second.len(), MOVES_PER_MINUTE - 1);
|
|
assert!(moves(&second).iter().all(|&(_, from, _)| from == 0));
|
|
// Ten minutes on, connection 0 holds one market: it is left alone.
|
|
assert!(tracker.rebalance(now + MOVED_STAYS).is_empty());
|
|
assert_eq!(tracker.connections[0].tokens, 2);
|
|
}
|
|
|
|
/// Over budget on its own, a market would only carry its load from
|
|
/// connection to connection every ten minutes.
|
|
#[test]
|
|
fn a_market_alone_on_its_connection_is_not_moved() {
|
|
let mut tracker = Tracker::new(1, 7);
|
|
tracker.add(market(0));
|
|
report(&mut tracker, &[(0, 2 * CONNECTION_BYTES_PER_SECOND)]);
|
|
assert!(tracker.rebalance(Instant::now()).is_empty());
|
|
assert_eq!(tracker.connections.len(), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn a_market_moved_less_than_ten_minutes_ago_stays() {
|
|
let mut tracker = Tracker::new(1, 7);
|
|
let budget = CONNECTION_BYTES_PER_SECOND;
|
|
tracker.deal(vec![market(0), market(1)]);
|
|
report(&mut tracker, &[(0, budget * 7 / 10), (1, budget / 2)]);
|
|
let now = Instant::now();
|
|
// Market 0 moves to a new connection, which then turns busy.
|
|
assert_eq!(moves(&tracker.rebalance(now)), [("0x0", 0, 1)]);
|
|
tracker.connections[0].tokens = CONNECTION_TOKENS;
|
|
tracker.add(market(2));
|
|
assert_eq!(connections(&tracker, 2), [1]);
|
|
report(&mut tracker, &[(0, budget * 7 / 10), (2, budget * 6 / 10)]);
|
|
let commands = tracker.rebalance(now + Duration::from_secs(60));
|
|
assert_eq!(moves(&commands), [("0x2", 1, 2)]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_connection_closed_twice_within_ten_minutes_is_split() {
|
|
let mut tracker = Tracker::new(1, 7);
|
|
tracker.deal((0..10).map(market).collect());
|
|
assert_eq!(tracker.connections.len(), 1);
|
|
let loads: Vec<_> = (0..10).map(|id| (id, 1_000 * (id as u64 + 1))).collect();
|
|
report(&mut tracker, &loads);
|
|
let now = Instant::now();
|
|
// One close: the server's moment, perhaps.
|
|
assert!(tracker.closed(0, now).is_empty());
|
|
// A second within ten minutes: every second market, busiest first.
|
|
let moved = tracker.closed(0, now + Duration::from_secs(5 * 60));
|
|
let ids: Vec<&str> = moves(&moved).into_iter().map(|(id, _, _)| id).collect();
|
|
assert_eq!(ids, ["0x8", "0x6", "0x4", "0x2", "0x0"]);
|
|
assert!(
|
|
moves(&moved)
|
|
.iter()
|
|
.all(|&(_, from, to)| from == 0 && to != 0)
|
|
);
|
|
assert_eq!(tracker.connections[0].tokens, 10);
|
|
// The split connection is left alone for ten minutes.
|
|
let later = now + Duration::from_secs(6 * 60);
|
|
assert!(tracker.closed(0, later).is_empty());
|
|
assert!(
|
|
tracker
|
|
.closed(0, later + Duration::from_secs(60))
|
|
.is_empty()
|
|
);
|
|
// Ten minutes after both, two closes split it again.
|
|
let much_later = now + Duration::from_secs(30 * 60);
|
|
assert!(tracker.closed(0, much_later).is_empty());
|
|
assert_eq!(
|
|
tracker
|
|
.closed(0, much_later + Duration::from_secs(60))
|
|
.len(),
|
|
2
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn closes_ten_minutes_apart_split_nothing() {
|
|
let mut tracker = Tracker::new(2, 7);
|
|
tracker.deal((0..10).map(market).collect());
|
|
let now = Instant::now();
|
|
assert!(tracker.closed(0, now).is_empty());
|
|
assert!(tracker.closed(0, now + SPLIT_WITHIN).is_empty());
|
|
// An unknown connection is ignored.
|
|
assert!(tracker.closed(99, now).is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn a_completed_rescan_removes_only_the_known_markets_it_did_not_see() {
|
|
let mut tracker = tracker();
|
|
for id in 0..3 {
|
|
tracker.add(market(id));
|
|
}
|
|
tracker.scan_started();
|
|
tracker.add(market(0));
|
|
// Added by a minute check during the scan, on a page already passed.
|
|
tracker.add(market(9));
|
|
// Closed during the scan.
|
|
tracker.remove("0x1");
|
|
let removed = tracker.scan_done();
|
|
|
|
assert!(matches!(
|
|
removed.as_slice(),
|
|
[BookCommand::Remove(market)] if market.condition_id == "0x2"
|
|
));
|
|
let mut tracked: Vec<_> = tracker.markets.keys().cloned().collect();
|
|
tracked.sort();
|
|
assert_eq!(tracked, ["0x0", "0x9"]);
|
|
assert_eq!(tokens(&tracker), [4, 4]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_failed_rescan_removes_nothing() {
|
|
let mut tracker = tracker();
|
|
tracker.add(market(0));
|
|
tracker.scan_started();
|
|
tracker.scan_failed();
|
|
assert!(tracker.scan_done().is_empty());
|
|
assert_eq!(tracker.markets.len(), 1);
|
|
}
|
|
}
|