//! The collector (roadmap/04, roadmap/06): an orchestrator keeps the tracked //! markets equal to the exchange's live ones and places them on connections //! (read, stamp, decode, route), which feed the book threads (fold, check, //! resync), all writing through one writer (SBE). //! Usage: `engineering-challenge EXCHANGE [OUTPUT]`, EXCHANGE one of //! `polymarket`, `kalshi` or `kalshi-demo`, OUTPUT defaulting to stdout; logs //! go to stderr. use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::thread; use crossbeam::channel::{Receiver, unbounded}; use engineering_challenge::util::AlignedSbeWriter; /// Prints a line to stderr, prefixed with the UTC time. macro_rules! log { ($($arg:tt)*) => { eprintln!("{} {}", chrono::Utc::now().format("%H:%M:%S%.3f"), format_args!($($arg)*)) }; } mod core; mod kalshi; mod polymarket; use engineering_challenge::book::Output; /// `AlignedSbeReader::open` reads nothing else. const CHUNK: usize = 64 << 20; const USAGE: &str = "usage: engineering-challenge [OUTPUT]"; fn main() -> anyhow::Result<()> { let mut args = std::env::args().skip(1); let exchange = args.next().ok_or_else(|| anyhow::anyhow!(USAGE))?; let output = args.next().unwrap_or_else(|| "/dev/stdout".to_string()); // Before anything starts: a key that can't be read stops the run here. match exchange.as_str() { "polymarket" => collect( polymarket::Polymarket { url: polymarket::MARKET_CHANNEL_URL.to_string(), }, output, ), "kalshi" => collect( kalshi::Kalshi::production(kalshi::sign::Key::from_env()?), output, ), "kalshi-demo" => collect(kalshi::Kalshi::demo(kalshi::sign::Key::from_env()?), output), _ => anyhow::bail!("unknown exchange {exchange:?}; {USAGE}"), } } /// Collects `exchange` into `output` until SIGINT. fn collect(exchange: E, output: String) -> anyhow::Result<()> { let stop = Arc::new(AtomicBool::new(false)); let on_sigint = Arc::clone(&stop); ctrlc::set_handler(move || on_sigint.store(true, Ordering::Relaxed))?; let (outputs_tx, outputs_rx) = unbounded(); let writer_stop = Arc::clone(&stop); let writer = thread::spawn(move || { let written = write(&output, outputs_rx); // A writer that failed stops the rest: nothing it reads is kept. writer_stop.store(true, Ordering::Relaxed); written }); // SIGINT (and SIGTERM, SIGHUP) stops the orchestrator and the // connections; the book threads stop once every connection has, and the // writer flushes once they all have, even after a panic upstream (the run // still fails then). let failed = core::orchestrator::run(Arc::new(exchange), outputs_tx, &stop); let written = writer .join() .map_err(|_| anyhow::anyhow!("the writer thread panicked"))?; log!("output closed"); written?; anyhow::ensure!(failed == 0, "{failed} threads failed"); Ok(()) } /// The writer thread: writes outputs in arrival order until every book /// thread is gone, then flushes the last chunk. // ponytail: the chunk being filled lives only in memory until it is full, so // a hard crash loses it; flushing more often pads every flush to 64 MiB. // Kept as a ceiling in M4 (roadmap/04): writing to a file of our own and // copying it into chunks would lift it. fn write(path: &str, outputs: Receiver) -> anyhow::Result<()> { let mut writer = AlignedSbeWriter::create(path, CHUNK)?; for output in outputs { match output { Output::Snapshot(snapshot) => writer.write_msg(&snapshot)?, Output::Delta(delta) => writer.write_msg(&delta)?, } } writer.flush() }