This repository has been archived on 2026-11-16. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
orderbook-collector/src/main.rs
T
pascal 1a5bbcd602 feat!: a generic core, and Kalshi through it
The orchestrator, the connection loop and the book thread's shell move to
src/core/, generic over an Exchange and its loss rule (ADR 0004).
Polymarket runs through it unchanged. Kalshi joins: a signed handshake
(KALSHI_KEY_ID_FILE, KALSHI_PRIVATE_KEY_FILE), a reconnect on a skipped
sequence number, first one wins by clock and rank, and its market listing
with markets tracked from three hours before they open.

Breaking: the exchange is now the first argument,
`engineering-challenge <polymarket|kalshi|kalshi-demo> [OUTPUT]`.
2026-11-11 20:54:17 +00:00

99 lines
3.8 KiB
Rust

//! The collector (roadmap/04, roadmap/06): an orchestrator keeps the tracked
//! markets equal to the exchange's live ones and places them on connections
//! (read, stamp, decode, route), which feed the book threads (fold, check,
//! resync), all writing through one writer (SBE).
//! Usage: `engineering-challenge EXCHANGE [OUTPUT]`, EXCHANGE one of
//! `polymarket`, `kalshi` or `kalshi-demo`, OUTPUT defaulting to stdout; logs
//! go to stderr.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;
use crossbeam::channel::{Receiver, unbounded};
use engineering_challenge::util::AlignedSbeWriter;
/// Prints a line to stderr, prefixed with the UTC time.
macro_rules! log {
($($arg:tt)*) => {
eprintln!("{} {}", chrono::Utc::now().format("%H:%M:%S%.3f"), format_args!($($arg)*))
};
}
mod core;
mod kalshi;
mod polymarket;
use engineering_challenge::book::Output;
/// `AlignedSbeReader::open` reads nothing else.
const CHUNK: usize = 64 << 20;
const USAGE: &str = "usage: engineering-challenge <polymarket|kalshi|kalshi-demo> [OUTPUT]";
fn main() -> anyhow::Result<()> {
let mut args = std::env::args().skip(1);
let exchange = args.next().ok_or_else(|| anyhow::anyhow!(USAGE))?;
let output = args.next().unwrap_or_else(|| "/dev/stdout".to_string());
// Before anything starts: a key that can't be read stops the run here.
match exchange.as_str() {
"polymarket" => collect(
polymarket::Polymarket {
url: polymarket::MARKET_CHANNEL_URL.to_string(),
},
output,
),
"kalshi" => collect(
kalshi::Kalshi::production(kalshi::sign::Key::from_env()?),
output,
),
"kalshi-demo" => collect(kalshi::Kalshi::demo(kalshi::sign::Key::from_env()?), output),
_ => anyhow::bail!("unknown exchange {exchange:?}; {USAGE}"),
}
}
/// Collects `exchange` into `output` until SIGINT.
fn collect<E: core::Exchange>(exchange: E, output: String) -> anyhow::Result<()> {
let stop = Arc::new(AtomicBool::new(false));
let on_sigint = Arc::clone(&stop);
ctrlc::set_handler(move || on_sigint.store(true, Ordering::Relaxed))?;
let (outputs_tx, outputs_rx) = unbounded();
let writer_stop = Arc::clone(&stop);
let writer = thread::spawn(move || {
let written = write(&output, outputs_rx);
// A writer that failed stops the rest: nothing it reads is kept.
writer_stop.store(true, Ordering::Relaxed);
written
});
// SIGINT (and SIGTERM, SIGHUP) stops the orchestrator and the
// connections; the book threads stop once every connection has, and the
// writer flushes once they all have, even after a panic upstream (the run
// still fails then).
let failed = core::orchestrator::run(Arc::new(exchange), outputs_tx, &stop);
let written = writer
.join()
.map_err(|_| anyhow::anyhow!("the writer thread panicked"))?;
log!("output closed");
written?;
anyhow::ensure!(failed == 0, "{failed} threads failed");
Ok(())
}
/// The writer thread: writes outputs in arrival order until every book
/// thread is gone, then flushes the last chunk.
// ponytail: the chunk being filled lives only in memory until it is full, so
// a hard crash loses it; flushing more often pads every flush to 64 MiB.
// Kept as a ceiling in M4 (roadmap/04): writing to a file of our own and
// copying it into chunks would lift it.
fn write(path: &str, outputs: Receiver<Output>) -> anyhow::Result<()> {
let mut writer = AlignedSbeWriter::create(path, CHUNK)?;
for output in outputs {
match output {
Output::Snapshot(snapshot) => writer.write_msg(&snapshot)?,
Output::Delta(delta) => writer.write_msg(&delta)?,
}
}
writer.flush()
}