Archived
The orchestrator, the connection loop and the book thread's shell move to src/core/, generic over an Exchange and its loss rule (ADR 0004). Polymarket runs through it unchanged. Kalshi joins: a signed handshake (KALSHI_KEY_ID_FILE, KALSHI_PRIVATE_KEY_FILE), a reconnect on a skipped sequence number, first one wins by clock and rank, and its market listing with markets tracked from three hours before they open. Breaking: the exchange is now the first argument, `engineering-challenge <polymarket|kalshi|kalshi-demo> [OUTPUT]`.
99 lines
3.8 KiB
Rust
99 lines
3.8 KiB
Rust
//! The collector (roadmap/04, roadmap/06): an orchestrator keeps the tracked
|
|
//! markets equal to the exchange's live ones and places them on connections
|
|
//! (read, stamp, decode, route), which feed the book threads (fold, check,
|
|
//! resync), all writing through one writer (SBE).
|
|
//! Usage: `engineering-challenge EXCHANGE [OUTPUT]`, EXCHANGE one of
|
|
//! `polymarket`, `kalshi` or `kalshi-demo`, OUTPUT defaulting to stdout; logs
|
|
//! go to stderr.
|
|
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
use std::thread;
|
|
|
|
use crossbeam::channel::{Receiver, unbounded};
|
|
use engineering_challenge::util::AlignedSbeWriter;
|
|
|
|
/// Prints a line to stderr, prefixed with the UTC time.
|
|
macro_rules! log {
|
|
($($arg:tt)*) => {
|
|
eprintln!("{} {}", chrono::Utc::now().format("%H:%M:%S%.3f"), format_args!($($arg)*))
|
|
};
|
|
}
|
|
|
|
mod core;
|
|
mod kalshi;
|
|
mod polymarket;
|
|
|
|
use engineering_challenge::book::Output;
|
|
|
|
/// `AlignedSbeReader::open` reads nothing else.
|
|
const CHUNK: usize = 64 << 20;
|
|
|
|
const USAGE: &str = "usage: engineering-challenge <polymarket|kalshi|kalshi-demo> [OUTPUT]";
|
|
|
|
fn main() -> anyhow::Result<()> {
|
|
let mut args = std::env::args().skip(1);
|
|
let exchange = args.next().ok_or_else(|| anyhow::anyhow!(USAGE))?;
|
|
let output = args.next().unwrap_or_else(|| "/dev/stdout".to_string());
|
|
// Before anything starts: a key that can't be read stops the run here.
|
|
match exchange.as_str() {
|
|
"polymarket" => collect(
|
|
polymarket::Polymarket {
|
|
url: polymarket::MARKET_CHANNEL_URL.to_string(),
|
|
},
|
|
output,
|
|
),
|
|
"kalshi" => collect(
|
|
kalshi::Kalshi::production(kalshi::sign::Key::from_env()?),
|
|
output,
|
|
),
|
|
"kalshi-demo" => collect(kalshi::Kalshi::demo(kalshi::sign::Key::from_env()?), output),
|
|
_ => anyhow::bail!("unknown exchange {exchange:?}; {USAGE}"),
|
|
}
|
|
}
|
|
|
|
/// Collects `exchange` into `output` until SIGINT.
|
|
fn collect<E: core::Exchange>(exchange: E, output: String) -> anyhow::Result<()> {
|
|
let stop = Arc::new(AtomicBool::new(false));
|
|
let on_sigint = Arc::clone(&stop);
|
|
ctrlc::set_handler(move || on_sigint.store(true, Ordering::Relaxed))?;
|
|
|
|
let (outputs_tx, outputs_rx) = unbounded();
|
|
let writer_stop = Arc::clone(&stop);
|
|
let writer = thread::spawn(move || {
|
|
let written = write(&output, outputs_rx);
|
|
// A writer that failed stops the rest: nothing it reads is kept.
|
|
writer_stop.store(true, Ordering::Relaxed);
|
|
written
|
|
});
|
|
// SIGINT (and SIGTERM, SIGHUP) stops the orchestrator and the
|
|
// connections; the book threads stop once every connection has, and the
|
|
// writer flushes once they all have, even after a panic upstream (the run
|
|
// still fails then).
|
|
let failed = core::orchestrator::run(Arc::new(exchange), outputs_tx, &stop);
|
|
let written = writer
|
|
.join()
|
|
.map_err(|_| anyhow::anyhow!("the writer thread panicked"))?;
|
|
log!("output closed");
|
|
written?;
|
|
anyhow::ensure!(failed == 0, "{failed} threads failed");
|
|
Ok(())
|
|
}
|
|
|
|
/// The writer thread: writes outputs in arrival order until every book
|
|
/// thread is gone, then flushes the last chunk.
|
|
// ponytail: the chunk being filled lives only in memory until it is full, so
|
|
// a hard crash loses it; flushing more often pads every flush to 64 MiB.
|
|
// Kept as a ceiling in M4 (roadmap/04): writing to a file of our own and
|
|
// copying it into chunks would lift it.
|
|
fn write(path: &str, outputs: Receiver<Output>) -> anyhow::Result<()> {
|
|
let mut writer = AlignedSbeWriter::create(path, CHUNK)?;
|
|
for output in outputs {
|
|
match output {
|
|
Output::Snapshot(snapshot) => writer.write_msg(&snapshot)?,
|
|
Output::Delta(delta) => writer.write_msg(&delta)?,
|
|
}
|
|
}
|
|
writer.flush()
|
|
}
|