feat(polymarket): hash-verified order books for a subset of symbols

Collects the order books of Polymarket's 20 busiest markets (40 symbols)
into the template's SBE format: a Delta per level change, plus Snapshots on
subscribe, every minute, and whenever a fresh book corrects ours. Every
change is checked against Polymarket's own book hash; a failed check
resyncs the symbol on the open socket, and the log counts losses apart from
the few ms our own resync hides. Runs until SIGINT, which closes the socket
and flushes the output; reconnects when the server drops it. A `reconcile`
binary checks an output file against itself.
This commit is contained in:
pascal committed 2026-11-04 17:10:17 +01:00
1 parent 0e687e7eba
commit adb7fee8d8
22 files changed
+6423 -84

No files matched your search

+76 -1
View File
@@ -1,5 +1,80 @@
//! The collector: three threads joined by channels (roadmap/02):
//! socket (read, stamp) → book (parse, fold, check, resync) → writer (SBE).
//! Usage: `engineering-challenge [OUTPUT]`, OUTPUT defaulting to stdout; logs
//! go to stderr.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;
use crossbeam::channel::{Receiver, unbounded};
use engineering_challenge::util::AlignedSbeWriter;
/// Prints a line to stderr, prefixed with the UTC time.
macro_rules! log {
($($arg:tt)*) => {
eprintln!("{} {}", chrono::Utc::now().format("%H:%M:%S%.3f"), format_args!($($arg)*))
};
}
mod polymarket;
use engineering_challenge::book::Output;
use polymarket::feed::Feed;
/// `AlignedSbeReader::open` reads nothing else.
const CHUNK: usize = 64 << 20;
const MARKETS: u32 = 20;
fn main() -> anyhow::Result<()> {
polymarket::run_spike()
let output = std::env::args()
.nth(1)
.unwrap_or_else(|| "/dev/stdout".to_string());
let markets = polymarket::top_markets(MARKETS)?;
let feed = Feed::new(&markets);
let tokens = feed.tokens();
let stop = Arc::new(AtomicBool::new(false));
let on_sigint = Arc::clone(&stop);
ctrlc::set_handler(move || on_sigint.store(true, Ordering::Relaxed))?;
let (frames_tx, frames_rx) = unbounded();
let (resyncs_tx, resyncs_rx) = unbounded();
let (outputs_tx, outputs_rx) = unbounded();
// Each stage stops when the one before it does, so SIGINT only reaches the
// socket thread; the writer flushes whatever came before.
let socket =
thread::spawn(move || polymarket::socket::run(&tokens, &frames_tx, &resyncs_rx, &stop));
let book =
thread::spawn(move || polymarket::feed::run(feed, frames_rx, outputs_tx, resyncs_tx));
let writer = thread::spawn(move || write(&output, outputs_rx));
// Every thread is joined, so the writer flushes even after a panic
// upstream; the run still fails then.
let socket_panicked = socket.join().is_err();
let book_panicked = book.join().is_err();
let written = writer
.join()
.map_err(|_| anyhow::anyhow!("the writer thread panicked"))?;
log!("output closed");
written?;
anyhow::ensure!(!socket_panicked, "the socket thread panicked");
anyhow::ensure!(!book_panicked, "the book thread panicked");
Ok(())
}
/// The writer thread: writes outputs in arrival order until the book thread
/// is gone, then flushes the last chunk.
// ponytail: the chunk being filled lives only in memory until it is full, so
// a hard crash loses it (hours of data); flushing more often pads every flush
// to 64 MiB. Taken up in M4.
fn write(path: &str, outputs: Receiver<Output>) -> anyhow::Result<()> {
let mut writer = AlignedSbeWriter::create(path, CHUNK)?;
for output in outputs {
match output {
Output::Snapshot(snapshot) => writer.write_msg(&snapshot)?,
Output::Delta(delta) => writer.write_msg(&delta)?,
}
}
writer.flush()
}