Archived
feat(polymarket): hash-verified order books for a subset of symbols
Collects the order books of Polymarket's 20 busiest markets (40 symbols) into the template's SBE format: a Delta per level change, plus Snapshots on subscribe, every minute, and whenever a fresh book corrects ours. Every change is checked against Polymarket's own book hash; a failed check resyncs the symbol on the open socket, and the log counts losses apart from the few ms our own resync hides. Runs until SIGINT, which closes the socket and flushes the output; reconnects when the server drops it. A `reconcile` binary checks an output file against itself.
This commit is contained in:
22 files changed
+6423
-84
No files matched your search
@@ -0,0 +1,24 @@
|
||||
//! `reconcile OUTPUT`: runs the reconciliation check on a collector's output
|
||||
//! and fails if any Snapshot disagrees with the Deltas before it.
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let path = std::env::args()
|
||||
.nth(1)
|
||||
.ok_or_else(|| anyhow::anyhow!("usage: reconcile OUTPUT"))?;
|
||||
let report = engineering_challenge::reconcile::reconcile(path)?;
|
||||
println!(
|
||||
"{} snapshots, {} deltas, {} orphan deltas, {} mismatches",
|
||||
report.snapshots,
|
||||
report.deltas,
|
||||
report.orphan_deltas,
|
||||
report.mismatches.len()
|
||||
);
|
||||
for (symbol, received_at) in &report.mismatches {
|
||||
println!("mismatch: {symbol} at {received_at}");
|
||||
}
|
||||
anyhow::ensure!(
|
||||
report.mismatches.is_empty() && report.orphan_deltas == 0,
|
||||
"the output doesn't reconcile"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
//! A symbol's book, the exact decimal ↔ fixed-point conversion of its levels,
|
||||
//! and the output messages built from it.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::Side;
|
||||
use crate::util::dtos::challenge_orderbook::{
|
||||
PriceLevelUpdateDomain, SnapshotAsksEntryDomain, SnapshotBidsEntryDomain, SnapshotDomain,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
/// The schema's fixed point: 1_000_000 = 1 USDC, or 1 share.
|
||||
const SCALE: i64 = 1_000_000;
|
||||
const DECIMALS: usize = 6;
|
||||
|
||||
/// What the book thread hands the writer thread.
|
||||
#[derive(Debug)]
|
||||
pub enum Output {
|
||||
Snapshot(SnapshotDomain),
|
||||
Delta(PriceLevelUpdateDomain),
|
||||
}
|
||||
|
||||
/// Parses a non-negative decimal string (`"0.77"`, `"26350.28"`, `"5"`)
|
||||
/// exactly, never through a float. `None` if it isn't one, has more than 6
|
||||
/// decimals, or overflows.
|
||||
pub fn parse_fixed(text: &str) -> Option<i64> {
|
||||
let (integer, fraction) = text.split_once('.').unwrap_or((text, ""));
|
||||
let digits = |s: &str| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit());
|
||||
if !digits(integer) || (text.contains('.') && !digits(fraction)) || fraction.len() > DECIMALS {
|
||||
return None;
|
||||
}
|
||||
let fraction = if fraction.is_empty() {
|
||||
0
|
||||
} else {
|
||||
format!("{fraction:0<DECIMALS$}").parse::<i64>().ok()?
|
||||
};
|
||||
integer
|
||||
.parse::<i64>()
|
||||
.ok()?
|
||||
.checked_mul(SCALE)?
|
||||
.checked_add(fraction)
|
||||
}
|
||||
|
||||
/// The canonical decimal for a fixed-point value: no trailing zeros, no `.`
|
||||
/// for an integer. The inverse of [`parse_fixed`] on canonical input.
|
||||
pub fn format_fixed(value: i64) -> String {
|
||||
let (integer, fraction) = (value / SCALE, value % SCALE);
|
||||
if fraction == 0 {
|
||||
integer.to_string()
|
||||
} else {
|
||||
format!("{integer}.{fraction:0DECIMALS$}")
|
||||
.trim_end_matches('0')
|
||||
.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Bids and asks, each price → absolute quantity, in fixed point.
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||
pub struct Book {
|
||||
pub bids: BTreeMap<i64, i64>,
|
||||
pub asks: BTreeMap<i64, i64>,
|
||||
}
|
||||
|
||||
impl Book {
|
||||
/// Sets a level to its new absolute quantity, 0 removing it, and returns
|
||||
/// the quantity it replaced (0 if there was none).
|
||||
pub fn set(&mut self, side: Side, price: i64, quantity: i64) -> i64 {
|
||||
let levels = match side {
|
||||
Side::Buy => &mut self.bids,
|
||||
Side::Sell => &mut self.asks,
|
||||
};
|
||||
let previous = if quantity == 0 {
|
||||
levels.remove(&price)
|
||||
} else {
|
||||
levels.insert(price, quantity)
|
||||
};
|
||||
previous.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Both sides in ascending price order.
|
||||
pub fn snapshot(
|
||||
&self,
|
||||
symbol: &str,
|
||||
received_at: DateTime<Utc>,
|
||||
exchange_timestamp: DateTime<Utc>,
|
||||
) -> Output {
|
||||
Output::Snapshot(SnapshotDomain {
|
||||
received_at,
|
||||
exchange_timestamp,
|
||||
bids: self
|
||||
.bids
|
||||
.iter()
|
||||
.map(|(&price_level, &quantity)| SnapshotBidsEntryDomain {
|
||||
price_level,
|
||||
quantity,
|
||||
})
|
||||
.collect(),
|
||||
asks: self
|
||||
.asks
|
||||
.iter()
|
||||
.map(|(&price_level, &quantity)| SnapshotAsksEntryDomain {
|
||||
price_level,
|
||||
quantity,
|
||||
})
|
||||
.collect(),
|
||||
symbol: symbol.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_exactly() {
|
||||
assert_eq!(parse_fixed("0"), Some(0));
|
||||
assert_eq!(parse_fixed("5"), Some(5_000_000));
|
||||
assert_eq!(parse_fixed("0.001"), Some(1_000));
|
||||
assert_eq!(parse_fixed("26350.28"), Some(26_350_280_000));
|
||||
assert_eq!(parse_fixed("0.123456"), Some(123_456));
|
||||
// A float would turn this into ...0.099999...
|
||||
assert_eq!(parse_fixed("4351919.1"), Some(4_351_919_100_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_what_is_not_an_exact_decimal() {
|
||||
for text in [
|
||||
"",
|
||||
".",
|
||||
"5.",
|
||||
".5",
|
||||
"-1",
|
||||
"1e3",
|
||||
"0.1234567",
|
||||
" 1",
|
||||
"1,5",
|
||||
"99999999999999",
|
||||
] {
|
||||
assert_eq!(parse_fixed(text), None, "{text:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formats_canonically() {
|
||||
for text in [
|
||||
"0",
|
||||
"5",
|
||||
"0.001",
|
||||
"0.5",
|
||||
"26350.28",
|
||||
"4351919.06",
|
||||
"0.123456",
|
||||
] {
|
||||
assert_eq!(format_fixed(parse_fixed(text).unwrap()), text);
|
||||
}
|
||||
assert_eq!(format_fixed(parse_fixed("0.50").unwrap()), "0.5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zero_quantity_removes_the_level() {
|
||||
let mut book = Book::default();
|
||||
assert_eq!(book.set(Side::Buy, 500_000, 10), 0);
|
||||
assert_eq!(book.set(Side::Buy, 500_000, 20), 10);
|
||||
book.set(Side::Sell, 600_000, 5);
|
||||
assert_eq!(book.bids, BTreeMap::from([(500_000, 20)]));
|
||||
assert_eq!(book.set(Side::Buy, 500_000, 0), 20);
|
||||
assert_eq!(book.set(Side::Buy, 400_000, 0), 0);
|
||||
assert!(book.bids.is_empty());
|
||||
assert_eq!(book.asks.len(), 1);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
pub mod book;
|
||||
pub mod reconcile;
|
||||
pub mod util;
|
||||
|
||||
mod side;
|
||||
|
||||
+76
-1
@@ -1,5 +1,80 @@
|
||||
//! The collector: three threads joined by channels (roadmap/02):
|
||||
//! socket (read, stamp) → book (parse, fold, check, resync) → writer (SBE).
|
||||
//! Usage: `engineering-challenge [OUTPUT]`, OUTPUT defaulting to stdout; logs
|
||||
//! go to stderr.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::thread;
|
||||
|
||||
use crossbeam::channel::{Receiver, unbounded};
|
||||
use engineering_challenge::util::AlignedSbeWriter;
|
||||
|
||||
/// Prints a line to stderr, prefixed with the UTC time.
|
||||
macro_rules! log {
|
||||
($($arg:tt)*) => {
|
||||
eprintln!("{} {}", chrono::Utc::now().format("%H:%M:%S%.3f"), format_args!($($arg)*))
|
||||
};
|
||||
}
|
||||
|
||||
mod polymarket;
|
||||
|
||||
use engineering_challenge::book::Output;
|
||||
use polymarket::feed::Feed;
|
||||
|
||||
/// `AlignedSbeReader::open` reads nothing else.
|
||||
const CHUNK: usize = 64 << 20;
|
||||
const MARKETS: u32 = 20;
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
polymarket::run_spike()
|
||||
let output = std::env::args()
|
||||
.nth(1)
|
||||
.unwrap_or_else(|| "/dev/stdout".to_string());
|
||||
let markets = polymarket::top_markets(MARKETS)?;
|
||||
let feed = Feed::new(&markets);
|
||||
let tokens = feed.tokens();
|
||||
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let on_sigint = Arc::clone(&stop);
|
||||
ctrlc::set_handler(move || on_sigint.store(true, Ordering::Relaxed))?;
|
||||
|
||||
let (frames_tx, frames_rx) = unbounded();
|
||||
let (resyncs_tx, resyncs_rx) = unbounded();
|
||||
let (outputs_tx, outputs_rx) = unbounded();
|
||||
// Each stage stops when the one before it does, so SIGINT only reaches the
|
||||
// socket thread; the writer flushes whatever came before.
|
||||
let socket =
|
||||
thread::spawn(move || polymarket::socket::run(&tokens, &frames_tx, &resyncs_rx, &stop));
|
||||
let book =
|
||||
thread::spawn(move || polymarket::feed::run(feed, frames_rx, outputs_tx, resyncs_tx));
|
||||
let writer = thread::spawn(move || write(&output, outputs_rx));
|
||||
|
||||
// Every thread is joined, so the writer flushes even after a panic
|
||||
// upstream; the run still fails then.
|
||||
let socket_panicked = socket.join().is_err();
|
||||
let book_panicked = book.join().is_err();
|
||||
let written = writer
|
||||
.join()
|
||||
.map_err(|_| anyhow::anyhow!("the writer thread panicked"))?;
|
||||
log!("output closed");
|
||||
written?;
|
||||
anyhow::ensure!(!socket_panicked, "the socket thread panicked");
|
||||
anyhow::ensure!(!book_panicked, "the book thread panicked");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The writer thread: writes outputs in arrival order until the book thread
|
||||
/// is gone, then flushes the last chunk.
|
||||
// ponytail: the chunk being filled lives only in memory until it is full, so
|
||||
// a hard crash loses it (hours of data); flushing more often pads every flush
|
||||
// to 64 MiB. Taken up in M4.
|
||||
fn write(path: &str, outputs: Receiver<Output>) -> anyhow::Result<()> {
|
||||
let mut writer = AlignedSbeWriter::create(path, CHUNK)?;
|
||||
for output in outputs {
|
||||
match output {
|
||||
Output::Snapshot(snapshot) => writer.write_msg(&snapshot)?,
|
||||
Output::Delta(delta) => writer.write_msg(&delta)?,
|
||||
}
|
||||
}
|
||||
writer.flush()
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
//! Connectivity spike: connect to Polymarket's public market channel, subscribe
|
||||
//! to one market, and log every raw frame with its receive time.
|
||||
|
||||
use chrono::{SecondsFormat, Utc};
|
||||
use tungstenite::Message;
|
||||
|
||||
const MARKET_CHANNEL_URL: &str = "wss://ws-subscriptions-clob.polymarket.com/ws/market";
|
||||
|
||||
/// Both tokens (Yes, No) of one market, hardcoded for the spike (discovery comes
|
||||
/// in M3): "2026 Balance of Power: D Senate, D House", which closes 2026-11-04,
|
||||
/// so pick a fresh market to rerun this.
|
||||
const SUBSCRIBE_MESSAGE: &str = r#"{"type":"market","assets_ids":["34722410608062854697106861099776685947172185964394483545370684749662285977831","24483563228660028121020430722896740519524131851526579619586176931221901842798"]}"#;
|
||||
|
||||
/// Connects, sends [`SUBSCRIBE_MESSAGE`], and logs frames until the server
|
||||
/// closes the connection.
|
||||
pub fn run_spike() -> anyhow::Result<()> {
|
||||
let (mut socket, _response) = tungstenite::connect(MARKET_CHANNEL_URL)?;
|
||||
socket.send(Message::text(SUBSCRIBE_MESSAGE))?;
|
||||
|
||||
loop {
|
||||
let message = socket.read()?;
|
||||
// Stamp as soon as the frame is off the wire, before any processing.
|
||||
let received_at = Utc::now().to_rfc3339_opts(SecondsFormat::Micros, true);
|
||||
match message {
|
||||
Message::Text(text) => println!("{received_at} {text}"),
|
||||
Message::Close(frame) => {
|
||||
println!("{received_at} close {frame:?}");
|
||||
return Ok(());
|
||||
}
|
||||
other => println!("{received_at} {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,76 @@
|
||||
//! Polymarket: which markets to track (Gamma), the market-channel socket, and
|
||||
//! the feed that turns its frames into books and output.
|
||||
|
||||
pub mod feed;
|
||||
pub mod socket;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
const GAMMA_MARKETS_URL: &str = "https://gamma-api.polymarket.com/markets";
|
||||
|
||||
/// A market's symbols and the two inputs of their book hash that the
|
||||
/// websocket doesn't carry (docs/notes.md).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Market {
|
||||
/// Its token ids, one per outcome: each is a symbol.
|
||||
pub tokens: Vec<String>,
|
||||
/// Exactly as the hash expects it, e.g. `"5"`.
|
||||
pub min_order_size: String,
|
||||
pub neg_risk: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct GammaMarket {
|
||||
/// A JSON array inside a string.
|
||||
clob_token_ids: String,
|
||||
order_min_size: serde_json::Number,
|
||||
neg_risk: bool,
|
||||
enable_order_book: bool,
|
||||
}
|
||||
|
||||
/// The `limit` live markets with the most 24 h volume, keeping those with an
|
||||
/// order book. Fixed for the run: discovery is M3.
|
||||
pub fn top_markets(limit: u32) -> anyhow::Result<Vec<Market>> {
|
||||
let body = ureq::get(GAMMA_MARKETS_URL)
|
||||
.query("limit", limit.to_string())
|
||||
.query("active", "true")
|
||||
.query("closed", "false")
|
||||
.query("order", "volume24hr")
|
||||
.query("ascending", "false")
|
||||
.call()?
|
||||
.body_mut()
|
||||
.read_to_string()?;
|
||||
parse_markets(&body)
|
||||
}
|
||||
|
||||
fn parse_markets(body: &str) -> anyhow::Result<Vec<Market>> {
|
||||
let markets: Vec<GammaMarket> = serde_json::from_str(body)?;
|
||||
markets
|
||||
.into_iter()
|
||||
.filter(|market| market.enable_order_book)
|
||||
.map(|market| {
|
||||
Ok(Market {
|
||||
tokens: serde_json::from_str(&market.clob_token_ids)?,
|
||||
min_order_size: market.order_min_size.to_string(),
|
||||
neg_risk: market.neg_risk,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn fixture_markets() -> Vec<Market> {
|
||||
parse_markets(include_str!("../../tests/fixtures/gamma-markets.json")).unwrap()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn parses_gamma_markets() {
|
||||
let markets = super::fixture_markets();
|
||||
assert_eq!(markets.len(), 20);
|
||||
assert!(markets.iter().all(|market| market.tokens.len() == 2));
|
||||
assert_eq!(markets[0].min_order_size, "5");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
//! The socket thread: reads the market channel and stamps each frame's receive
|
||||
//! time, and nothing else. Parsing is the book thread's.
|
||||
|
||||
use std::io::ErrorKind;
|
||||
use std::net::{TcpStream, ToSocketAddrs};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use crossbeam::channel::{Receiver, Sender};
|
||||
use serde_json::json;
|
||||
use tungstenite::stream::MaybeTlsStream;
|
||||
use tungstenite::{Message, Utf8Bytes, WebSocket};
|
||||
|
||||
const MARKET_CHANNEL_HOST: &str = "ws-subscriptions-clob.polymarket.com";
|
||||
const MARKET_CHANNEL_URL: &str = "wss://ws-subscriptions-clob.polymarket.com/ws/market";
|
||||
/// For the TCP connect and the TLS + websocket handshake, and for every send.
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
/// How often the loop wakes when no frame comes: tungstenite has no `split`,
|
||||
/// so the reads and the writes below (PING, resyncs, close) share this thread.
|
||||
const READ_TIMEOUT: Duration = Duration::from_millis(200);
|
||||
const PING_EVERY: Duration = Duration::from_secs(10);
|
||||
const RECONNECT_AFTER: Duration = Duration::from_secs(1);
|
||||
const CLOSE_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// A frame's text with its receive time.
|
||||
pub type Frame = (DateTime<Utc>, Utf8Bytes);
|
||||
|
||||
type Socket = WebSocket<MaybeTlsStream<TcpStream>>;
|
||||
|
||||
/// Runs until `stop` is set or the book thread is gone, reconnecting (and
|
||||
/// subscribing every token again) whenever the connection drops. `resyncs`
|
||||
/// carries the tokens to unsubscribe and subscribe again.
|
||||
pub fn run(
|
||||
tokens: &[String],
|
||||
frames: &Sender<Frame>,
|
||||
resyncs: &Receiver<String>,
|
||||
stop: &AtomicBool,
|
||||
) {
|
||||
while !stop.load(Ordering::Relaxed) {
|
||||
match connect(tokens) {
|
||||
Ok(mut socket) => match read(&mut socket, frames, resyncs, stop) {
|
||||
Ok(Ended::Stopped) => return close(&mut socket, frames),
|
||||
Ok(Ended::BookThreadGone) => return,
|
||||
Err(error) => log!("polymarket: connection lost: {error}"),
|
||||
},
|
||||
Err(error) => log!("polymarket: connect failed: {error}"),
|
||||
}
|
||||
thread::sleep(RECONNECT_AFTER);
|
||||
}
|
||||
}
|
||||
|
||||
enum Ended {
|
||||
Stopped,
|
||||
BookThreadGone,
|
||||
}
|
||||
|
||||
fn connect(tokens: &[String]) -> anyhow::Result<Socket> {
|
||||
// Timed, unlike `tungstenite::connect`, so a stalled connect or handshake
|
||||
// can't keep SIGINT waiting. The name lookup itself has no timeout.
|
||||
let address = (MARKET_CHANNEL_HOST, 443)
|
||||
.to_socket_addrs()?
|
||||
.next()
|
||||
.ok_or_else(|| anyhow::anyhow!("{MARKET_CHANNEL_HOST} has no address"))?;
|
||||
let stream = TcpStream::connect_timeout(&address, CONNECT_TIMEOUT)?;
|
||||
stream.set_read_timeout(Some(CONNECT_TIMEOUT))?;
|
||||
stream.set_write_timeout(Some(CONNECT_TIMEOUT))?;
|
||||
let (mut socket, _response) = tungstenite::client_tls(MARKET_CHANNEL_URL, stream)
|
||||
.map_err(|error| anyhow::anyhow!("handshake failed: {error}"))?;
|
||||
let stream = match socket.get_ref() {
|
||||
MaybeTlsStream::Plain(stream) => stream,
|
||||
MaybeTlsStream::Rustls(stream) => stream.get_ref(),
|
||||
_ => anyhow::bail!("unexpected TLS stream"),
|
||||
};
|
||||
stream.set_read_timeout(Some(READ_TIMEOUT))?;
|
||||
socket.send(Message::text(
|
||||
json!({"type": "market", "assets_ids": tokens}).to_string(),
|
||||
))?;
|
||||
log!("polymarket: connected, subscribed {} symbols", tokens.len());
|
||||
Ok(socket)
|
||||
}
|
||||
|
||||
fn read(
|
||||
socket: &mut Socket,
|
||||
frames: &Sender<Frame>,
|
||||
resyncs: &Receiver<String>,
|
||||
stop: &AtomicBool,
|
||||
) -> anyhow::Result<Ended> {
|
||||
let mut last_ping = Instant::now();
|
||||
loop {
|
||||
let message = socket.read();
|
||||
// Stamp as soon as the frame is off the wire, before any processing.
|
||||
let received_at = Utc::now();
|
||||
match message {
|
||||
Ok(Message::Text(text)) => {
|
||||
if frames.send((received_at, text)).is_err() {
|
||||
return Ok(Ended::BookThreadGone);
|
||||
}
|
||||
}
|
||||
Ok(Message::Close(frame)) => anyhow::bail!("closed by the server: {frame:?}"),
|
||||
Ok(_) => {}
|
||||
Err(tungstenite::Error::Io(error)) if timed_out(&error) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
return Ok(Ended::Stopped);
|
||||
}
|
||||
if last_ping.elapsed() >= PING_EVERY {
|
||||
socket.send(Message::text("PING"))?;
|
||||
last_ping = Instant::now();
|
||||
}
|
||||
for token in resyncs.try_iter() {
|
||||
// Subscribing a token already subscribed sends nothing back but
|
||||
// `NO NEW ASSETS`: unsubscribing first makes the server send a
|
||||
// fresh book.
|
||||
socket.send(Message::text(
|
||||
json!({"assets_ids": [&token], "operation": "unsubscribe"}).to_string(),
|
||||
))?;
|
||||
socket.send(Message::text(
|
||||
json!({"assets_ids": [&token], "operation": "subscribe"}).to_string(),
|
||||
))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Closes the connection, forwarding what still arrives until the server
|
||||
/// confirms or [`CLOSE_TIMEOUT`] passes.
|
||||
fn close(socket: &mut Socket, frames: &Sender<Frame>) {
|
||||
if let Err(error) = socket.close(None) {
|
||||
return log!("polymarket: close failed: {error}");
|
||||
}
|
||||
let deadline = Instant::now() + CLOSE_TIMEOUT;
|
||||
while Instant::now() < deadline {
|
||||
let message = socket.read();
|
||||
let received_at = Utc::now();
|
||||
match message {
|
||||
Ok(Message::Text(text)) => _ = frames.send((received_at, text)),
|
||||
Ok(_) => {}
|
||||
Err(tungstenite::Error::Io(error)) if timed_out(&error) => {}
|
||||
// ConnectionClosed: the close handshake is done.
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
log!("polymarket: socket closed");
|
||||
}
|
||||
|
||||
/// A read timeout shows up as `WouldBlock` on macOS and `TimedOut` elsewhere;
|
||||
/// tungstenite keeps a partly read frame, so the next read resumes it.
|
||||
fn timed_out(error: &std::io::Error) -> bool {
|
||||
matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut)
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
//! Checks an output file against itself: per symbol, the Deltas written since
|
||||
//! a Snapshot, applied to it, must give the next Snapshot. A mismatch is a loss
|
||||
//! or a resync gap (the fresh book differs), each logged by the collector.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::book::Book;
|
||||
use crate::util::dtos::challenge_orderbook::{AnyMessage, PriceLevelUpdateDomain, SnapshotDomain};
|
||||
use crate::util::{AlignedSbeReader, OrderbookSchema};
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Report {
|
||||
pub snapshots: usize,
|
||||
pub deltas: usize,
|
||||
/// Deltas for a symbol with no Snapshot yet: the collector never writes one.
|
||||
pub orphan_deltas: usize,
|
||||
/// Each Snapshot that didn't match its symbol's book: (symbol, its `received_at`).
|
||||
pub mismatches: Vec<(String, chrono::DateTime<chrono::Utc>)>,
|
||||
}
|
||||
|
||||
pub fn reconcile(path: impl AsRef<Path>) -> anyhow::Result<Report> {
|
||||
let mut reader = AlignedSbeReader::<OrderbookSchema>::open(path)?;
|
||||
let mut books: HashMap<String, Book> = HashMap::new();
|
||||
let mut report = Report::default();
|
||||
while let Some(message) = reader.read_msg()? {
|
||||
match message {
|
||||
AnyMessage::Snapshot(decoder) => {
|
||||
let snapshot = SnapshotDomain::try_from_decoder(decoder)?;
|
||||
report.snapshots += 1;
|
||||
let book = Book {
|
||||
bids: snapshot
|
||||
.bids
|
||||
.iter()
|
||||
.map(|level| (level.price_level, level.quantity))
|
||||
.collect(),
|
||||
asks: snapshot
|
||||
.asks
|
||||
.iter()
|
||||
.map(|level| (level.price_level, level.quantity))
|
||||
.collect(),
|
||||
};
|
||||
if books
|
||||
.get(&snapshot.symbol)
|
||||
.is_some_and(|ours| *ours != book)
|
||||
{
|
||||
report
|
||||
.mismatches
|
||||
.push((snapshot.symbol.clone(), snapshot.received_at));
|
||||
}
|
||||
books.insert(snapshot.symbol, book);
|
||||
}
|
||||
AnyMessage::PriceLevelUpdate(decoder) => {
|
||||
let delta = PriceLevelUpdateDomain::try_from_decoder(decoder)?;
|
||||
report.deltas += 1;
|
||||
match books.get_mut(&delta.symbol) {
|
||||
Some(book) => _ = book.set(delta.side, delta.price_level, delta.quantity),
|
||||
None => report.orphan_deltas += 1,
|
||||
}
|
||||
}
|
||||
_ => anyhow::bail!("unknown message in the output"),
|
||||
}
|
||||
}
|
||||
Ok(report)
|
||||
}
|
||||
Reference in new issue
Block a user