Archived
feat(polymarket)!: two connections per market, first one wins
Every live market is subscribed on two connections chosen at random, and its book takes each change from whichever connection delivers it first: the book hash says whether a change is a duplicate, the next one, or one to hold until the change before it arrives. A change one connection misses costs nothing, each Delta carries the earliest receive time, and a connection that drops is covered by the other. When a change fits on no connection in time, the book follows one connection unchecked and asks the other for a fresh book. Connections now decode and route events to 16 book threads, independent of them; a move leaves its old connection only once the new one delivers, so it costs no change; a connection closed with 1013 twice within ten minutes is split. reconcile reads the collector's log and fails only on a mismatch no cause line explains. SIGTERM and SIGHUP stop the collector cleanly. Breaking: lanes are gone (connections and book threads are separate), and reconcile takes the log as a second argument.
This commit is contained in:
24 files changed
+8984
-1214
No files matched your search
+17
-9
@@ -1,11 +1,14 @@
|
||||
//! `reconcile OUTPUT`: runs the reconciliation check on a collector's output
|
||||
//! and fails if any Snapshot disagrees with the Deltas before it.
|
||||
//! `reconcile OUTPUT [LOG]`: runs the reconciliation check on a collector's
|
||||
//! output and fails on an orphan Delta, or on a Snapshot that disagrees with
|
||||
//! the Deltas before it, unless the collector's log explains it.
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let path = std::env::args()
|
||||
.nth(1)
|
||||
.ok_or_else(|| anyhow::anyhow!("usage: reconcile OUTPUT"))?;
|
||||
let report = engineering_challenge::reconcile::reconcile(path)?;
|
||||
let mut args = std::env::args().skip(1);
|
||||
let path = args
|
||||
.next()
|
||||
.ok_or_else(|| anyhow::anyhow!("usage: reconcile OUTPUT [LOG]"))?;
|
||||
let log = args.next().map(std::path::PathBuf::from);
|
||||
let report = engineering_challenge::reconcile::reconcile(path, log.as_deref())?;
|
||||
println!(
|
||||
"{} snapshots, {} deltas, {} orphan deltas, {} mismatches",
|
||||
report.snapshots,
|
||||
@@ -13,11 +16,16 @@ fn main() -> anyhow::Result<()> {
|
||||
report.orphan_deltas,
|
||||
report.mismatches.len()
|
||||
);
|
||||
for (symbol, received_at) in &report.mismatches {
|
||||
println!("mismatch: {symbol} at {received_at}");
|
||||
let mut explained: Vec<_> = report.explained.iter().collect();
|
||||
explained.sort_by_key(|(cause, _)| cause.to_string());
|
||||
for (cause, count) in explained {
|
||||
println!("explained by {cause}: {count}");
|
||||
}
|
||||
for (symbol, received_at) in &report.unexplained {
|
||||
println!("unexplained: {symbol} at {received_at}");
|
||||
}
|
||||
anyhow::ensure!(
|
||||
report.mismatches.is_empty() && report.orphan_deltas == 0,
|
||||
report.unexplained.is_empty() && report.orphan_deltas == 0,
|
||||
"the output doesn't reconcile"
|
||||
);
|
||||
Ok(())
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
//! Why a Snapshot outside the minute was written: one log line per Snapshot,
|
||||
//! written by the collector and read back by `reconcile`, which pairs it with
|
||||
//! a mismatch by symbol and receive time.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
/// What a Snapshot that may differ from the book before it stands for.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum Cause {
|
||||
/// A fresh book differed from ours: changes no connection delivered.
|
||||
Loss,
|
||||
/// A fresh book differed from ours before any change of the symbol was
|
||||
/// verified: changes our own subscribe kept from us.
|
||||
SubscriptionGap,
|
||||
/// The connection followed changed: the new one's book, which may hold
|
||||
/// changes the old one never delivered.
|
||||
Switch,
|
||||
}
|
||||
|
||||
impl Cause {
|
||||
const ALL: [Cause; 3] = [Cause::Loss, Cause::SubscriptionGap, Cause::Switch];
|
||||
|
||||
fn name(self) -> &'static str {
|
||||
match self {
|
||||
Cause::Loss => "loss",
|
||||
Cause::SubscriptionGap => "subscription gap",
|
||||
Cause::Switch => "switch",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Cause {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.write_str(self.name())
|
||||
}
|
||||
}
|
||||
|
||||
const MARK: &str = "Snapshot of ";
|
||||
|
||||
/// The line's text: `Snapshot of <symbol> at <receive time, µs>: <cause>`,
|
||||
/// then any detail for a reader.
|
||||
pub fn line(symbol: &str, received_at: DateTime<Utc>, cause: Cause, detail: &str) -> String {
|
||||
let at = received_at.timestamp_micros();
|
||||
format!("{MARK}{symbol} at {at}: {cause}{detail}")
|
||||
}
|
||||
|
||||
/// The symbol, receive time and cause of a log line written from [`line`],
|
||||
/// wherever it sits in the line.
|
||||
pub fn parse(log_line: &str) -> Option<(String, DateTime<Utc>, Cause)> {
|
||||
let rest = &log_line[log_line.find(MARK)? + MARK.len()..];
|
||||
let (symbol, rest) = rest.split_once(" at ")?;
|
||||
let (at, rest) = rest.split_once(": ")?;
|
||||
let received_at = DateTime::from_timestamp_micros(at.parse().ok()?)?;
|
||||
let cause = Cause::ALL
|
||||
.into_iter()
|
||||
.find(|cause| rest.starts_with(cause.name()))?;
|
||||
Some((symbol.to_string(), received_at, cause))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_line_reads_back() {
|
||||
let at = DateTime::from_timestamp_micros(1_794_390_726_365_123).unwrap();
|
||||
for cause in Cause::ALL {
|
||||
let text = line("123", at, cause, " (its time 12:00:00.000)");
|
||||
let logged = format!("12:00:00.001 polymarket: {text}");
|
||||
assert_eq!(parse(&logged), Some(("123".to_string(), at, cause)));
|
||||
}
|
||||
assert_eq!(
|
||||
parse("12:00:00.001 polymarket: connection 3: connected"),
|
||||
None
|
||||
);
|
||||
assert_eq!(parse("Snapshot of 1 at soon: loss"), None);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod book;
|
||||
pub mod cause;
|
||||
pub mod reconcile;
|
||||
pub mod util;
|
||||
|
||||
|
||||
+10
-8
@@ -1,7 +1,7 @@
|
||||
//! The collector (roadmap/03): an orchestrator keeps the tracked markets equal
|
||||
//! to Gamma's live ones and spreads them over lanes, each a connection (read,
|
||||
//! stamp) and a book thread (parse, fold, check, resync), all writing through
|
||||
//! one writer (SBE).
|
||||
//! The collector (roadmap/04): an orchestrator keeps the tracked markets equal
|
||||
//! to Gamma's live ones and places them on connections (read, stamp, decode,
|
||||
//! route), which feed the book threads (fold, check, resync), all writing
|
||||
//! through one writer (SBE).
|
||||
//! Usage: `engineering-challenge [OUTPUT]`, OUTPUT defaulting to stdout; logs
|
||||
//! go to stderr.
|
||||
|
||||
@@ -43,9 +43,10 @@ fn main() -> anyhow::Result<()> {
|
||||
writer_stop.store(true, Ordering::Relaxed);
|
||||
written
|
||||
});
|
||||
// SIGINT stops the orchestrator and the connections; each book thread
|
||||
// stops when its connection does, and the writer flushes once they all
|
||||
// have, even after a panic upstream (the run still fails then).
|
||||
// SIGINT (and SIGTERM, SIGHUP) stops the orchestrator and the
|
||||
// connections; the book threads stop once every connection has, and the
|
||||
// writer flushes once they all have, even after a panic upstream (the run
|
||||
// still fails then).
|
||||
let failed = polymarket::orchestrator::run(outputs_tx, &stop);
|
||||
let written = writer
|
||||
.join()
|
||||
@@ -60,7 +61,8 @@ fn main() -> anyhow::Result<()> {
|
||||
/// thread is gone, then flushes the last chunk.
|
||||
// ponytail: the chunk being filled lives only in memory until it is full, so
|
||||
// a hard crash loses it; flushing more often pads every flush to 64 MiB.
|
||||
// Taken up in M4.
|
||||
// Kept as a ceiling in M4 (roadmap/04): writing to a file of our own and
|
||||
// copying it into chunks would lift it.
|
||||
fn write(path: &str, outputs: Receiver<Output>) -> anyhow::Result<()> {
|
||||
let mut writer = AlignedSbeWriter::create(path, CHUNK)?;
|
||||
for output in outputs {
|
||||
|
||||
+1987
-693
File diff suppressed because it is too large.
Load diff
+37
-2
@@ -1,10 +1,25 @@
|
||||
//! Polymarket: which markets are live (Gamma), the orchestrator that spreads
|
||||
//! them over lanes, and each lane's connection and feed (roadmap/03).
|
||||
//! Polymarket: which markets are live (Gamma), the orchestrator that places
|
||||
//! them on connections, the connections, and the book threads they feed
|
||||
//! (docs/adr/0003-first-one-wins.md).
|
||||
|
||||
pub mod feed;
|
||||
pub mod gamma;
|
||||
pub mod orchestrator;
|
||||
pub mod socket;
|
||||
pub mod wire;
|
||||
|
||||
/// How many book threads own the books. One handled ~30,000 frames a second
|
||||
/// in sprint 02.
|
||||
pub const BOOK_THREADS: usize = 16;
|
||||
|
||||
/// The book thread that owns a market's books, from its `conditionId` (`0x`
|
||||
/// and 64 hex digits, a hash: any 16 of them spread evenly). Computed, so the
|
||||
/// orchestrator and every connection agree without a shared table.
|
||||
pub fn book_thread(condition_id: &str) -> usize {
|
||||
let digits = condition_id.get(2..18).unwrap_or_default();
|
||||
// An id of another shape still gets a thread, always the same one.
|
||||
u64::from_str_radix(digits, 16).unwrap_or(0) as usize % BOOK_THREADS
|
||||
}
|
||||
|
||||
/// A live market: its symbols and the two inputs of their book hash that the
|
||||
/// websocket doesn't carry (docs/notes.md).
|
||||
@@ -23,3 +38,23 @@ pub struct Market {
|
||||
pub(crate) fn fixture_markets() -> Vec<Market> {
|
||||
gamma::parse_markets(include_str!("../../tests/fixtures/gamma-markets.json"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_market_always_gets_the_same_book_thread_and_markets_spread() {
|
||||
let markets = fixture_markets();
|
||||
let threads: std::collections::BTreeSet<usize> = markets
|
||||
.iter()
|
||||
.map(|market| book_thread(&market.condition_id))
|
||||
.collect();
|
||||
assert!(threads.iter().all(|&thread| thread < BOOK_THREADS));
|
||||
// 20 markets over 16 threads: several threads, not one.
|
||||
assert!(threads.len() > 5, "{threads:?}");
|
||||
let id = &markets[0].condition_id;
|
||||
assert_eq!(book_thread(id), book_thread(&id.clone()));
|
||||
assert_eq!(book_thread("not an id"), 0);
|
||||
}
|
||||
}
|
||||
+694
-355
File diff suppressed because it is too large.
Load diff
+152
-26
@@ -1,6 +1,6 @@
|
||||
//! A lane's connection: reads the market channel and stamps each frame's
|
||||
//! receive time, and sends what its book thread asks for (subscriptions,
|
||||
//! resyncs). Parsing is the book thread's.
|
||||
//! A connection: reads the market channel, stamps each frame's receive time,
|
||||
//! decodes it and sends each event to the book thread of its market; and
|
||||
//! sends the exchange what the book threads ask for (subscriptions, resyncs).
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::io::ErrorKind;
|
||||
@@ -13,8 +13,12 @@ use chrono::{DateTime, Utc};
|
||||
use crossbeam::channel::{Receiver, Sender};
|
||||
use serde_json::json;
|
||||
use tungstenite::http::Uri;
|
||||
use tungstenite::protocol::frame::coding::CloseCode;
|
||||
use tungstenite::stream::MaybeTlsStream;
|
||||
use tungstenite::{Message, Utf8Bytes, WebSocket};
|
||||
use tungstenite::{Message, WebSocket};
|
||||
|
||||
use super::book_thread;
|
||||
use super::wire::{self, Arrival, Received};
|
||||
|
||||
pub const MARKET_CHANNEL_URL: &str = "wss://ws-subscriptions-clob.polymarket.com/ws/market";
|
||||
/// For the TCP connect and the TLS + websocket handshake, and for every send.
|
||||
@@ -29,10 +33,7 @@ const CLOSE_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
/// 500 gets no opening books (docs/notes.md).
|
||||
const CHUNK: usize = 500;
|
||||
|
||||
/// A frame's text with its receive time.
|
||||
pub type Frame = (DateTime<Utc>, Utf8Bytes);
|
||||
|
||||
/// What the book thread asks its connection for.
|
||||
/// What a book thread asks a connection for.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Command {
|
||||
Subscribe(Vec<String>),
|
||||
@@ -43,15 +44,23 @@ pub enum Command {
|
||||
|
||||
type Socket = WebSocket<MaybeTlsStream<TcpStream>>;
|
||||
|
||||
/// Runs until `stop` is set or the book thread is gone, reconnecting (and
|
||||
/// subscribing its tokens again) whenever the connection drops.
|
||||
/// Runs until `stop` is set or a book thread is gone, reconnecting (and
|
||||
/// subscribing its tokens again) whenever the connection drops, and telling
|
||||
/// every book thread when it comes up and goes down. `book_threads` holds
|
||||
/// every book thread's channel, by number; `closes` hears of each 1013 close.
|
||||
pub fn run(
|
||||
url: &str,
|
||||
lane: usize,
|
||||
frames: &Sender<Frame>,
|
||||
number: usize,
|
||||
book_threads: &[Sender<Received>],
|
||||
commands: &Receiver<Command>,
|
||||
closes: &Sender<usize>,
|
||||
stop: &AtomicBool,
|
||||
) {
|
||||
let tell = |received: fn(usize) -> Received| {
|
||||
for book_thread in book_threads {
|
||||
_ = book_thread.send(received(number));
|
||||
}
|
||||
};
|
||||
let mut tokens = BTreeSet::new();
|
||||
while !stop.load(Ordering::Relaxed) {
|
||||
// Asked for while disconnected: the subscribe on connect covers them,
|
||||
@@ -62,16 +71,36 @@ pub fn run(
|
||||
match connect(url, &tokens) {
|
||||
Ok(mut socket) => {
|
||||
log!(
|
||||
"polymarket: lane {lane}: connected, {} symbols",
|
||||
"polymarket: connection {number}: connected, {} symbols",
|
||||
tokens.len()
|
||||
);
|
||||
match read(&mut socket, &mut tokens, frames, commands, stop) {
|
||||
Ok(Ended::Stopped) => return close(&mut socket, lane, frames),
|
||||
Ok(Ended::BookThreadGone) => return,
|
||||
Err(error) => log!("polymarket: lane {lane}: connection lost: {error}"),
|
||||
tell(Received::Up);
|
||||
let ended = read(
|
||||
&mut socket,
|
||||
number,
|
||||
&mut tokens,
|
||||
book_threads,
|
||||
commands,
|
||||
closes,
|
||||
stop,
|
||||
);
|
||||
if !matches!(ended, Ok(Ended::Stopped)) {
|
||||
tell(Received::Down);
|
||||
}
|
||||
match ended {
|
||||
Ok(Ended::Stopped) => return close(&mut socket, number, book_threads),
|
||||
Ok(Ended::BookThreadGone) => {
|
||||
// Only a panic ends a book thread before us: its
|
||||
// markets would go unserved, so the run stops.
|
||||
log!("polymarket: connection {number}: a book thread is gone, stopping");
|
||||
return stop.store(true, Ordering::Relaxed);
|
||||
}
|
||||
Err(error) => {
|
||||
log!("polymarket: connection {number}: connection lost: {error}")
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => log!("polymarket: lane {lane}: connect failed: {error}"),
|
||||
Err(error) => log!("polymarket: connection {number}: connect failed: {error}"),
|
||||
}
|
||||
thread::sleep(RECONNECT_AFTER);
|
||||
}
|
||||
@@ -150,11 +179,53 @@ fn connect(url: &str, tokens: &BTreeSet<String>) -> anyhow::Result<Socket> {
|
||||
Ok(socket)
|
||||
}
|
||||
|
||||
/// Decodes a frame and sends each event to the book thread of its market;
|
||||
/// `false` once a book thread is gone.
|
||||
fn route(
|
||||
received_at: DateTime<Utc>,
|
||||
text: &str,
|
||||
number: usize,
|
||||
book_threads: &[Sender<Received>],
|
||||
) -> bool {
|
||||
let events = match wire::decode(text) {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
log!("polymarket: unreadable frame ({error}): {}", truncate(text));
|
||||
return true;
|
||||
}
|
||||
};
|
||||
let bytes = (text.len() / events.len().max(1)) as u64;
|
||||
for event in events {
|
||||
// An event of no market (none we ask for) has no book thread.
|
||||
let Some(thread) = event.market().map(book_thread) else {
|
||||
continue;
|
||||
};
|
||||
let arrival = Arrival {
|
||||
received_at,
|
||||
connection: number,
|
||||
bytes,
|
||||
event,
|
||||
};
|
||||
if book_threads[thread].send(Received::Event(arrival)).is_err() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn truncate(text: &str) -> &str {
|
||||
text.char_indices()
|
||||
.nth(200)
|
||||
.map_or(text, |(end, _)| &text[..end])
|
||||
}
|
||||
|
||||
fn read(
|
||||
socket: &mut Socket,
|
||||
number: usize,
|
||||
tokens: &mut BTreeSet<String>,
|
||||
frames: &Sender<Frame>,
|
||||
book_threads: &[Sender<Received>],
|
||||
commands: &Receiver<Command>,
|
||||
closes: &Sender<usize>,
|
||||
stop: &AtomicBool,
|
||||
) -> anyhow::Result<Ended> {
|
||||
let mut last_ping = Instant::now();
|
||||
@@ -164,11 +235,21 @@ fn read(
|
||||
let received_at = Utc::now();
|
||||
match message {
|
||||
Ok(Message::Text(text)) => {
|
||||
if frames.send((received_at, text)).is_err() {
|
||||
if !route(received_at, text.as_str(), number, book_threads) {
|
||||
return Ok(Ended::BookThreadGone);
|
||||
}
|
||||
}
|
||||
Ok(Message::Close(frame)) => anyhow::bail!("closed by the server: {frame:?}"),
|
||||
Ok(Message::Close(frame)) => {
|
||||
// 1013: the server couldn't send to us fast enough. The
|
||||
// orchestrator splits a connection closed so twice.
|
||||
if frame
|
||||
.as_ref()
|
||||
.is_some_and(|frame| frame.code == CloseCode::Again)
|
||||
{
|
||||
_ = closes.send(number);
|
||||
}
|
||||
anyhow::bail!("closed by the server: {frame:?}")
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(tungstenite::Error::Io(error)) if timed_out(&error) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
@@ -191,16 +272,16 @@ fn read(
|
||||
|
||||
/// Closes the connection, forwarding what still arrives until the server
|
||||
/// confirms or [`CLOSE_TIMEOUT`] passes.
|
||||
fn close(socket: &mut Socket, lane: usize, frames: &Sender<Frame>) {
|
||||
fn close(socket: &mut Socket, number: usize, book_threads: &[Sender<Received>]) {
|
||||
if let Err(error) = socket.close(None) {
|
||||
return log!("polymarket: lane {lane}: close failed: {error}");
|
||||
return log!("polymarket: connection {number}: close failed: {error}");
|
||||
}
|
||||
let deadline = Instant::now() + CLOSE_TIMEOUT;
|
||||
while Instant::now() < deadline {
|
||||
let message = socket.read();
|
||||
let received_at = Utc::now();
|
||||
match message {
|
||||
Ok(Message::Text(text)) => _ = frames.send((received_at, text)),
|
||||
Ok(Message::Text(text)) => _ = route(received_at, text.as_str(), number, book_threads),
|
||||
Ok(_) => {}
|
||||
Err(tungstenite::Error::Io(error)) if timed_out(&error) => {}
|
||||
// ConnectionClosed: the close handshake is done.
|
||||
@@ -263,14 +344,16 @@ mod tests {
|
||||
connections
|
||||
});
|
||||
|
||||
let (frames_tx, _frames_rx) = unbounded();
|
||||
let (events_tx, _events_rx) = unbounded();
|
||||
let (commands_tx, commands_rx) = unbounded();
|
||||
commands_tx
|
||||
.send(Command::Subscribe(tokens.clone()))
|
||||
.unwrap();
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let client_stop = Arc::clone(&stop);
|
||||
let client = thread::spawn(move || run(&url, 0, &frames_tx, &commands_rx, &client_stop));
|
||||
let (closes, _closes_rx) = unbounded();
|
||||
let client =
|
||||
thread::spawn(move || run(&url, 0, &[events_tx], &commands_rx, &closes, &client_stop));
|
||||
|
||||
let connections = server.join().unwrap();
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
@@ -285,6 +368,49 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// The recording's first frame holds the opening books of 40 tokens, 20
|
||||
/// markets: each book goes to its market's book thread, with its share of
|
||||
/// the frame's bytes.
|
||||
#[test]
|
||||
fn a_frame_of_several_markets_reaches_each_markets_book_thread() {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Line {
|
||||
text: String,
|
||||
}
|
||||
let path = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/polymarket-frames.jsonl"
|
||||
);
|
||||
let recording = std::fs::read_to_string(path).unwrap();
|
||||
let line: Line = serde_json::from_str(recording.lines().next().unwrap()).unwrap();
|
||||
let (senders, receivers): (Vec<_>, Vec<_>) = (0..crate::polymarket::BOOK_THREADS)
|
||||
.map(|_| unbounded())
|
||||
.unzip();
|
||||
|
||||
assert!(route(Utc::now(), &line.text, 3, &senders));
|
||||
let mut books = 0;
|
||||
for (thread, receiver) in receivers.iter().enumerate() {
|
||||
for received in receiver.try_iter() {
|
||||
let Received::Event(arrival) = received else {
|
||||
panic!("not an event: {received:?}");
|
||||
};
|
||||
let market = arrival.event.market().unwrap().to_string();
|
||||
assert_eq!(book_thread(&market), thread);
|
||||
assert_eq!(arrival.connection, 3);
|
||||
assert_eq!(arrival.bytes, line.text.len() as u64 / 40);
|
||||
books += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(books, 40);
|
||||
// Neither a keepalive's answer nor an unreadable frame reaches one.
|
||||
assert!(route(Utc::now(), "PONG", 3, &senders));
|
||||
assert!(route(Utc::now(), "not json", 3, &senders));
|
||||
assert!(receivers.iter().all(|receiver| receiver.is_empty()));
|
||||
// A book thread gone stops the connection.
|
||||
drop(receivers);
|
||||
assert!(!route(Utc::now(), &line.text, 3, &senders));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commands_update_the_token_set_and_say_what_to_send() {
|
||||
let mut tokens = BTreeSet::new();
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
//! The market channel's messages, as the exchange writes them, and how a
|
||||
//! frame's text becomes events (docs/notes.md).
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::Deserialize;
|
||||
|
||||
/// What a connection sends a book thread.
|
||||
// Nearly every message is an event: boxing it to shrink the two rare markers
|
||||
// would cost an allocation per event for nothing.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
#[derive(Debug)]
|
||||
pub enum Received {
|
||||
Event(Arrival),
|
||||
/// The connection came up: its subscriptions are being sent.
|
||||
Up(usize),
|
||||
/// The connection went down: nothing more comes from it until it is up.
|
||||
Down(usize),
|
||||
}
|
||||
|
||||
/// One event of a frame on its way to its market's book thread.
|
||||
#[derive(Debug)]
|
||||
pub struct Arrival {
|
||||
/// The frame's receive time, stamped right after the socket read.
|
||||
pub received_at: DateTime<Utc>,
|
||||
/// The connection that delivered it.
|
||||
pub connection: usize,
|
||||
/// Its share of the frame's bytes: a frame is one market's change, or
|
||||
/// books of several, each taking its share.
|
||||
pub bytes: u64,
|
||||
pub event: Event,
|
||||
}
|
||||
|
||||
/// The events of a frame's text; none for the keepalive's `PONG` and for
|
||||
/// `NO NEW ASSETS`.
|
||||
pub fn decode(text: &str) -> Result<Vec<Event>, serde_json::Error> {
|
||||
match text.trim() {
|
||||
"PONG" | "NO NEW ASSETS" => Ok(Vec::new()),
|
||||
text => Ok(match serde_json::from_str(text)? {
|
||||
Events::Many(events) => events,
|
||||
Events::One(event) => vec![event],
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
impl Event {
|
||||
/// The market it belongs to, which decides its book thread.
|
||||
pub fn market(&self) -> Option<&str> {
|
||||
match self {
|
||||
Event::Book(book) => Some(&book.market),
|
||||
Event::PriceChange(change) => Some(&change.market),
|
||||
Event::TickSizeChange { market, .. } | Event::LastTradePrice { market } => {
|
||||
market.as_deref()
|
||||
}
|
||||
Event::Other => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Events {
|
||||
Many(Vec<Event>),
|
||||
One(Event),
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "event_type", rename_all = "snake_case")]
|
||||
pub enum Event {
|
||||
Book(BookMessage),
|
||||
PriceChange(PriceChangeMessage),
|
||||
/// Only the symbol and its market are read: the hashes change with the
|
||||
/// tick size, and a fresh book carries the new one. The server sends each
|
||||
/// one twice (docs/notes.md). `market` was on every one seen; one without
|
||||
/// it can't be routed and is dropped.
|
||||
TickSizeChange {
|
||||
market: Option<String>,
|
||||
asset_id: String,
|
||||
},
|
||||
LastTradePrice {
|
||||
market: Option<String>,
|
||||
},
|
||||
/// Dropped: the others come only with `custom_feature_enabled`, which we
|
||||
/// don't ask for.
|
||||
#[serde(other)]
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct BookMessage {
|
||||
pub market: String,
|
||||
pub asset_id: String,
|
||||
pub timestamp: String,
|
||||
pub hash: String,
|
||||
pub bids: Vec<LevelMessage>,
|
||||
pub asks: Vec<LevelMessage>,
|
||||
pub tick_size: Option<String>,
|
||||
pub last_trade_price: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct LevelMessage {
|
||||
pub price: String,
|
||||
pub size: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct PriceChangeMessage {
|
||||
pub market: String,
|
||||
pub timestamp: String,
|
||||
pub price_changes: Vec<ChangeMessage>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ChangeMessage {
|
||||
pub asset_id: String,
|
||||
pub price: String,
|
||||
pub size: String,
|
||||
pub side: String,
|
||||
pub hash: String,
|
||||
}
|
||||
+93
-5
@@ -1,11 +1,15 @@
|
||||
//! Checks an output file against itself: per symbol, the Deltas written since
|
||||
//! a Snapshot, applied to it, must give the next Snapshot. A mismatch is a loss
|
||||
//! or a subscription gap (the fresh book differs), each logged by the collector.
|
||||
//! a Snapshot, applied to it, must give the next Snapshot. A Snapshot that
|
||||
//! doesn't is a mismatch, and the collector logs each with its cause
|
||||
//! ([`crate::cause`]): with the log, every mismatch must have its line.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use crate::book::Book;
|
||||
use crate::cause::{self, Cause};
|
||||
use crate::util::dtos::challenge_orderbook::{AnyMessage, PriceLevelUpdateDomain, SnapshotDomain};
|
||||
use crate::util::{AlignedSbeReader, OrderbookSchema};
|
||||
|
||||
@@ -16,10 +20,15 @@ pub struct Report {
|
||||
/// Deltas for a symbol with no Snapshot yet: the collector never writes one.
|
||||
pub orphan_deltas: usize,
|
||||
/// Each Snapshot that didn't match its symbol's book: (symbol, its `received_at`).
|
||||
pub mismatches: Vec<(String, chrono::DateTime<chrono::Utc>)>,
|
||||
pub mismatches: Vec<(String, DateTime<Utc>)>,
|
||||
/// The mismatches each cause explains, with the log.
|
||||
pub explained: HashMap<Cause, usize>,
|
||||
/// The mismatches no line of the log explains; all of them without a log.
|
||||
pub unexplained: Vec<(String, DateTime<Utc>)>,
|
||||
}
|
||||
|
||||
pub fn reconcile(path: impl AsRef<Path>) -> anyhow::Result<Report> {
|
||||
/// Reads the output at `path` and, if given, the collector's log.
|
||||
pub fn reconcile(path: impl AsRef<Path>, log: Option<&Path>) -> anyhow::Result<Report> {
|
||||
let mut reader = AlignedSbeReader::<OrderbookSchema>::open(path)?;
|
||||
let mut books: HashMap<String, Book> = HashMap::new();
|
||||
let mut report = Report::default();
|
||||
@@ -61,5 +70,84 @@ pub fn reconcile(path: impl AsRef<Path>) -> anyhow::Result<Report> {
|
||||
_ => anyhow::bail!("unknown message in the output"),
|
||||
}
|
||||
}
|
||||
let causes: HashMap<(String, DateTime<Utc>), Cause> = match log {
|
||||
Some(log) => std::fs::read_to_string(log)?
|
||||
.lines()
|
||||
.filter_map(cause::parse)
|
||||
.map(|(symbol, received_at, cause)| ((symbol, received_at), cause))
|
||||
.collect(),
|
||||
None => HashMap::new(),
|
||||
};
|
||||
let mut seen = HashSet::new();
|
||||
for mismatch in &report.mismatches {
|
||||
match causes.get(mismatch) {
|
||||
// A line explains one Snapshot: the same one twice would be a bug.
|
||||
Some(cause) if seen.insert(mismatch.clone()) => {
|
||||
*report.explained.entry(*cause).or_default() += 1
|
||||
}
|
||||
_ => report.unexplained.push(mismatch.clone()),
|
||||
}
|
||||
}
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::Side;
|
||||
use crate::util::AlignedSbeWriter;
|
||||
use crate::util::dtos::challenge_orderbook::{PriceLevelUpdateDomain, SnapshotBidsEntryDomain};
|
||||
|
||||
/// An output with a symbol's Snapshot, a Delta, then a Snapshot that
|
||||
/// misses it; and a log line for that second Snapshot when `explained`.
|
||||
fn reconcile_with(explained: bool) -> Report {
|
||||
let dir = std::env::temp_dir();
|
||||
let id = format!("{}-{explained}", std::process::id());
|
||||
let (output, log) = (
|
||||
dir.join(format!("reconcile-{id}.sbe")),
|
||||
dir.join(format!("reconcile-{id}.log")),
|
||||
);
|
||||
let at = |seconds: i64| DateTime::from_timestamp(1_794_390_000 + seconds, 0).unwrap();
|
||||
let snapshot = |received_at, quantity| SnapshotDomain {
|
||||
received_at,
|
||||
exchange_timestamp: received_at,
|
||||
bids: vec![SnapshotBidsEntryDomain {
|
||||
price_level: 500_000,
|
||||
quantity,
|
||||
}],
|
||||
asks: Vec::new(),
|
||||
symbol: "1".to_string(),
|
||||
};
|
||||
let mut writer = AlignedSbeWriter::create(output.to_str().unwrap(), 64 << 20).unwrap();
|
||||
writer.write_msg(&snapshot(at(0), 1_000_000)).unwrap();
|
||||
writer
|
||||
.write_msg(&PriceLevelUpdateDomain {
|
||||
backtest_timestamp: at(1),
|
||||
exchange_timestamp: at(1),
|
||||
price_level: 500_000,
|
||||
quantity: 2_000_000,
|
||||
side: Side::Buy,
|
||||
symbol: "1".to_string(),
|
||||
})
|
||||
.unwrap();
|
||||
writer.write_msg(&snapshot(at(2), 3_000_000)).unwrap();
|
||||
writer.flush().unwrap();
|
||||
let line = cause::line("1", at(if explained { 2 } else { 3 }), Cause::Loss, "");
|
||||
std::fs::write(&log, format!("12:00:00.000 polymarket: {line}\n")).unwrap();
|
||||
let report = reconcile(&output, Some(&log)).unwrap();
|
||||
std::fs::remove_file(output).unwrap();
|
||||
std::fs::remove_file(log).unwrap();
|
||||
report
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_mismatch_with_its_log_line_is_explained_and_one_without_is_not() {
|
||||
let report = reconcile_with(true);
|
||||
assert_eq!(report.mismatches.len(), 1);
|
||||
assert_eq!(report.explained, HashMap::from([(Cause::Loss, 1)]));
|
||||
assert!(report.unexplained.is_empty());
|
||||
let report = reconcile_with(false);
|
||||
assert_eq!(report.unexplained.len(), 1);
|
||||
assert!(report.explained.is_empty());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user