Archived
feat(polymarket)!: two connections per market, first one wins
Every live market is subscribed on two connections chosen at random, and its book takes each change from whichever connection delivers it first: the book hash says whether a change is a duplicate, the next one, or one to hold until the change before it arrives. A change one connection misses costs nothing, each Delta carries the earliest receive time, and a connection that drops is covered by the other. When a change fits on no connection in time, the book follows one connection unchecked and asks the other for a fresh book. Connections now decode and route events to 16 book threads, independent of them; a move leaves its old connection only once the new one delivers, so it costs no change; a connection closed with 1013 twice within ten minutes is split. reconcile reads the collector's log and fails only on a mismatch no cause line explains. SIGTERM and SIGHUP stop the collector cleanly. Breaking: lanes are gone (connections and book threads are separate), and reconcile takes the log as a second argument.
This commit is contained in:
24 files changed
+8984
-1214
No files matched your search
+1987
-693
File diff suppressed because it is too large.
Load diff
+37
-2
@@ -1,10 +1,25 @@
|
||||
//! Polymarket: which markets are live (Gamma), the orchestrator that spreads
|
||||
//! them over lanes, and each lane's connection and feed (roadmap/03).
|
||||
//! Polymarket: which markets are live (Gamma), the orchestrator that places
|
||||
//! them on connections, the connections, and the book threads they feed
|
||||
//! (docs/adr/0003-first-one-wins.md).
|
||||
|
||||
pub mod feed;
|
||||
pub mod gamma;
|
||||
pub mod orchestrator;
|
||||
pub mod socket;
|
||||
pub mod wire;
|
||||
|
||||
/// How many book threads own the books. One handled ~30,000 frames a second
|
||||
/// in sprint 02.
|
||||
pub const BOOK_THREADS: usize = 16;
|
||||
|
||||
/// The book thread that owns a market's books, from its `conditionId` (`0x`
|
||||
/// and 64 hex digits, a hash: any 16 of them spread evenly). Computed, so the
|
||||
/// orchestrator and every connection agree without a shared table.
|
||||
pub fn book_thread(condition_id: &str) -> usize {
|
||||
let digits = condition_id.get(2..18).unwrap_or_default();
|
||||
// An id of another shape still gets a thread, always the same one.
|
||||
u64::from_str_radix(digits, 16).unwrap_or(0) as usize % BOOK_THREADS
|
||||
}
|
||||
|
||||
/// A live market: its symbols and the two inputs of their book hash that the
|
||||
/// websocket doesn't carry (docs/notes.md).
|
||||
@@ -23,3 +38,23 @@ pub struct Market {
|
||||
pub(crate) fn fixture_markets() -> Vec<Market> {
|
||||
gamma::parse_markets(include_str!("../../tests/fixtures/gamma-markets.json"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_market_always_gets_the_same_book_thread_and_markets_spread() {
|
||||
let markets = fixture_markets();
|
||||
let threads: std::collections::BTreeSet<usize> = markets
|
||||
.iter()
|
||||
.map(|market| book_thread(&market.condition_id))
|
||||
.collect();
|
||||
assert!(threads.iter().all(|&thread| thread < BOOK_THREADS));
|
||||
// 20 markets over 16 threads: several threads, not one.
|
||||
assert!(threads.len() > 5, "{threads:?}");
|
||||
let id = &markets[0].condition_id;
|
||||
assert_eq!(book_thread(id), book_thread(&id.clone()));
|
||||
assert_eq!(book_thread("not an id"), 0);
|
||||
}
|
||||
}
|
||||
+694
-355
File diff suppressed because it is too large.
Load diff
+152
-26
@@ -1,6 +1,6 @@
|
||||
//! A lane's connection: reads the market channel and stamps each frame's
|
||||
//! receive time, and sends what its book thread asks for (subscriptions,
|
||||
//! resyncs). Parsing is the book thread's.
|
||||
//! A connection: reads the market channel, stamps each frame's receive time,
|
||||
//! decodes it and sends each event to the book thread of its market; and
|
||||
//! sends the exchange what the book threads ask for (subscriptions, resyncs).
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::io::ErrorKind;
|
||||
@@ -13,8 +13,12 @@ use chrono::{DateTime, Utc};
|
||||
use crossbeam::channel::{Receiver, Sender};
|
||||
use serde_json::json;
|
||||
use tungstenite::http::Uri;
|
||||
use tungstenite::protocol::frame::coding::CloseCode;
|
||||
use tungstenite::stream::MaybeTlsStream;
|
||||
use tungstenite::{Message, Utf8Bytes, WebSocket};
|
||||
use tungstenite::{Message, WebSocket};
|
||||
|
||||
use super::book_thread;
|
||||
use super::wire::{self, Arrival, Received};
|
||||
|
||||
pub const MARKET_CHANNEL_URL: &str = "wss://ws-subscriptions-clob.polymarket.com/ws/market";
|
||||
/// For the TCP connect and the TLS + websocket handshake, and for every send.
|
||||
@@ -29,10 +33,7 @@ const CLOSE_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
/// 500 gets no opening books (docs/notes.md).
|
||||
const CHUNK: usize = 500;
|
||||
|
||||
/// A frame's text with its receive time.
|
||||
pub type Frame = (DateTime<Utc>, Utf8Bytes);
|
||||
|
||||
/// What the book thread asks its connection for.
|
||||
/// What a book thread asks a connection for.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Command {
|
||||
Subscribe(Vec<String>),
|
||||
@@ -43,15 +44,23 @@ pub enum Command {
|
||||
|
||||
type Socket = WebSocket<MaybeTlsStream<TcpStream>>;
|
||||
|
||||
/// Runs until `stop` is set or the book thread is gone, reconnecting (and
|
||||
/// subscribing its tokens again) whenever the connection drops.
|
||||
/// Runs until `stop` is set or a book thread is gone, reconnecting (and
|
||||
/// subscribing its tokens again) whenever the connection drops, and telling
|
||||
/// every book thread when it comes up and goes down. `book_threads` holds
|
||||
/// every book thread's channel, by number; `closes` hears of each 1013 close.
|
||||
pub fn run(
|
||||
url: &str,
|
||||
lane: usize,
|
||||
frames: &Sender<Frame>,
|
||||
number: usize,
|
||||
book_threads: &[Sender<Received>],
|
||||
commands: &Receiver<Command>,
|
||||
closes: &Sender<usize>,
|
||||
stop: &AtomicBool,
|
||||
) {
|
||||
let tell = |received: fn(usize) -> Received| {
|
||||
for book_thread in book_threads {
|
||||
_ = book_thread.send(received(number));
|
||||
}
|
||||
};
|
||||
let mut tokens = BTreeSet::new();
|
||||
while !stop.load(Ordering::Relaxed) {
|
||||
// Asked for while disconnected: the subscribe on connect covers them,
|
||||
@@ -62,16 +71,36 @@ pub fn run(
|
||||
match connect(url, &tokens) {
|
||||
Ok(mut socket) => {
|
||||
log!(
|
||||
"polymarket: lane {lane}: connected, {} symbols",
|
||||
"polymarket: connection {number}: connected, {} symbols",
|
||||
tokens.len()
|
||||
);
|
||||
match read(&mut socket, &mut tokens, frames, commands, stop) {
|
||||
Ok(Ended::Stopped) => return close(&mut socket, lane, frames),
|
||||
Ok(Ended::BookThreadGone) => return,
|
||||
Err(error) => log!("polymarket: lane {lane}: connection lost: {error}"),
|
||||
tell(Received::Up);
|
||||
let ended = read(
|
||||
&mut socket,
|
||||
number,
|
||||
&mut tokens,
|
||||
book_threads,
|
||||
commands,
|
||||
closes,
|
||||
stop,
|
||||
);
|
||||
if !matches!(ended, Ok(Ended::Stopped)) {
|
||||
tell(Received::Down);
|
||||
}
|
||||
match ended {
|
||||
Ok(Ended::Stopped) => return close(&mut socket, number, book_threads),
|
||||
Ok(Ended::BookThreadGone) => {
|
||||
// Only a panic ends a book thread before us: its
|
||||
// markets would go unserved, so the run stops.
|
||||
log!("polymarket: connection {number}: a book thread is gone, stopping");
|
||||
return stop.store(true, Ordering::Relaxed);
|
||||
}
|
||||
Err(error) => {
|
||||
log!("polymarket: connection {number}: connection lost: {error}")
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => log!("polymarket: lane {lane}: connect failed: {error}"),
|
||||
Err(error) => log!("polymarket: connection {number}: connect failed: {error}"),
|
||||
}
|
||||
thread::sleep(RECONNECT_AFTER);
|
||||
}
|
||||
@@ -150,11 +179,53 @@ fn connect(url: &str, tokens: &BTreeSet<String>) -> anyhow::Result<Socket> {
|
||||
Ok(socket)
|
||||
}
|
||||
|
||||
/// Decodes a frame and sends each event to the book thread of its market;
|
||||
/// `false` once a book thread is gone.
|
||||
fn route(
|
||||
received_at: DateTime<Utc>,
|
||||
text: &str,
|
||||
number: usize,
|
||||
book_threads: &[Sender<Received>],
|
||||
) -> bool {
|
||||
let events = match wire::decode(text) {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
log!("polymarket: unreadable frame ({error}): {}", truncate(text));
|
||||
return true;
|
||||
}
|
||||
};
|
||||
let bytes = (text.len() / events.len().max(1)) as u64;
|
||||
for event in events {
|
||||
// An event of no market (none we ask for) has no book thread.
|
||||
let Some(thread) = event.market().map(book_thread) else {
|
||||
continue;
|
||||
};
|
||||
let arrival = Arrival {
|
||||
received_at,
|
||||
connection: number,
|
||||
bytes,
|
||||
event,
|
||||
};
|
||||
if book_threads[thread].send(Received::Event(arrival)).is_err() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn truncate(text: &str) -> &str {
|
||||
text.char_indices()
|
||||
.nth(200)
|
||||
.map_or(text, |(end, _)| &text[..end])
|
||||
}
|
||||
|
||||
fn read(
|
||||
socket: &mut Socket,
|
||||
number: usize,
|
||||
tokens: &mut BTreeSet<String>,
|
||||
frames: &Sender<Frame>,
|
||||
book_threads: &[Sender<Received>],
|
||||
commands: &Receiver<Command>,
|
||||
closes: &Sender<usize>,
|
||||
stop: &AtomicBool,
|
||||
) -> anyhow::Result<Ended> {
|
||||
let mut last_ping = Instant::now();
|
||||
@@ -164,11 +235,21 @@ fn read(
|
||||
let received_at = Utc::now();
|
||||
match message {
|
||||
Ok(Message::Text(text)) => {
|
||||
if frames.send((received_at, text)).is_err() {
|
||||
if !route(received_at, text.as_str(), number, book_threads) {
|
||||
return Ok(Ended::BookThreadGone);
|
||||
}
|
||||
}
|
||||
Ok(Message::Close(frame)) => anyhow::bail!("closed by the server: {frame:?}"),
|
||||
Ok(Message::Close(frame)) => {
|
||||
// 1013: the server couldn't send to us fast enough. The
|
||||
// orchestrator splits a connection closed so twice.
|
||||
if frame
|
||||
.as_ref()
|
||||
.is_some_and(|frame| frame.code == CloseCode::Again)
|
||||
{
|
||||
_ = closes.send(number);
|
||||
}
|
||||
anyhow::bail!("closed by the server: {frame:?}")
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(tungstenite::Error::Io(error)) if timed_out(&error) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
@@ -191,16 +272,16 @@ fn read(
|
||||
|
||||
/// Closes the connection, forwarding what still arrives until the server
|
||||
/// confirms or [`CLOSE_TIMEOUT`] passes.
|
||||
fn close(socket: &mut Socket, lane: usize, frames: &Sender<Frame>) {
|
||||
fn close(socket: &mut Socket, number: usize, book_threads: &[Sender<Received>]) {
|
||||
if let Err(error) = socket.close(None) {
|
||||
return log!("polymarket: lane {lane}: close failed: {error}");
|
||||
return log!("polymarket: connection {number}: close failed: {error}");
|
||||
}
|
||||
let deadline = Instant::now() + CLOSE_TIMEOUT;
|
||||
while Instant::now() < deadline {
|
||||
let message = socket.read();
|
||||
let received_at = Utc::now();
|
||||
match message {
|
||||
Ok(Message::Text(text)) => _ = frames.send((received_at, text)),
|
||||
Ok(Message::Text(text)) => _ = route(received_at, text.as_str(), number, book_threads),
|
||||
Ok(_) => {}
|
||||
Err(tungstenite::Error::Io(error)) if timed_out(&error) => {}
|
||||
// ConnectionClosed: the close handshake is done.
|
||||
@@ -263,14 +344,16 @@ mod tests {
|
||||
connections
|
||||
});
|
||||
|
||||
let (frames_tx, _frames_rx) = unbounded();
|
||||
let (events_tx, _events_rx) = unbounded();
|
||||
let (commands_tx, commands_rx) = unbounded();
|
||||
commands_tx
|
||||
.send(Command::Subscribe(tokens.clone()))
|
||||
.unwrap();
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let client_stop = Arc::clone(&stop);
|
||||
let client = thread::spawn(move || run(&url, 0, &frames_tx, &commands_rx, &client_stop));
|
||||
let (closes, _closes_rx) = unbounded();
|
||||
let client =
|
||||
thread::spawn(move || run(&url, 0, &[events_tx], &commands_rx, &closes, &client_stop));
|
||||
|
||||
let connections = server.join().unwrap();
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
@@ -285,6 +368,49 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// The recording's first frame holds the opening books of 40 tokens, 20
|
||||
/// markets: each book goes to its market's book thread, with its share of
|
||||
/// the frame's bytes.
|
||||
#[test]
|
||||
fn a_frame_of_several_markets_reaches_each_markets_book_thread() {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Line {
|
||||
text: String,
|
||||
}
|
||||
let path = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/polymarket-frames.jsonl"
|
||||
);
|
||||
let recording = std::fs::read_to_string(path).unwrap();
|
||||
let line: Line = serde_json::from_str(recording.lines().next().unwrap()).unwrap();
|
||||
let (senders, receivers): (Vec<_>, Vec<_>) = (0..crate::polymarket::BOOK_THREADS)
|
||||
.map(|_| unbounded())
|
||||
.unzip();
|
||||
|
||||
assert!(route(Utc::now(), &line.text, 3, &senders));
|
||||
let mut books = 0;
|
||||
for (thread, receiver) in receivers.iter().enumerate() {
|
||||
for received in receiver.try_iter() {
|
||||
let Received::Event(arrival) = received else {
|
||||
panic!("not an event: {received:?}");
|
||||
};
|
||||
let market = arrival.event.market().unwrap().to_string();
|
||||
assert_eq!(book_thread(&market), thread);
|
||||
assert_eq!(arrival.connection, 3);
|
||||
assert_eq!(arrival.bytes, line.text.len() as u64 / 40);
|
||||
books += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(books, 40);
|
||||
// Neither a keepalive's answer nor an unreadable frame reaches one.
|
||||
assert!(route(Utc::now(), "PONG", 3, &senders));
|
||||
assert!(route(Utc::now(), "not json", 3, &senders));
|
||||
assert!(receivers.iter().all(|receiver| receiver.is_empty()));
|
||||
// A book thread gone stops the connection.
|
||||
drop(receivers);
|
||||
assert!(!route(Utc::now(), &line.text, 3, &senders));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commands_update_the_token_set_and_say_what_to_send() {
|
||||
let mut tokens = BTreeSet::new();
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
//! The market channel's messages, as the exchange writes them, and how a
|
||||
//! frame's text becomes events (docs/notes.md).
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::Deserialize;
|
||||
|
||||
/// What a connection sends a book thread.
|
||||
// Nearly every message is an event: boxing it to shrink the two rare markers
|
||||
// would cost an allocation per event for nothing.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
#[derive(Debug)]
|
||||
pub enum Received {
|
||||
Event(Arrival),
|
||||
/// The connection came up: its subscriptions are being sent.
|
||||
Up(usize),
|
||||
/// The connection went down: nothing more comes from it until it is up.
|
||||
Down(usize),
|
||||
}
|
||||
|
||||
/// One event of a frame on its way to its market's book thread.
|
||||
#[derive(Debug)]
|
||||
pub struct Arrival {
|
||||
/// The frame's receive time, stamped right after the socket read.
|
||||
pub received_at: DateTime<Utc>,
|
||||
/// The connection that delivered it.
|
||||
pub connection: usize,
|
||||
/// Its share of the frame's bytes: a frame is one market's change, or
|
||||
/// books of several, each taking its share.
|
||||
pub bytes: u64,
|
||||
pub event: Event,
|
||||
}
|
||||
|
||||
/// The events of a frame's text; none for the keepalive's `PONG` and for
|
||||
/// `NO NEW ASSETS`.
|
||||
pub fn decode(text: &str) -> Result<Vec<Event>, serde_json::Error> {
|
||||
match text.trim() {
|
||||
"PONG" | "NO NEW ASSETS" => Ok(Vec::new()),
|
||||
text => Ok(match serde_json::from_str(text)? {
|
||||
Events::Many(events) => events,
|
||||
Events::One(event) => vec![event],
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
impl Event {
|
||||
/// The market it belongs to, which decides its book thread.
|
||||
pub fn market(&self) -> Option<&str> {
|
||||
match self {
|
||||
Event::Book(book) => Some(&book.market),
|
||||
Event::PriceChange(change) => Some(&change.market),
|
||||
Event::TickSizeChange { market, .. } | Event::LastTradePrice { market } => {
|
||||
market.as_deref()
|
||||
}
|
||||
Event::Other => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Events {
|
||||
Many(Vec<Event>),
|
||||
One(Event),
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "event_type", rename_all = "snake_case")]
|
||||
pub enum Event {
|
||||
Book(BookMessage),
|
||||
PriceChange(PriceChangeMessage),
|
||||
/// Only the symbol and its market are read: the hashes change with the
|
||||
/// tick size, and a fresh book carries the new one. The server sends each
|
||||
/// one twice (docs/notes.md). `market` was on every one seen; one without
|
||||
/// it can't be routed and is dropped.
|
||||
TickSizeChange {
|
||||
market: Option<String>,
|
||||
asset_id: String,
|
||||
},
|
||||
LastTradePrice {
|
||||
market: Option<String>,
|
||||
},
|
||||
/// Dropped: the others come only with `custom_feature_enabled`, which we
|
||||
/// don't ask for.
|
||||
#[serde(other)]
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct BookMessage {
|
||||
pub market: String,
|
||||
pub asset_id: String,
|
||||
pub timestamp: String,
|
||||
pub hash: String,
|
||||
pub bids: Vec<LevelMessage>,
|
||||
pub asks: Vec<LevelMessage>,
|
||||
pub tick_size: Option<String>,
|
||||
pub last_trade_price: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct LevelMessage {
|
||||
pub price: String,
|
||||
pub size: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct PriceChangeMessage {
|
||||
pub market: String,
|
||||
pub timestamp: String,
|
||||
pub price_changes: Vec<ChangeMessage>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ChangeMessage {
|
||||
pub asset_id: String,
|
||||
pub price: String,
|
||||
pub size: String,
|
||||
pub side: String,
|
||||
pub hash: String,
|
||||
}
|
||||
Reference in new issue
Block a user