Archived
feat(polymarket): hash-verified order books for a subset of symbols
Collects the order books of Polymarket's 20 busiest markets (40 symbols) into the template's SBE format: a Delta per level change, plus Snapshots on subscribe, every minute, and whenever a fresh book corrects ours. Every change is checked against Polymarket's own book hash; a failed check resyncs the symbol on the open socket, and the log counts losses apart from the few ms our own resync hides. Runs until SIGINT, which closes the socket and flushes the output; reconnects when the server drops it. A `reconcile` binary checks an output file against itself.
This commit is contained in:
22 files changed
+6423
-84
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,76 @@
|
||||
//! Polymarket: which markets to track (Gamma), the market-channel socket, and
|
||||
//! the feed that turns its frames into books and output.
|
||||
|
||||
pub mod feed;
|
||||
pub mod socket;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
const GAMMA_MARKETS_URL: &str = "https://gamma-api.polymarket.com/markets";
|
||||
|
||||
/// A market's symbols and the two inputs of their book hash that the
|
||||
/// websocket doesn't carry (docs/notes.md).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Market {
|
||||
/// Its token ids, one per outcome: each is a symbol.
|
||||
pub tokens: Vec<String>,
|
||||
/// Exactly as the hash expects it, e.g. `"5"`.
|
||||
pub min_order_size: String,
|
||||
pub neg_risk: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct GammaMarket {
|
||||
/// A JSON array inside a string.
|
||||
clob_token_ids: String,
|
||||
order_min_size: serde_json::Number,
|
||||
neg_risk: bool,
|
||||
enable_order_book: bool,
|
||||
}
|
||||
|
||||
/// The `limit` live markets with the most 24 h volume, keeping those with an
|
||||
/// order book. Fixed for the run: discovery is M3.
|
||||
pub fn top_markets(limit: u32) -> anyhow::Result<Vec<Market>> {
|
||||
let body = ureq::get(GAMMA_MARKETS_URL)
|
||||
.query("limit", limit.to_string())
|
||||
.query("active", "true")
|
||||
.query("closed", "false")
|
||||
.query("order", "volume24hr")
|
||||
.query("ascending", "false")
|
||||
.call()?
|
||||
.body_mut()
|
||||
.read_to_string()?;
|
||||
parse_markets(&body)
|
||||
}
|
||||
|
||||
fn parse_markets(body: &str) -> anyhow::Result<Vec<Market>> {
|
||||
let markets: Vec<GammaMarket> = serde_json::from_str(body)?;
|
||||
markets
|
||||
.into_iter()
|
||||
.filter(|market| market.enable_order_book)
|
||||
.map(|market| {
|
||||
Ok(Market {
|
||||
tokens: serde_json::from_str(&market.clob_token_ids)?,
|
||||
min_order_size: market.order_min_size.to_string(),
|
||||
neg_risk: market.neg_risk,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn fixture_markets() -> Vec<Market> {
|
||||
parse_markets(include_str!("../../tests/fixtures/gamma-markets.json")).unwrap()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn parses_gamma_markets() {
|
||||
let markets = super::fixture_markets();
|
||||
assert_eq!(markets.len(), 20);
|
||||
assert!(markets.iter().all(|market| market.tokens.len() == 2));
|
||||
assert_eq!(markets[0].min_order_size, "5");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
//! The socket thread: reads the market channel and stamps each frame's receive
|
||||
//! time, and nothing else. Parsing is the book thread's.
|
||||
|
||||
use std::io::ErrorKind;
|
||||
use std::net::{TcpStream, ToSocketAddrs};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use crossbeam::channel::{Receiver, Sender};
|
||||
use serde_json::json;
|
||||
use tungstenite::stream::MaybeTlsStream;
|
||||
use tungstenite::{Message, Utf8Bytes, WebSocket};
|
||||
|
||||
const MARKET_CHANNEL_HOST: &str = "ws-subscriptions-clob.polymarket.com";
|
||||
const MARKET_CHANNEL_URL: &str = "wss://ws-subscriptions-clob.polymarket.com/ws/market";
|
||||
/// For the TCP connect and the TLS + websocket handshake, and for every send.
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
/// How often the loop wakes when no frame comes: tungstenite has no `split`,
|
||||
/// so the reads and the writes below (PING, resyncs, close) share this thread.
|
||||
const READ_TIMEOUT: Duration = Duration::from_millis(200);
|
||||
const PING_EVERY: Duration = Duration::from_secs(10);
|
||||
const RECONNECT_AFTER: Duration = Duration::from_secs(1);
|
||||
const CLOSE_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// A frame's text with its receive time.
|
||||
pub type Frame = (DateTime<Utc>, Utf8Bytes);
|
||||
|
||||
type Socket = WebSocket<MaybeTlsStream<TcpStream>>;
|
||||
|
||||
/// Runs until `stop` is set or the book thread is gone, reconnecting (and
|
||||
/// subscribing every token again) whenever the connection drops. `resyncs`
|
||||
/// carries the tokens to unsubscribe and subscribe again.
|
||||
pub fn run(
|
||||
tokens: &[String],
|
||||
frames: &Sender<Frame>,
|
||||
resyncs: &Receiver<String>,
|
||||
stop: &AtomicBool,
|
||||
) {
|
||||
while !stop.load(Ordering::Relaxed) {
|
||||
match connect(tokens) {
|
||||
Ok(mut socket) => match read(&mut socket, frames, resyncs, stop) {
|
||||
Ok(Ended::Stopped) => return close(&mut socket, frames),
|
||||
Ok(Ended::BookThreadGone) => return,
|
||||
Err(error) => log!("polymarket: connection lost: {error}"),
|
||||
},
|
||||
Err(error) => log!("polymarket: connect failed: {error}"),
|
||||
}
|
||||
thread::sleep(RECONNECT_AFTER);
|
||||
}
|
||||
}
|
||||
|
||||
enum Ended {
|
||||
Stopped,
|
||||
BookThreadGone,
|
||||
}
|
||||
|
||||
fn connect(tokens: &[String]) -> anyhow::Result<Socket> {
|
||||
// Timed, unlike `tungstenite::connect`, so a stalled connect or handshake
|
||||
// can't keep SIGINT waiting. The name lookup itself has no timeout.
|
||||
let address = (MARKET_CHANNEL_HOST, 443)
|
||||
.to_socket_addrs()?
|
||||
.next()
|
||||
.ok_or_else(|| anyhow::anyhow!("{MARKET_CHANNEL_HOST} has no address"))?;
|
||||
let stream = TcpStream::connect_timeout(&address, CONNECT_TIMEOUT)?;
|
||||
stream.set_read_timeout(Some(CONNECT_TIMEOUT))?;
|
||||
stream.set_write_timeout(Some(CONNECT_TIMEOUT))?;
|
||||
let (mut socket, _response) = tungstenite::client_tls(MARKET_CHANNEL_URL, stream)
|
||||
.map_err(|error| anyhow::anyhow!("handshake failed: {error}"))?;
|
||||
let stream = match socket.get_ref() {
|
||||
MaybeTlsStream::Plain(stream) => stream,
|
||||
MaybeTlsStream::Rustls(stream) => stream.get_ref(),
|
||||
_ => anyhow::bail!("unexpected TLS stream"),
|
||||
};
|
||||
stream.set_read_timeout(Some(READ_TIMEOUT))?;
|
||||
socket.send(Message::text(
|
||||
json!({"type": "market", "assets_ids": tokens}).to_string(),
|
||||
))?;
|
||||
log!("polymarket: connected, subscribed {} symbols", tokens.len());
|
||||
Ok(socket)
|
||||
}
|
||||
|
||||
fn read(
|
||||
socket: &mut Socket,
|
||||
frames: &Sender<Frame>,
|
||||
resyncs: &Receiver<String>,
|
||||
stop: &AtomicBool,
|
||||
) -> anyhow::Result<Ended> {
|
||||
let mut last_ping = Instant::now();
|
||||
loop {
|
||||
let message = socket.read();
|
||||
// Stamp as soon as the frame is off the wire, before any processing.
|
||||
let received_at = Utc::now();
|
||||
match message {
|
||||
Ok(Message::Text(text)) => {
|
||||
if frames.send((received_at, text)).is_err() {
|
||||
return Ok(Ended::BookThreadGone);
|
||||
}
|
||||
}
|
||||
Ok(Message::Close(frame)) => anyhow::bail!("closed by the server: {frame:?}"),
|
||||
Ok(_) => {}
|
||||
Err(tungstenite::Error::Io(error)) if timed_out(&error) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
return Ok(Ended::Stopped);
|
||||
}
|
||||
if last_ping.elapsed() >= PING_EVERY {
|
||||
socket.send(Message::text("PING"))?;
|
||||
last_ping = Instant::now();
|
||||
}
|
||||
for token in resyncs.try_iter() {
|
||||
// Subscribing a token already subscribed sends nothing back but
|
||||
// `NO NEW ASSETS`: unsubscribing first makes the server send a
|
||||
// fresh book.
|
||||
socket.send(Message::text(
|
||||
json!({"assets_ids": [&token], "operation": "unsubscribe"}).to_string(),
|
||||
))?;
|
||||
socket.send(Message::text(
|
||||
json!({"assets_ids": [&token], "operation": "subscribe"}).to_string(),
|
||||
))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Closes the connection, forwarding what still arrives until the server
|
||||
/// confirms or [`CLOSE_TIMEOUT`] passes.
|
||||
fn close(socket: &mut Socket, frames: &Sender<Frame>) {
|
||||
if let Err(error) = socket.close(None) {
|
||||
return log!("polymarket: close failed: {error}");
|
||||
}
|
||||
let deadline = Instant::now() + CLOSE_TIMEOUT;
|
||||
while Instant::now() < deadline {
|
||||
let message = socket.read();
|
||||
let received_at = Utc::now();
|
||||
match message {
|
||||
Ok(Message::Text(text)) => _ = frames.send((received_at, text)),
|
||||
Ok(_) => {}
|
||||
Err(tungstenite::Error::Io(error)) if timed_out(&error) => {}
|
||||
// ConnectionClosed: the close handshake is done.
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
log!("polymarket: socket closed");
|
||||
}
|
||||
|
||||
/// A read timeout shows up as `WouldBlock` on macOS and `TimedOut` elsewhere;
|
||||
/// tungstenite keeps a partly read frame, so the next read resumes it.
|
||||
fn timed_out(error: &std::io::Error) -> bool {
|
||||
matches!(error.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut)
|
||||
}
|
||||
Reference in new issue
Block a user