Archived
feat(polymarket): track every live market over load-balanced lanes
An orchestrator keeps the tracked set equal to Gamma's live markets (a scan at startup and every hour, new and closed markets every minute, reaching back 15 minutes for Gamma's indexing delay) and spreads them over lanes, each a connection and a book thread, all writing through the one writer. Lanes report each market's load in bytes per second; the orchestrator puts new markets on the least-loaded lane and moves the busiest markets off lanes over a 300 KB/s budget, busiest lane first. Differing books our own subscribe explains are subscription gaps, not losses, and every first book after a move is logged, so reconcile's mismatches are all accounted for in the log. A 75-minute live run tracked ~195k markets (~390k symbols) in 89 lanes.
This commit is contained in:
18 files changed
+2826
-320
No files matched your search
+8
-59
@@ -1,17 +1,17 @@
|
||||
//! Polymarket: which markets to track (Gamma), the market-channel socket, and
|
||||
//! the feed that turns its frames into books and output.
|
||||
//! Polymarket: which markets are live (Gamma), the orchestrator that spreads
|
||||
//! them over lanes, and each lane's connection and feed (roadmap/03).
|
||||
|
||||
pub mod feed;
|
||||
pub mod gamma;
|
||||
pub mod orchestrator;
|
||||
pub mod socket;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
const GAMMA_MARKETS_URL: &str = "https://gamma-api.polymarket.com/markets";
|
||||
|
||||
/// A market's symbols and the two inputs of their book hash that the
|
||||
/// A live market: its symbols and the two inputs of their book hash that the
|
||||
/// websocket doesn't carry (docs/notes.md).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Market {
|
||||
/// Gamma's `conditionId`, the websocket's `market`.
|
||||
pub condition_id: String,
|
||||
/// Its token ids, one per outcome: each is a symbol.
|
||||
pub tokens: Vec<String>,
|
||||
/// Exactly as the hash expects it, e.g. `"5"`.
|
||||
@@ -19,58 +19,7 @@ pub struct Market {
|
||||
pub neg_risk: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct GammaMarket {
|
||||
/// A JSON array inside a string.
|
||||
clob_token_ids: String,
|
||||
order_min_size: serde_json::Number,
|
||||
neg_risk: bool,
|
||||
enable_order_book: bool,
|
||||
}
|
||||
|
||||
/// The `limit` live markets with the most 24 h volume, keeping those with an
|
||||
/// order book. Fixed for the run: discovery is M3.
|
||||
pub fn top_markets(limit: u32) -> anyhow::Result<Vec<Market>> {
|
||||
let body = ureq::get(GAMMA_MARKETS_URL)
|
||||
.query("limit", limit.to_string())
|
||||
.query("active", "true")
|
||||
.query("closed", "false")
|
||||
.query("order", "volume24hr")
|
||||
.query("ascending", "false")
|
||||
.call()?
|
||||
.body_mut()
|
||||
.read_to_string()?;
|
||||
parse_markets(&body)
|
||||
}
|
||||
|
||||
fn parse_markets(body: &str) -> anyhow::Result<Vec<Market>> {
|
||||
let markets: Vec<GammaMarket> = serde_json::from_str(body)?;
|
||||
markets
|
||||
.into_iter()
|
||||
.filter(|market| market.enable_order_book)
|
||||
.map(|market| {
|
||||
Ok(Market {
|
||||
tokens: serde_json::from_str(&market.clob_token_ids)?,
|
||||
min_order_size: market.order_min_size.to_string(),
|
||||
neg_risk: market.neg_risk,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn fixture_markets() -> Vec<Market> {
|
||||
parse_markets(include_str!("../../tests/fixtures/gamma-markets.json")).unwrap()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn parses_gamma_markets() {
|
||||
let markets = super::fixture_markets();
|
||||
assert_eq!(markets.len(), 20);
|
||||
assert!(markets.iter().all(|market| market.tokens.len() == 2));
|
||||
assert_eq!(markets[0].min_order_size, "5");
|
||||
}
|
||||
gamma::parse_markets(include_str!("../../tests/fixtures/gamma-markets.json"))
|
||||
}
|
||||
Reference in new issue
Block a user