Archived
feat!: a generic core, and Kalshi through it
The orchestrator, the connection loop and the book thread's shell move to src/core/, generic over an Exchange and its loss rule (ADR 0004). Polymarket runs through it unchanged. Kalshi joins: a signed handshake (KALSHI_KEY_ID_FILE, KALSHI_PRIVATE_KEY_FILE), a reconnect on a skipped sequence number, first one wins by clock and rank, and its market listing with markets tracked from three hours before they open. Breaking: the exchange is now the first argument, `engineering-challenge <polymarket|kalshi|kalshi-demo> [OUTPUT]`.
This commit is contained in:
33 files changed
+8150
-1846
No files matched your search
@@ -0,0 +1,823 @@
|
||||
//! Kalshi's rule for a book thread: first one wins by clock and rank
|
||||
//! (roadmap/06). A market's changes carry its clock, the same on every
|
||||
//! connection, and come in the same order on each; several can share a
|
||||
//! clock, so a change is named by its clock and its rank among the changes of
|
||||
//! that clock. The book stands at the position of the last change it holds;
|
||||
//! each connection's stream at the position of the last change it delivered,
|
||||
//! counted from the book it last delivered. A change passing the book's
|
||||
//! position is the next one; any other is a duplicate. Nothing is held.
|
||||
//! A connection that lost a message reconnects (its `seq`, in
|
||||
//! `src/kalshi/mod.rs`); the book then comes fresh.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use engineering_challenge::book::Book;
|
||||
use engineering_challenge::cause::{self, Cause};
|
||||
|
||||
use super::Market;
|
||||
use super::wire::{BookMessage, ChangeMessage, Event};
|
||||
use crate::core::feed::{self, Effect, Minute, Rule, Shell, is_up};
|
||||
use crate::core::socket::Command;
|
||||
|
||||
/// A clock, and a rank among the changes of that clock.
|
||||
type Position = (u64, u32);
|
||||
|
||||
/// A book's rank: it holds every change of its clock.
|
||||
const BOOK: u32 = u32::MAX;
|
||||
|
||||
/// A symbol's state for the rule.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Clocked {
|
||||
/// Where the book stands; none before its first book.
|
||||
at: Option<Position>,
|
||||
/// By connection: where its stream stands, since the book it last
|
||||
/// delivered. A connection with none has delivered no book since it came
|
||||
/// up or subscribed: its changes have nowhere to go.
|
||||
on: BTreeMap<usize, Position>,
|
||||
/// The connection a fresh book was asked from: its answer replaces ours.
|
||||
asked: Option<usize>,
|
||||
}
|
||||
|
||||
type Symbol = feed::Symbol<Clocked>;
|
||||
|
||||
/// What the minute line counts, since the start.
|
||||
#[derive(Debug, Default)]
|
||||
struct Counts {
|
||||
written: u64,
|
||||
duplicates: u64,
|
||||
/// Changes from a connection that had delivered no book of the symbol.
|
||||
unplaced: u64,
|
||||
/// Fresh books after a symbol's first.
|
||||
books: u64,
|
||||
/// Of them, books at or past ours that differed: written as a Snapshot.
|
||||
losses: u64,
|
||||
/// Changes from a connection past the book, after the book went back to
|
||||
/// an answer: dropped until the book reaches it.
|
||||
ahead: u64,
|
||||
/// Changes that took a level below zero: our book was wrong.
|
||||
below_zero: u64,
|
||||
resyncs: u64,
|
||||
}
|
||||
|
||||
/// Kalshi's rule, for one book thread.
|
||||
#[derive(Default)]
|
||||
pub struct Clock {
|
||||
counts: Counts,
|
||||
}
|
||||
|
||||
impl Clock {
|
||||
fn on_book(
|
||||
&mut self,
|
||||
shell: &mut Shell<Clocked>,
|
||||
received_at: DateTime<Utc>,
|
||||
connection: usize,
|
||||
message: BookMessage,
|
||||
effects: &mut Vec<Effect>,
|
||||
) {
|
||||
let ticker = message.ticker;
|
||||
let Some(symbol) = shell.symbols.get_mut(&ticker) else {
|
||||
return;
|
||||
};
|
||||
if !symbol.connections.contains(&connection) {
|
||||
return;
|
||||
}
|
||||
let position = (message.clock, BOOK);
|
||||
symbol.rule.on.insert(connection, position);
|
||||
let book = Book {
|
||||
bids: message.bids.into_iter().collect(),
|
||||
asks: message.asks.into_iter().collect(),
|
||||
};
|
||||
let asked = symbol.rule.asked == Some(connection);
|
||||
let cause = match symbol.rule.at {
|
||||
None => None,
|
||||
Some(at) => {
|
||||
self.counts.books += 1;
|
||||
// Behind ours, unasked: it only says where its connection is.
|
||||
if position < at && !asked {
|
||||
return;
|
||||
}
|
||||
if book == symbol.book {
|
||||
symbol.rule.at = Some(at.max(position));
|
||||
if asked {
|
||||
symbol.rule.asked = None;
|
||||
}
|
||||
return;
|
||||
}
|
||||
Some(at)
|
||||
}
|
||||
};
|
||||
// The first book, an answer, or a book at or past ours that differs:
|
||||
// ours is replaced. Which connection missed what can't be told: the
|
||||
// other may only be behind.
|
||||
symbol.book = book;
|
||||
symbol.exchange_timestamp = message.sent_at;
|
||||
symbol.rule.at = Some(position);
|
||||
// A first book answers whichever connection it was asked from.
|
||||
if asked || cause.is_none() {
|
||||
symbol.rule.asked = None;
|
||||
}
|
||||
effects.push(Effect::Write(symbol.snapshot(&ticker, received_at)));
|
||||
if let Some(ours) = cause {
|
||||
self.counts.losses += 1;
|
||||
let detail = format!(" (its clock {}, ours {})", message.clock, ours.0);
|
||||
log!(
|
||||
"kalshi: {}",
|
||||
cause::line(&ticker, received_at, Cause::Loss, &detail)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn on_change(
|
||||
&mut self,
|
||||
shell: &mut Shell<Clocked>,
|
||||
received_at: DateTime<Utc>,
|
||||
connection: usize,
|
||||
change: ChangeMessage,
|
||||
effects: &mut Vec<Effect>,
|
||||
) {
|
||||
let ticker = change.ticker;
|
||||
shell.count_received(&ticker, connection);
|
||||
let Some(symbol) = shell.symbols.get_mut(&ticker) else {
|
||||
return;
|
||||
};
|
||||
let Some(on) = symbol.rule.on.get_mut(&connection) else {
|
||||
self.counts.unplaced += 1;
|
||||
return;
|
||||
};
|
||||
let before = *on;
|
||||
*on = match before {
|
||||
(clock, rank) if clock == change.clock => (clock, rank.saturating_add(1)),
|
||||
_ => (change.clock, 1),
|
||||
};
|
||||
let position = *on;
|
||||
match symbol.rule.at {
|
||||
Some(at) if position <= at => {
|
||||
self.counts.duplicates += 1;
|
||||
return;
|
||||
}
|
||||
// Its stream was already past the book, which went back to an
|
||||
// answer: the changes between are still to come from the
|
||||
// connection that answered, and this one would skip them.
|
||||
Some(at) if before > at => {
|
||||
self.counts.ahead += 1;
|
||||
return;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
symbol.rule.at = Some(position);
|
||||
let quantity = symbol.book.add(change.side, change.price, change.delta);
|
||||
let quantity = if quantity < 0 {
|
||||
// A level can't hold less than nothing: ours was wrong.
|
||||
self.counts.below_zero += 1;
|
||||
log!(
|
||||
"kalshi: {ticker}: a change takes a level below zero, resync on connection \
|
||||
{connection}"
|
||||
);
|
||||
if symbol.rule.asked.is_none() {
|
||||
Self::ask_fresh_book(&ticker, symbol, connection, effects);
|
||||
self.counts.resyncs += 1;
|
||||
}
|
||||
0
|
||||
} else {
|
||||
quantity
|
||||
};
|
||||
symbol.exchange_timestamp = change.at;
|
||||
self.counts.written += 1;
|
||||
effects.push(Effect::Write(symbol.delta(
|
||||
&ticker,
|
||||
change.side,
|
||||
change.price,
|
||||
quantity,
|
||||
received_at,
|
||||
change.at,
|
||||
)));
|
||||
}
|
||||
|
||||
/// Asks `connection` for a fresh book: unsubscribe and subscribe again.
|
||||
fn ask_fresh_book(
|
||||
ticker: &str,
|
||||
symbol: &mut Symbol,
|
||||
connection: usize,
|
||||
effects: &mut Vec<Effect>,
|
||||
) {
|
||||
symbol.rule.asked = Some(connection);
|
||||
effects.push(Effect::Send(
|
||||
connection,
|
||||
Command::Resubscribe(ticker.to_string()),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
impl Rule for Clock {
|
||||
type Market = Market;
|
||||
type Event = Event;
|
||||
type Symbol = Clocked;
|
||||
|
||||
fn symbol(&self, _market: &Market) -> Clocked {
|
||||
Clocked::default()
|
||||
}
|
||||
|
||||
fn on_event(
|
||||
&mut self,
|
||||
shell: &mut Shell<Clocked>,
|
||||
connection: usize,
|
||||
received_at: DateTime<Utc>,
|
||||
event: Event,
|
||||
effects: &mut Vec<Effect>,
|
||||
) {
|
||||
match event {
|
||||
Event::Book(book) => self.on_book(shell, received_at, connection, book, effects),
|
||||
Event::Change(change) => {
|
||||
self.on_change(shell, received_at, connection, change, effects)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Its stream there counts no more; a fresh book asked from it is asked
|
||||
/// from another.
|
||||
fn on_leave(
|
||||
&mut self,
|
||||
up: &[bool],
|
||||
ticker: &str,
|
||||
symbol: &mut Symbol,
|
||||
connection: usize,
|
||||
effects: &mut Vec<Effect>,
|
||||
) {
|
||||
symbol.rule.on.remove(&connection);
|
||||
if symbol.rule.asked == Some(connection)
|
||||
&& let Some(other) = symbol.other_up(up, connection)
|
||||
{
|
||||
Self::ask_fresh_book(ticker, symbol, other, effects);
|
||||
self.counts.resyncs += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Its streams count no more: back up, it delivers fresh books, and one
|
||||
/// asked from it is its answer.
|
||||
fn on_down(
|
||||
&mut self,
|
||||
shell: &mut Shell<Clocked>,
|
||||
connection: usize,
|
||||
_effects: &mut Vec<Effect>,
|
||||
) {
|
||||
for symbol in shell.symbols.values_mut() {
|
||||
symbol.rule.on.remove(&connection);
|
||||
}
|
||||
}
|
||||
|
||||
fn on_timer(
|
||||
&mut self,
|
||||
_shell: &mut Shell<Clocked>,
|
||||
_now: DateTime<Utc>,
|
||||
_effects: &mut Vec<Effect>,
|
||||
) {
|
||||
}
|
||||
|
||||
fn wakes_soon(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn on_removed(&mut self, _ticker: &str) {}
|
||||
|
||||
/// While no connection delivers it, the book may be missing changes: no
|
||||
/// Snapshot of it until one does.
|
||||
fn has_book(symbol: &Clocked) -> bool {
|
||||
symbol.at.is_some() && !symbol.on.is_empty()
|
||||
}
|
||||
|
||||
fn may_be_quiet(symbol: &Clocked) -> bool {
|
||||
symbol.at.is_some() && symbol.asked.is_none()
|
||||
}
|
||||
|
||||
/// Asks again for a fresh book still not come, and for the book of a
|
||||
/// symbol no connection delivers: its first, or one a reconnect didn't
|
||||
/// bring.
|
||||
fn on_minute(
|
||||
&mut self,
|
||||
up: &[bool],
|
||||
ticker: &str,
|
||||
symbol: &mut Symbol,
|
||||
_now: DateTime<Utc>,
|
||||
effects: &mut Vec<Effect>,
|
||||
) {
|
||||
let again = symbol.rule.asked.or_else(|| {
|
||||
let up = |&connection: &usize| is_up(up, connection);
|
||||
symbol
|
||||
.rule
|
||||
.on
|
||||
.is_empty()
|
||||
.then(|| symbol.connections.iter().copied().find(up))
|
||||
.flatten()
|
||||
});
|
||||
if let Some(connection) = again {
|
||||
Self::ask_fresh_book(ticker, symbol, connection, effects);
|
||||
self.counts.resyncs += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn log_minute(&mut self, minute: &Minute) {
|
||||
let counts = &self.counts;
|
||||
log!(
|
||||
"kalshi: book thread {}: {} symbols; so far {} changes written, {} duplicates, \
|
||||
{} before their connection's book, {} ahead of the book, {} fresh books \
|
||||
({} losses), {} below zero, \
|
||||
{} resyncs, {} quiet subscriptions resubscribed, {} markets reported quiet again; \
|
||||
this minute {} changes received, at most {} events waited",
|
||||
minute.number,
|
||||
minute.symbols,
|
||||
counts.written,
|
||||
counts.duplicates,
|
||||
counts.unplaced,
|
||||
counts.ahead,
|
||||
counts.books,
|
||||
counts.losses,
|
||||
counts.below_zero,
|
||||
counts.resyncs,
|
||||
minute.quiet,
|
||||
minute.quiet_reported,
|
||||
minute.received,
|
||||
minute.peak_waiting
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use chrono::TimeDelta;
|
||||
use engineering_challenge::util::dtos::challenge_orderbook::PriceLevelUpdateDomain;
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::*;
|
||||
use crate::core::{Arrival, Received, feed::next_minute};
|
||||
use crate::kalshi::wire;
|
||||
use engineering_challenge::Side;
|
||||
use engineering_challenge::book::Output;
|
||||
|
||||
type Feed = feed::Feed<Clock>;
|
||||
|
||||
fn at(millis: i64) -> DateTime<Utc> {
|
||||
DateTime::from_timestamp_millis(1_794_422_000_000 + millis).unwrap()
|
||||
}
|
||||
|
||||
fn feed_of(tickers: &[&str], connections: &[usize]) -> Feed {
|
||||
let mut feed = Feed::new("kalshi", 0, Clock::default());
|
||||
for ticker in tickers {
|
||||
feed.add_market(
|
||||
&Market {
|
||||
ticker: ticker.to_string(),
|
||||
},
|
||||
connections,
|
||||
at(0),
|
||||
);
|
||||
}
|
||||
feed
|
||||
}
|
||||
|
||||
fn book(ticker: &str, clock: u64, bids: &[(i64, i64)], asks: &[(i64, i64)]) -> Event {
|
||||
Event::Book(BookMessage {
|
||||
ticker: ticker.into(),
|
||||
clock,
|
||||
bids: bids.to_vec(),
|
||||
asks: asks.to_vec(),
|
||||
sent_at: at(0),
|
||||
})
|
||||
}
|
||||
|
||||
fn change(ticker: &str, clock: u64, side: Side, price: i64, delta: i64) -> Event {
|
||||
Event::Change(ChangeMessage {
|
||||
ticker: ticker.into(),
|
||||
clock,
|
||||
side,
|
||||
price,
|
||||
delta,
|
||||
at: at(0),
|
||||
})
|
||||
}
|
||||
|
||||
impl Feed {
|
||||
fn from(&mut self, connection: usize, millis: i64, event: Event) -> Vec<Effect> {
|
||||
self.on_event(Arrival {
|
||||
received_at: at(millis),
|
||||
connection,
|
||||
bytes: 0,
|
||||
event,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn deltas(effects: &[Effect]) -> Vec<&PriceLevelUpdateDomain> {
|
||||
effects
|
||||
.iter()
|
||||
.filter_map(|effect| match effect {
|
||||
Effect::Write(Output::Delta(delta)) => Some(delta),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn snapshots(effects: &[Effect]) -> usize {
|
||||
effects
|
||||
.iter()
|
||||
.filter(|effect| matches!(effect, Effect::Write(Output::Snapshot(_))))
|
||||
.count()
|
||||
}
|
||||
|
||||
fn resyncs(effects: &[Effect]) -> Vec<(usize, &str)> {
|
||||
effects
|
||||
.iter()
|
||||
.filter_map(|effect| match effect {
|
||||
Effect::Send(connection, Command::Resubscribe(ticker)) => {
|
||||
Some((*connection, ticker.as_str()))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A market with the same book from both connections.
|
||||
fn both_on(ticker: &str) -> Feed {
|
||||
let mut feed = feed_of(&[ticker], &[0, 1]);
|
||||
let first = feed.from(0, 1, book(ticker, 10, &[(400_000, 5_000_000)], &[]));
|
||||
assert_eq!(snapshots(&first), 1);
|
||||
let second = feed.from(1, 2, book(ticker, 10, &[(400_000, 5_000_000)], &[]));
|
||||
assert!(second.is_empty(), "{second:?}");
|
||||
feed
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_change_is_written_once_from_whichever_delivers_it_first() {
|
||||
let mut feed = both_on("M");
|
||||
let first = feed.from(1, 5, change("M", 11, Side::Buy, 400_000, 2_000_000));
|
||||
let written = deltas(&first);
|
||||
assert_eq!(written.len(), 1);
|
||||
// The absolute quantity, at the first copy's receive time.
|
||||
assert_eq!(written[0].quantity, 7_000_000);
|
||||
assert_eq!(written[0].backtest_timestamp, at(5));
|
||||
assert!(
|
||||
deltas(&feed.from(0, 6, change("M", 11, Side::Buy, 400_000, 2_000_000))).is_empty()
|
||||
);
|
||||
assert_eq!(feed.rule.counts.duplicates, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_change_of_one_clock_is_applied_once() {
|
||||
let mut feed = both_on("M");
|
||||
// Two identical changes share a clock: both are real.
|
||||
let twice = [
|
||||
change("M", 11, Side::Sell, 600_000, 1_000_000),
|
||||
change("M", 11, Side::Sell, 600_000, 1_000_000),
|
||||
];
|
||||
let mut written = Vec::new();
|
||||
for event in twice {
|
||||
written.extend(feed.from(0, 5, event));
|
||||
}
|
||||
let quantities: Vec<i64> = deltas(&written)
|
||||
.iter()
|
||||
.map(|delta| delta.quantity)
|
||||
.collect();
|
||||
assert_eq!(quantities, [1_000_000, 2_000_000]);
|
||||
for event in [
|
||||
change("M", 11, Side::Sell, 600_000, 1_000_000),
|
||||
change("M", 11, Side::Sell, 600_000, 1_000_000),
|
||||
] {
|
||||
assert!(deltas(&feed.from(1, 6, event)).is_empty());
|
||||
}
|
||||
// The second connection's third change of that clock is new.
|
||||
let third = feed.from(1, 7, change("M", 11, Side::Sell, 600_000, 1_000_000));
|
||||
assert_eq!(deltas(&third)[0].quantity, 3_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_connection_counts_for_a_symbol_only_from_its_book() {
|
||||
let mut feed = feed_of(&["M"], &[0, 1]);
|
||||
feed.from(0, 1, book("M", 10, &[], &[]));
|
||||
// Connection 1 hasn't delivered its book: its change has no place.
|
||||
assert!(feed.from(1, 2, change("M", 11, Side::Buy, 1, 1)).is_empty());
|
||||
assert_eq!(feed.rule.counts.unplaced, 1);
|
||||
// Down and up again, connection 0 too waits for its book.
|
||||
feed.on_received(Received::Down(0));
|
||||
feed.on_received(Received::Up(0));
|
||||
assert!(feed.from(0, 3, change("M", 12, Side::Buy, 1, 1)).is_empty());
|
||||
assert_eq!(feed.rule.counts.unplaced, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fresh_book_replaces_ours_only_at_or_past_it_and_different() {
|
||||
let mut feed = both_on("M");
|
||||
feed.from(0, 3, change("M", 11, Side::Buy, 400_000, 1_000_000));
|
||||
// Older than ours: connection 1 is only behind.
|
||||
assert!(
|
||||
feed.from(1, 4, book("M", 10, &[(400_000, 5_000_000)], &[]))
|
||||
.is_empty()
|
||||
);
|
||||
// At our clock and the same: nothing.
|
||||
assert!(
|
||||
feed.from(1, 5, book("M", 11, &[(400_000, 6_000_000)], &[]))
|
||||
.is_empty()
|
||||
);
|
||||
// Past ours and the same: nothing either.
|
||||
assert!(
|
||||
feed.from(1, 6, book("M", 15, &[(400_000, 6_000_000)], &[]))
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(feed.rule.counts.losses, 0);
|
||||
// Past ours and different: it replaces ours, as a Snapshot.
|
||||
let different = feed.from(1, 7, book("M", 20, &[(400_000, 1_000_000)], &[]));
|
||||
assert_eq!(snapshots(&different), 1);
|
||||
assert_eq!(feed.rule.counts.losses, 1);
|
||||
assert_eq!(feed.shell.symbols["M"].book.bids[&400_000], 1_000_000);
|
||||
// Its changes up to that book are duplicates now.
|
||||
assert!(deltas(&feed.from(0, 8, change("M", 16, Side::Buy, 400_000, 1))).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_level_below_zero_resyncs_and_the_answer_replaces_ours() {
|
||||
let mut feed = both_on("M");
|
||||
let effects = feed.from(0, 3, change("M", 11, Side::Buy, 400_000, -6_000_000));
|
||||
assert_eq!(deltas(&effects)[0].quantity, 0);
|
||||
assert_eq!(resyncs(&effects), [(0, "M")]);
|
||||
assert_eq!(feed.rule.counts.below_zero, 1);
|
||||
assert!(!Clock::may_be_quiet(&feed.shell.symbols["M"].rule));
|
||||
// The answer replaces ours even behind it, and differing is a loss.
|
||||
let answer = feed.from(0, 4, book("M", 9, &[(400_000, 9_000_000)], &[]));
|
||||
assert_eq!(snapshots(&answer), 1);
|
||||
assert_eq!(feed.shell.symbols["M"].book.bids[&400_000], 9_000_000);
|
||||
assert_eq!(feed.shell.symbols["M"].rule.asked, None);
|
||||
// The stream that answered goes on from its book.
|
||||
let after = feed.from(0, 5, change("M", 10, Side::Buy, 400_000, 1_000_000));
|
||||
assert_eq!(deltas(&after)[0].quantity, 10_000_000);
|
||||
}
|
||||
|
||||
/// The answer to a resync can be behind what the other connection has
|
||||
/// delivered: the book goes back to it, and only a stream that continues
|
||||
/// from there is taken until the other fits again.
|
||||
#[test]
|
||||
fn after_an_answer_behind_ours_a_connection_ahead_waits_for_the_book() {
|
||||
let mut feed = both_on("M");
|
||||
feed.from(0, 3, change("M", 11, Side::Buy, 400_000, -6_000_000));
|
||||
feed.from(1, 4, change("M", 11, Side::Buy, 400_000, -6_000_000));
|
||||
feed.from(1, 5, change("M", 12, Side::Buy, 400_000, 1_000_000));
|
||||
// Connection 0's answer holds clock 11, not yet 12.
|
||||
let answer = feed.from(0, 6, book("M", 11, &[(400_000, 2_000_000)], &[]));
|
||||
assert_eq!(snapshots(&answer), 1);
|
||||
// Connection 1 is past it: its next change would skip clock 12.
|
||||
let ahead = feed.from(1, 7, change("M", 13, Side::Buy, 400_000, 1_000_000));
|
||||
assert!(deltas(&ahead).is_empty(), "{ahead:?}");
|
||||
let twelve = feed.from(0, 8, change("M", 12, Side::Buy, 400_000, 1_000_000));
|
||||
assert_eq!(deltas(&twelve)[0].quantity, 3_000_000);
|
||||
let thirteen = feed.from(0, 9, change("M", 13, Side::Buy, 400_000, 1_000_000));
|
||||
assert_eq!(deltas(&thirteen)[0].quantity, 4_000_000);
|
||||
// The book has caught up with connection 1: it counts again.
|
||||
let fourteen = feed.from(1, 10, change("M", 14, Side::Buy, 400_000, 1_000_000));
|
||||
assert_eq!(deltas(&fourteen)[0].quantity, 5_000_000);
|
||||
assert_eq!(feed.rule.counts.ahead, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_first_book_asked_from_one_connection_can_come_from_the_other() {
|
||||
let mut feed = feed_of(&["N"], &[0, 1]);
|
||||
feed.on_received(Received::Up(0));
|
||||
feed.from(0, 5, change("N", 1, Side::Buy, 1, 1));
|
||||
assert_eq!(resyncs(&feed.tick(next_minute(at(10)))), [(0, "N")]);
|
||||
let first = feed.from(1, 61_000, book("N", 20, &[(400_000, 5_000_000)], &[]));
|
||||
assert_eq!(snapshots(&first), 1);
|
||||
// Connection 0's answer, older and different, is only where it stands.
|
||||
let answer = feed.from(0, 61_001, book("N", 19, &[(400_000, 4_000_000)], &[]));
|
||||
assert!(answer.is_empty(), "{answer:?}");
|
||||
assert_eq!(feed.rule.counts.losses, 0);
|
||||
assert_eq!(feed.shell.symbols["N"].book.bids[&400_000], 5_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_symbol_no_connection_delivers_is_asked_for_at_the_minute() {
|
||||
let mut feed = both_on("M");
|
||||
feed.on_received(Received::Down(0));
|
||||
feed.on_received(Received::Down(1));
|
||||
feed.on_received(Received::Up(0));
|
||||
feed.on_received(Received::Up(1));
|
||||
// Its books didn't come with the reconnects.
|
||||
assert_eq!(resyncs(&feed.tick(next_minute(at(10)))), [(0, "M")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_resync_asked_from_a_connection_that_leaves_is_asked_from_another() {
|
||||
let mut feed = both_on("M");
|
||||
feed.from(0, 3, change("M", 11, Side::Buy, 400_000, -6_000_000));
|
||||
let market = Market { ticker: "M".into() };
|
||||
let moved = feed.move_market(&market, 0, 2, at(4));
|
||||
assert!(resyncs(&moved).is_empty());
|
||||
// Connection 2's book ends the move: connection 0 leaves.
|
||||
let ended = feed.from(2, 5, book("M", 12, &[], &[]));
|
||||
assert!(ended.iter().any(|effect| matches!(
|
||||
effect,
|
||||
Effect::Send(0, Command::Unsubscribe(tokens)) if tokens == &["M".to_string()]
|
||||
)));
|
||||
// It came from connection 2, not the one asked: not the answer.
|
||||
assert_eq!(resyncs(&ended), [(1, "M")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fresh_book_still_not_come_is_asked_again_at_the_minute() {
|
||||
let mut feed = both_on("M");
|
||||
feed.from(0, 3, change("M", 11, Side::Buy, 400_000, -6_000_000));
|
||||
let minute = feed.tick(next_minute(at(10)));
|
||||
assert_eq!(resyncs(&minute), [(0, "M")]);
|
||||
// A symbol with no book at all is asked for one too.
|
||||
let mut none = feed_of(&["N"], &[0]);
|
||||
none.on_received(Received::Up(0));
|
||||
// An event starts the book thread's minutes.
|
||||
none.from(0, 5, change("N", 1, Side::Buy, 1, 1));
|
||||
assert_eq!(resyncs(&none.tick(next_minute(at(10)))), [(0, "N")]);
|
||||
// One in order isn't.
|
||||
let mut fine = both_on("F");
|
||||
assert!(resyncs(&fine.tick(next_minute(at(10)))).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_minute_snapshot_while_no_connection_delivers_the_symbol() {
|
||||
let mut feed = both_on("M");
|
||||
let minute = next_minute(at(10));
|
||||
assert_eq!(snapshots(&feed.tick(minute)), 1);
|
||||
feed.on_received(Received::Down(0));
|
||||
feed.on_received(Received::Down(1));
|
||||
assert_eq!(snapshots(&feed.tick(minute + TimeDelta::minutes(1))), 0);
|
||||
// Back, its book compared: the same, so no Snapshot until the minute.
|
||||
assert!(
|
||||
feed.from(1, 70_000, book("M", 30, &[(400_000, 5_000_000)], &[]))
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(snapshots(&feed.tick(minute + TimeDelta::minutes(2))), 1);
|
||||
}
|
||||
|
||||
/// The demo's recording on two connections (tests/fixtures/README.md).
|
||||
fn recording() -> Vec<(DateTime<Utc>, usize, Event)> {
|
||||
#[derive(Deserialize)]
|
||||
struct Line {
|
||||
received_at_us: i64,
|
||||
connection: usize,
|
||||
text: String,
|
||||
}
|
||||
let path = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/kalshi-frames-two.jsonl"
|
||||
);
|
||||
std::fs::read_to_string(path)
|
||||
.unwrap()
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let line: Line = serde_json::from_str(line).unwrap();
|
||||
let frame: wire::Frame = serde_json::from_str(&line.text).unwrap();
|
||||
let event = wire::event(&frame).unwrap()?;
|
||||
let received_at = DateTime::from_timestamp_micros(line.received_at_us).unwrap();
|
||||
Some((received_at, line.connection, event))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn tickers(recording: &[(DateTime<Utc>, usize, Event)]) -> Vec<String> {
|
||||
let mut tickers: Vec<String> = recording
|
||||
.iter()
|
||||
.map(|(_, _, event)| crate::core::Event::market(event).unwrap().to_string())
|
||||
.collect();
|
||||
tickers.sort();
|
||||
tickers.dedup();
|
||||
tickers
|
||||
}
|
||||
|
||||
/// Each symbol's Deltas, as (side, price, quantity, receive time).
|
||||
type Written = HashMap<String, Vec<(Side, i64, i64, DateTime<Utc>)>>;
|
||||
|
||||
/// Replays the connections in `on`.
|
||||
fn replay(on: &[usize]) -> (Feed, Written) {
|
||||
let recording = recording();
|
||||
let tickers = tickers(&recording);
|
||||
let tickers: Vec<&str> = tickers.iter().map(String::as_str).collect();
|
||||
let mut feed = feed_of(&tickers, on);
|
||||
let mut written: HashMap<_, Vec<_>> = HashMap::new();
|
||||
for (received_at, connection, event) in recording {
|
||||
if !on.contains(&connection) {
|
||||
continue;
|
||||
}
|
||||
let arrival = Arrival {
|
||||
received_at,
|
||||
connection,
|
||||
bytes: 0,
|
||||
event,
|
||||
};
|
||||
for delta in deltas(&feed.on_event(arrival)) {
|
||||
written.entry(delta.symbol.clone()).or_default().push((
|
||||
delta.side,
|
||||
delta.price_level,
|
||||
delta.quantity,
|
||||
delta.backtest_timestamp,
|
||||
));
|
||||
}
|
||||
}
|
||||
(feed, written)
|
||||
}
|
||||
|
||||
/// Connection 0 asked for every book again every 3 s: each, compared
|
||||
/// with ours folded from the changes, was the same.
|
||||
#[test]
|
||||
fn the_recorded_changes_fold_into_the_exchanges_books() {
|
||||
let (feed, written) = replay(&[0]);
|
||||
let counts = &feed.rule.counts;
|
||||
assert!(counts.books > 700, "{counts:?}");
|
||||
assert_eq!(
|
||||
(counts.losses, counts.below_zero, counts.duplicates),
|
||||
(0, 0, 0)
|
||||
);
|
||||
assert!(written.values().map(Vec::len).sum::<usize>() > 1_000);
|
||||
// A NO bid arrived as the ask it is: no book is crossed. Taken at
|
||||
// one minus its price again, a third of them would be.
|
||||
let mut two_sided = 0;
|
||||
for (ticker, symbol) in &feed.shell.symbols {
|
||||
let best_bid = symbol.book.bids.keys().next_back();
|
||||
let best_ask = symbol.book.asks.keys().next();
|
||||
if let (Some(bid), Some(ask)) = (best_bid, best_ask) {
|
||||
assert!(bid < ask, "{ticker}: bid {bid}, ask {ask}");
|
||||
two_sided += 1;
|
||||
}
|
||||
}
|
||||
assert!(two_sided >= 10, "{two_sided}");
|
||||
}
|
||||
|
||||
/// The book and writer threads as main runs them, fed the recording:
|
||||
/// every Snapshot is the one before it with the Deltas between.
|
||||
#[test]
|
||||
fn the_output_reconciles() {
|
||||
let recording = recording();
|
||||
let tickers = tickers(&recording);
|
||||
let tickers: Vec<&str> = tickers.iter().map(String::as_str).collect();
|
||||
let feed = feed_of(&tickers, &[0, 1]);
|
||||
let (events_tx, events_rx) = crossbeam::channel::unbounded();
|
||||
let (outputs_tx, outputs_rx) = crossbeam::channel::unbounded();
|
||||
let (connection_tx, _connection_rx) = crossbeam::channel::unbounded();
|
||||
let (_commands_tx, commands_rx) = crossbeam::channel::unbounded();
|
||||
let (reports_tx, _reports_rx) = crossbeam::channel::unbounded();
|
||||
for (received_at, connection, event) in recording {
|
||||
let arrival = Arrival {
|
||||
received_at,
|
||||
connection,
|
||||
bytes: 0,
|
||||
event,
|
||||
};
|
||||
events_tx.send(Received::Event(arrival)).unwrap();
|
||||
}
|
||||
drop(events_tx);
|
||||
let book = std::thread::spawn(move || {
|
||||
feed::serve(
|
||||
feed,
|
||||
vec![connection_tx.clone(), connection_tx],
|
||||
events_rx,
|
||||
commands_rx,
|
||||
outputs_tx,
|
||||
reports_tx,
|
||||
)
|
||||
});
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("kalshi-reconcile-{}.sbe", std::process::id()));
|
||||
let written = crate::write(path.to_str().unwrap(), outputs_rx);
|
||||
book.join().unwrap();
|
||||
written.unwrap();
|
||||
let report = engineering_challenge::reconcile::reconcile(&path, None);
|
||||
std::fs::remove_file(&path).unwrap();
|
||||
|
||||
let report = report.unwrap();
|
||||
// The opening Snapshots, and one per symbol at the minute crossed.
|
||||
assert_eq!(report.snapshots, tickers.len() * 2, "{report:?}");
|
||||
assert!(report.deltas > 1_000, "{report:?}");
|
||||
assert_eq!(report.orphan_deltas, 0);
|
||||
assert_eq!(report.mismatches, []);
|
||||
}
|
||||
|
||||
/// Connection 1 left five markets for 5 s: connection 0 carried them.
|
||||
#[test]
|
||||
fn two_recorded_connections_write_the_changes_of_one_each_at_its_first_copy() {
|
||||
let (one, alone) = replay(&[0]);
|
||||
let (both, together) = replay(&[0, 1]);
|
||||
assert_eq!(both.rule.counts.losses, 0);
|
||||
assert!(both.rule.counts.duplicates > 900, "{:?}", both.rule.counts);
|
||||
assert_eq!(alone.keys().len(), together.keys().len());
|
||||
let mut earlier = 0;
|
||||
for (symbol, changes) in &alone {
|
||||
let content = |changes: &[(Side, i64, i64, DateTime<Utc>)]| {
|
||||
changes
|
||||
.iter()
|
||||
.map(|&(side, price, quantity, _)| (side, price, quantity))
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
assert_eq!(content(changes), content(&together[symbol]), "{symbol}");
|
||||
for (single, first) in changes.iter().zip(&together[symbol]) {
|
||||
assert!(first.3 <= single.3);
|
||||
earlier += usize::from(first.3 < single.3);
|
||||
}
|
||||
}
|
||||
// Connection 1 was first for most changes in the recording.
|
||||
assert!(earlier > 500, "{earlier}");
|
||||
assert_eq!(one.rule.counts.written, both.rule.counts.written);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
//! Kalshi's market listing: which markets are live, or about to be. Two fetcher threads page
|
||||
//! it on the orchestrator's request and stream what they read; they hold no
|
||||
//! state and no timer, as Gamma's (roadmap/06). Read without a key.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use crossbeam::channel::{Receiver, Sender};
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::Market;
|
||||
use crate::core::{Finding, Request, retried};
|
||||
|
||||
/// The listing's largest page.
|
||||
const PAGE_SIZE: &str = "1000";
|
||||
/// For every request, so SIGINT never waits on the listing longer than this.
|
||||
const TIMEOUT: Duration = Duration::from_secs(10);
|
||||
/// A page's tries, and the wait between them.
|
||||
const TRIES: u32 = 5;
|
||||
const RETRY_AFTER: Duration = Duration::from_secs(5);
|
||||
/// A market not yet open is tracked this long before its opening: its book is
|
||||
/// there, empty, from its first order. Three hourly rescans, so every market
|
||||
/// is reached by a scan before it opens even when one scan fails.
|
||||
const OPENING_AHEAD: TimeDelta = TimeDelta::hours(3);
|
||||
|
||||
/// A fetcher thread: answers each request until the orchestrator is gone.
|
||||
pub fn run(listing: &str, requests: Receiver<Request>, findings: Sender<Finding<Market>>) {
|
||||
let agent = ureq::Agent::new_with_config(
|
||||
ureq::Agent::config_builder()
|
||||
.timeout_global(Some(TIMEOUT))
|
||||
.build(),
|
||||
);
|
||||
let markets = format!("{listing}/markets");
|
||||
for request in requests {
|
||||
let sent = match request {
|
||||
Request::Rescan => rescan(&agent, &markets, &findings),
|
||||
Request::Recent { since } => recent(&agent, &markets, since, &findings),
|
||||
};
|
||||
if sent.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `Err` once the orchestrator is gone.
|
||||
type Sent = Result<(), crossbeam::channel::SendError<Finding<Market>>>;
|
||||
|
||||
fn rescan(agent: &ureq::Agent, markets: &str, findings: &Sender<Finding<Market>>) -> Sent {
|
||||
findings.send(Finding::ScanStarted)?;
|
||||
// Unopened first: a market that opens while the scan runs is then on one
|
||||
// list at least. Open first, it could be on neither, and be removed.
|
||||
for status in ["unopened", "open"] {
|
||||
let query = [("status", status.to_string()), mve_excluded()];
|
||||
let mut cursor = String::new();
|
||||
loop {
|
||||
let page = match retried(TRIES, RETRY_AFTER, || page(agent, markets, &query, &cursor)) {
|
||||
Ok(page) => page,
|
||||
Err(error) => {
|
||||
log!("kalshi: rescan failed: {error}");
|
||||
return findings.send(Finding::ScanFailed);
|
||||
}
|
||||
};
|
||||
let last = page.markets.is_empty() || page.cursor.is_empty();
|
||||
findings.send(Finding::ScanPage(tracked(page.markets, Utc::now())))?;
|
||||
if last {
|
||||
break;
|
||||
}
|
||||
cursor = page.cursor;
|
||||
}
|
||||
}
|
||||
findings.send(Finding::ScanDone)
|
||||
}
|
||||
|
||||
fn recent(
|
||||
agent: &ureq::Agent,
|
||||
markets: &str,
|
||||
since: DateTime<Utc>,
|
||||
findings: &Sender<Finding<Market>>,
|
||||
) -> Sent {
|
||||
let since = since.timestamp().to_string();
|
||||
// Created, whatever their status: open and unopened alike.
|
||||
let created = [("min_created_ts", since.clone()), mve_excluded()];
|
||||
let closed = [
|
||||
("status", "closed".to_string()),
|
||||
("min_close_ts", since),
|
||||
mve_excluded(),
|
||||
];
|
||||
let result = every_page(agent, markets, &created).and_then(|created| {
|
||||
findings.send(Finding::Created(tracked(created, Utc::now())))?;
|
||||
every_page(agent, markets, &closed)
|
||||
});
|
||||
match result {
|
||||
Ok(closed) => {
|
||||
let tickers = closed.into_iter().map(|market| market.ticker);
|
||||
findings.send(Finding::Closed(tickers.collect()))?;
|
||||
findings.send(Finding::RecentDone)
|
||||
}
|
||||
Err(error) => {
|
||||
log!("kalshi: recent check failed: {error}");
|
||||
findings.send(Finding::RecentFailed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Combos (MVEs) are not in the challenge.
|
||||
fn mve_excluded() -> (&'static str, String) {
|
||||
("mve_filter", "exclude".to_string())
|
||||
}
|
||||
|
||||
fn every_page(
|
||||
agent: &ureq::Agent,
|
||||
markets: &str,
|
||||
query: &[(&str, String)],
|
||||
) -> anyhow::Result<Vec<ListedMarket>> {
|
||||
let mut listed = Vec::new();
|
||||
let mut cursor = String::new();
|
||||
loop {
|
||||
let page = retried(TRIES, RETRY_AFTER, || page(agent, markets, query, &cursor))?;
|
||||
let last = page.markets.is_empty() || page.cursor.is_empty();
|
||||
listed.extend(page.markets);
|
||||
if last {
|
||||
return Ok(listed);
|
||||
}
|
||||
cursor = page.cursor;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Page {
|
||||
markets: Vec<ListedMarket>,
|
||||
/// Empty on the last page.
|
||||
#[serde(default)]
|
||||
cursor: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ListedMarket {
|
||||
ticker: String,
|
||||
/// `active` once open, `initialized` before.
|
||||
status: String,
|
||||
open_time: Option<String>,
|
||||
}
|
||||
|
||||
fn page(
|
||||
agent: &ureq::Agent,
|
||||
markets: &str,
|
||||
query: &[(&str, String)],
|
||||
cursor: &str,
|
||||
) -> anyhow::Result<Page> {
|
||||
let mut request = agent
|
||||
.get(markets)
|
||||
.query("limit", PAGE_SIZE)
|
||||
.query_pairs(query.iter().map(|(name, value)| (*name, value.as_str())));
|
||||
if !cursor.is_empty() {
|
||||
request = request.query("cursor", cursor);
|
||||
}
|
||||
let body = request.call()?.body_mut().read_to_string()?;
|
||||
Ok(serde_json::from_str(&body)?)
|
||||
}
|
||||
|
||||
/// The markets to track among `markets` at `now`: those live, and those
|
||||
/// opening within [`OPENING_AHEAD`].
|
||||
fn tracked(markets: Vec<ListedMarket>, now: DateTime<Utc>) -> Vec<Market> {
|
||||
markets
|
||||
.into_iter()
|
||||
.filter(|market| match market.status.as_str() {
|
||||
"active" => true,
|
||||
"initialized" => market
|
||||
.open_time
|
||||
.as_deref()
|
||||
.and_then(|time| DateTime::parse_from_rfc3339(time).ok())
|
||||
.is_some_and(|opens| opens.to_utc() - now <= OPENING_AHEAD),
|
||||
_ => false,
|
||||
})
|
||||
.map(|market| Market {
|
||||
ticker: market.ticker,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn fixture(name: &str) -> Page {
|
||||
let path = format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"));
|
||||
serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()
|
||||
}
|
||||
|
||||
fn time(text: &str) -> DateTime<Utc> {
|
||||
DateTime::parse_from_rfc3339(text).unwrap().to_utc()
|
||||
}
|
||||
|
||||
fn tickers(markets: &[Market]) -> Vec<&str> {
|
||||
markets
|
||||
.iter()
|
||||
.map(|market| market.ticker.as_str())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_markets_are_tracked_and_the_last_page_has_no_cursor() {
|
||||
let page = fixture("kalshi-open.json");
|
||||
assert!(!page.cursor.is_empty());
|
||||
let live = tracked(page.markets, time("2026-11-11T19:30:00Z"));
|
||||
assert_eq!(live.len(), 3);
|
||||
assert!(live[0].ticker.starts_with("KX"));
|
||||
let last: Page = serde_json::from_str(r#"{"markets":[],"cursor":""}"#).unwrap();
|
||||
assert!(last.cursor.is_empty());
|
||||
}
|
||||
|
||||
/// The fixture's unopened markets open at 19:19:08.
|
||||
#[test]
|
||||
fn an_unopened_market_is_tracked_from_three_hours_before_its_opening() {
|
||||
let opening = time("2026-11-11T19:19:08Z");
|
||||
let at = |now: DateTime<Utc>| tracked(fixture("kalshi-unopened.json").markets, now);
|
||||
assert!(at(opening - OPENING_AHEAD - TimeDelta::seconds(1)).is_empty());
|
||||
assert_eq!(at(opening - OPENING_AHEAD).len(), 3);
|
||||
// Past its opening and still listed unopened: tracked, empty.
|
||||
assert_eq!(
|
||||
tickers(&at(opening + TimeDelta::minutes(20)))[0],
|
||||
"KXEUROCUPTOTAL-26NOV121400MAXSKY-179"
|
||||
);
|
||||
}
|
||||
|
||||
/// A local listing: answers each request with the fixture its query asks
|
||||
/// for (an empty last page after a cursor), and tells the queries it got.
|
||||
fn listing(requests: usize) -> (String, std::thread::JoinHandle<Vec<String>>) {
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let server = std::thread::spawn(move || {
|
||||
let mut queries = Vec::new();
|
||||
for _ in 0..requests {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
let mut line = String::new();
|
||||
BufReader::new(&stream).read_line(&mut line).unwrap();
|
||||
let query = line.split(' ').nth(1).unwrap().to_string();
|
||||
let name = if query.contains("cursor=") {
|
||||
None
|
||||
} else if query.contains("status=unopened") {
|
||||
Some("kalshi-unopened.json")
|
||||
} else if query.contains("status=closed") {
|
||||
Some("kalshi-closed.json")
|
||||
} else {
|
||||
Some("kalshi-open.json")
|
||||
};
|
||||
let body = name.map_or(r#"{"markets":[],"cursor":""}"#.to_string(), |name| {
|
||||
let path = format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"));
|
||||
std::fs::read_to_string(path).unwrap()
|
||||
});
|
||||
write!(
|
||||
stream,
|
||||
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
.unwrap();
|
||||
queries.push(query);
|
||||
}
|
||||
queries
|
||||
});
|
||||
(url, server)
|
||||
}
|
||||
|
||||
fn found(findings: &[Finding<Market>]) -> Vec<String> {
|
||||
findings
|
||||
.iter()
|
||||
.map(|finding| match finding {
|
||||
Finding::ScanPage(markets) => format!("page {}", tickers(markets).join(" ")),
|
||||
Finding::Created(markets) => format!("created {}", tickers(markets).join(" ")),
|
||||
Finding::Closed(closed) => format!("closed {}", closed.join(" ")),
|
||||
other => format!("{other:?}"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rescan_pages_the_unopened_markets_then_the_open_ones() {
|
||||
let (url, server) = listing(4);
|
||||
let (requests_tx, requests) = crossbeam::channel::unbounded();
|
||||
let (findings_tx, findings) = crossbeam::channel::unbounded();
|
||||
requests_tx.send(Request::Rescan).unwrap();
|
||||
drop(requests_tx);
|
||||
run(&url, requests, findings_tx);
|
||||
let queries = server.join().unwrap();
|
||||
// Unopened first: a market opening meanwhile is on one list at least.
|
||||
assert!(queries[0].contains("status=unopened"), "{queries:?}");
|
||||
assert!(queries[1].contains("status=unopened") && queries[1].contains("cursor="));
|
||||
assert!(queries[2].contains("status=open") && !queries[2].contains("cursor="));
|
||||
for query in &queries {
|
||||
assert!(query.contains("mve_filter=exclude") && query.contains("limit=1000"));
|
||||
}
|
||||
let found = found(&findings.try_iter().collect::<Vec<_>>());
|
||||
assert_eq!(found.len(), 6, "{found:?}");
|
||||
assert_eq!(found[0], "ScanStarted");
|
||||
assert!(found[1].starts_with("page KXEUROCUPTOTAL-26NOV121400MAXSKY-179 "));
|
||||
assert_eq!(found[2], "page ");
|
||||
assert!(found[3].starts_with("page KXNCAAWBTOTAL-26NOV11QUINSJU-106 "));
|
||||
assert_eq!(found[5], "ScanDone");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_recent_check_reports_the_markets_created_and_those_closed() {
|
||||
let (url, server) = listing(4);
|
||||
let (requests_tx, requests) = crossbeam::channel::unbounded();
|
||||
let (findings_tx, findings) = crossbeam::channel::unbounded();
|
||||
let since = time("2026-11-11T19:00:00Z");
|
||||
requests_tx.send(Request::Recent { since }).unwrap();
|
||||
drop(requests_tx);
|
||||
run(&url, requests, findings_tx);
|
||||
let queries = server.join().unwrap();
|
||||
let since = format!("={}", since.timestamp());
|
||||
assert!(
|
||||
queries[0].contains(&format!("min_created_ts{since}"))
|
||||
&& !queries[0].contains("status=")
|
||||
);
|
||||
assert!(
|
||||
queries[2].contains(&format!("min_close_ts{since}"))
|
||||
&& queries[2].contains("status=closed")
|
||||
);
|
||||
let found = found(&findings.try_iter().collect::<Vec<_>>());
|
||||
assert_eq!(found.len(), 3, "{found:?}");
|
||||
assert!(found[0].starts_with("created KXNCAAWBTOTAL-26NOV11QUINSJU-106 "));
|
||||
assert!(found[1].starts_with("closed KXFIBAECUPSPREAD-26NOV111230SZOZIG-SZO23 "));
|
||||
assert_eq!(found[2], "RecentDone");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_closed_market_is_not_tracked() {
|
||||
let closed = fixture("kalshi-closed.json");
|
||||
assert_eq!(closed.markets[0].status, "determined");
|
||||
assert!(tracked(closed.markets, time("2026-11-11T19:30:00Z")).is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
//! Kalshi: its websocket's `orderbook_delta` channel (the connection's side of
|
||||
//! it, signed with an API key), its market listing, and the clock rule its
|
||||
//! book threads apply (docs/notes.md, Kalshi's websocket; roadmap/06).
|
||||
|
||||
pub mod feed;
|
||||
pub mod listing;
|
||||
pub mod sign;
|
||||
pub mod wire;
|
||||
|
||||
use chrono::Utc;
|
||||
use crossbeam::channel::{Receiver, Sender};
|
||||
use serde_json::json;
|
||||
use tungstenite::client::IntoClientRequest;
|
||||
|
||||
use crate::core::{self, Finding, Request, Undecoded};
|
||||
use sign::Key;
|
||||
|
||||
/// Tickers per subscribe or unsubscribe message.
|
||||
const CHUNK: usize = 500;
|
||||
/// Errors after which the subscription is gone (Kalshi's docs: a channel
|
||||
/// error, a subscription buffer overflow): the connection starts over.
|
||||
const TERMINAL_ERRORS: [u32; 2] = [10, 25];
|
||||
|
||||
/// A tracked market: one book, named by its ticker, which is its id and its
|
||||
/// only symbol.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Market {
|
||||
pub ticker: String,
|
||||
}
|
||||
|
||||
impl core::Market for Market {
|
||||
fn id(&self) -> &str {
|
||||
&self.ticker
|
||||
}
|
||||
|
||||
fn symbols(&self) -> &[String] {
|
||||
std::slice::from_ref(&self.ticker)
|
||||
}
|
||||
}
|
||||
|
||||
/// The exchange: its websocket and listing, and the key that signs the
|
||||
/// websocket's handshake.
|
||||
pub struct Kalshi {
|
||||
pub websocket: String,
|
||||
pub listing: String,
|
||||
pub key: Key,
|
||||
}
|
||||
|
||||
impl Kalshi {
|
||||
pub fn production(key: Key) -> Self {
|
||||
Self {
|
||||
websocket: "wss://external-api-ws.kalshi.com/trade-api/ws/v2".into(),
|
||||
listing: "https://external-api.kalshi.com/trade-api/v2".into(),
|
||||
key,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn demo(key: Key) -> Self {
|
||||
Self {
|
||||
websocket: "wss://external-api-ws.demo.kalshi.co/trade-api/ws/v2".into(),
|
||||
listing: "https://external-api.demo.kalshi.co/trade-api/v2".into(),
|
||||
key,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl core::Exchange for Kalshi {
|
||||
type Market = Market;
|
||||
type Event = wire::Event;
|
||||
type Session = Session;
|
||||
type Rule = feed::Clock;
|
||||
|
||||
const NAME: &'static str = "kalshi";
|
||||
|
||||
fn request(&self) -> anyhow::Result<tungstenite::handshake::client::Request> {
|
||||
let mut request = self.websocket.as_str().into_client_request()?;
|
||||
for (name, value) in self.key.headers(Utc::now().timestamp_millis()) {
|
||||
request.headers_mut().insert(name, value.parse()?);
|
||||
}
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
fn session(&self) -> Session {
|
||||
Session::default()
|
||||
}
|
||||
|
||||
fn rule(&self) -> feed::Clock {
|
||||
feed::Clock::default()
|
||||
}
|
||||
|
||||
fn fetch(&self, requests: Receiver<Request>, findings: Sender<Finding<Market>>) {
|
||||
listing::run(&self.listing, requests, findings)
|
||||
}
|
||||
}
|
||||
|
||||
/// A connection's subscription: one per connection, which every subscribe
|
||||
/// adds to.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Session {
|
||||
/// The last command's id.
|
||||
command: u64,
|
||||
/// The subscription's id, from its `subscribed` answer.
|
||||
sid: Option<u64>,
|
||||
/// The last message's `seq`: the next one must follow it.
|
||||
seq: Option<u64>,
|
||||
}
|
||||
|
||||
impl Session {
|
||||
fn command(&mut self, cmd: &str, params: serde_json::Value) -> String {
|
||||
self.command += 1;
|
||||
json!({"id": self.command, "cmd": cmd, "params": params}).to_string()
|
||||
}
|
||||
}
|
||||
|
||||
impl core::Session for Session {
|
||||
type Event = wire::Event;
|
||||
|
||||
/// Nothing: a subscribe can't name no market, so the first one creates the
|
||||
/// subscription.
|
||||
fn opening(&mut self) -> Vec<String> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// `use_yes_price` makes a NO bid come as the ask it is; sent, since its
|
||||
/// default is announced to flip.
|
||||
fn subscribe(&mut self, tickers: &[&String]) -> Vec<String> {
|
||||
tickers
|
||||
.chunks(CHUNK)
|
||||
.map(|chunk| {
|
||||
let params = json!({
|
||||
"channels": ["orderbook_delta"],
|
||||
"market_tickers": chunk,
|
||||
"use_yes_price": true,
|
||||
"skip_ticker_ack": true,
|
||||
});
|
||||
self.command("subscribe", params)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn unsubscribe(&mut self, tickers: &[&String]) -> Vec<String> {
|
||||
// Every fresh connection's subscription was 1 (docs/notes.md): an
|
||||
// unsubscribe sent before its answer names it so.
|
||||
let sid = self.sid.unwrap_or(1);
|
||||
tickers
|
||||
.chunks(CHUNK)
|
||||
.map(|chunk| {
|
||||
let params = json!({
|
||||
"sids": [sid],
|
||||
"market_tickers": chunk,
|
||||
"action": "delete_markets",
|
||||
"skip_ticker_ack": true,
|
||||
});
|
||||
self.command("update_subscription", params)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// None: Kalshi pings a connection it has nothing else to send to, and
|
||||
/// tungstenite answers.
|
||||
fn keepalive(&self) -> Option<&'static str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn decode(&mut self, text: &str) -> Result<Vec<wire::Event>, Undecoded> {
|
||||
let frame: wire::Frame =
|
||||
serde_json::from_str(text).map_err(|error| Undecoded::Unreadable(error.to_string()))?;
|
||||
if let Some(seq) = frame.seq {
|
||||
let expected = self.seq.map_or(1, |last| last + 1);
|
||||
if seq != expected {
|
||||
return Err(Undecoded::Lost(format!(
|
||||
"message {seq} came, {expected} expected"
|
||||
)));
|
||||
}
|
||||
self.seq = Some(seq);
|
||||
}
|
||||
match frame.kind.as_str() {
|
||||
"subscribed" => self.sid = self.sid.or(frame.msg.sid),
|
||||
"error" => {
|
||||
let (code, text) = (frame.msg.code.unwrap_or(0), frame.msg.msg.as_deref());
|
||||
let id = frame.id.unwrap_or(0);
|
||||
if TERMINAL_ERRORS.contains(&code) {
|
||||
return Err(Undecoded::Lost(format!("error {code} ({text:?})")));
|
||||
}
|
||||
log!("kalshi: command {id} refused: error {code} ({text:?})");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
wire::event(&frame)
|
||||
.map(|event| event.into_iter().collect())
|
||||
.map_err(Undecoded::Unreadable)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::core::Session as _;
|
||||
|
||||
fn decoded(session: &mut Session, text: &str) -> Result<usize, String> {
|
||||
match session.decode(text) {
|
||||
Ok(events) => Ok(events.len()),
|
||||
Err(Undecoded::Lost(why)) => Err(format!("lost: {why}")),
|
||||
Err(Undecoded::Unreadable(why)) => Err(format!("unreadable: {why}")),
|
||||
}
|
||||
}
|
||||
|
||||
const DELTA: &str = r#"{"type":"orderbook_delta","sid":1,"seq":SEQ,"msg":{"market_ticker":"M","market_id":"x","clock":5,"price_dollars":"0.5000","delta_fp":"1.00","side":"yes","ts_ms":1}}"#;
|
||||
|
||||
fn delta(seq: u64) -> String {
|
||||
DELTA.replace("SEQ", &seq.to_string())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_skipped_seq_is_a_lost_message() {
|
||||
let mut session = Session::default();
|
||||
let subscribed =
|
||||
r#"{"type":"subscribed","id":1,"msg":{"channel":"orderbook_delta","sid":3}}"#;
|
||||
assert_eq!(decoded(&mut session, subscribed), Ok(0));
|
||||
assert_eq!(session.sid, Some(3));
|
||||
assert_eq!(decoded(&mut session, &delta(1)), Ok(1));
|
||||
// An answer counts too.
|
||||
let ok = r#"{"type":"ok","id":2,"sid":3,"seq":2,"msg":{}}"#;
|
||||
assert_eq!(decoded(&mut session, ok), Ok(0));
|
||||
assert_eq!(decoded(&mut session, &delta(3)), Ok(1));
|
||||
assert_eq!(
|
||||
decoded(&mut session, &delta(5)),
|
||||
Err("lost: message 5 came, 4 expected".into())
|
||||
);
|
||||
// A connection's first message is 1.
|
||||
assert!(decoded(&mut Session::default(), &delta(2)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_terminal_error_starts_over_and_others_are_logged() {
|
||||
let error = |code: u32| {
|
||||
format!(r#"{{"type":"error","id":4,"seq":1,"msg":{{"code":{code},"msg":"x"}}}}"#)
|
||||
};
|
||||
assert_eq!(decoded(&mut Session::default(), &error(14)), Ok(0));
|
||||
assert!(
|
||||
decoded(&mut Session::default(), &error(25))
|
||||
.unwrap_err()
|
||||
.starts_with("lost")
|
||||
);
|
||||
assert!(
|
||||
decoded(&mut Session::default(), "not json")
|
||||
.unwrap_err()
|
||||
.starts_with("unreadable")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commands_carry_their_ids_and_the_subscription() {
|
||||
let mut session = Session::default();
|
||||
let tickers: Vec<String> = (0..501).map(|n| format!("T{n}")).collect();
|
||||
let refs: Vec<&String> = tickers.iter().collect();
|
||||
let sent: Vec<serde_json::Value> = session
|
||||
.subscribe(&refs)
|
||||
.iter()
|
||||
.map(|text| serde_json::from_str(text).unwrap())
|
||||
.collect();
|
||||
assert_eq!(sent.len(), 2);
|
||||
assert_eq!(sent[0]["id"], 1);
|
||||
assert_eq!(sent[1]["id"], 2);
|
||||
assert_eq!(sent[0]["cmd"], "subscribe");
|
||||
assert_eq!(
|
||||
sent[0]["params"]["market_tickers"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.len(),
|
||||
500
|
||||
);
|
||||
assert_eq!(sent[0]["params"]["use_yes_price"], true);
|
||||
session.sid = Some(7);
|
||||
let unsubscribe: serde_json::Value =
|
||||
serde_json::from_str(&session.unsubscribe(&refs[..1])[0]).unwrap();
|
||||
assert_eq!(unsubscribe["id"], 3);
|
||||
assert_eq!(unsubscribe["cmd"], "update_subscription");
|
||||
assert_eq!(unsubscribe["params"]["sids"], json!([7]));
|
||||
assert_eq!(unsubscribe["params"]["action"], "delete_markets");
|
||||
assert_eq!(unsubscribe["params"]["market_tickers"], json!(["T0"]));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod socket_tests;
|
||||
@@ -0,0 +1,186 @@
|
||||
//! The key that signs Kalshi's websocket handshake (docs/notes.md, Kalshi's
|
||||
//! websocket): an Ed25519 key, read once from the files that the environment
|
||||
//! names. Neither file's content is ever logged.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use base64::Engine;
|
||||
use ring::signature::Ed25519KeyPair;
|
||||
use rustls_pki_types::PrivateKeyDer;
|
||||
use rustls_pki_types::pem::PemObject;
|
||||
|
||||
/// The variables naming the two files.
|
||||
pub const KEY_ID_FILE: &str = "KALSHI_KEY_ID_FILE";
|
||||
pub const PRIVATE_KEY_FILE: &str = "KALSHI_PRIVATE_KEY_FILE";
|
||||
|
||||
/// What the handshake signs, after the time: the method and the path.
|
||||
const SIGNED_PATH: &str = "GET/trade-api/ws/v2";
|
||||
|
||||
pub struct Key {
|
||||
id: String,
|
||||
pair: Ed25519KeyPair,
|
||||
}
|
||||
|
||||
impl Key {
|
||||
/// The key from the files [`KEY_ID_FILE`] and [`PRIVATE_KEY_FILE`] name.
|
||||
pub fn from_env() -> anyhow::Result<Self> {
|
||||
let path = |variable: &str| {
|
||||
std::env::var_os(variable)
|
||||
.ok_or_else(|| anyhow::anyhow!("{variable} is not set: it names a file of the key"))
|
||||
};
|
||||
Self::from_files(
|
||||
path(KEY_ID_FILE)?.as_ref(),
|
||||
path(PRIVATE_KEY_FILE)?.as_ref(),
|
||||
)
|
||||
.map_err(|error| {
|
||||
anyhow::anyhow!("{error} (the files {KEY_ID_FILE} and {PRIVATE_KEY_FILE} name)")
|
||||
})
|
||||
}
|
||||
|
||||
/// The key id in `id_file`, the private key (a PKCS#8 PEM) in `key_file`.
|
||||
pub fn from_files(id_file: &Path, key_file: &Path) -> anyhow::Result<Self> {
|
||||
let id = std::fs::read_to_string(id_file)
|
||||
.map_err(|error| anyhow::anyhow!("{}: {error}", id_file.display()))?
|
||||
.trim()
|
||||
.to_string();
|
||||
anyhow::ensure!(!id.is_empty(), "{}: no key id in it", id_file.display());
|
||||
let pem = std::fs::read(key_file)
|
||||
.map_err(|error| anyhow::anyhow!("{}: {error}", key_file.display()))?;
|
||||
// The errors name the file, never its content: the PEM reader's own
|
||||
// error quotes what it read, so it is dropped.
|
||||
let der = PrivateKeyDer::from_pem_slice(&pem)
|
||||
.map_err(|_| anyhow::anyhow!("{}: not a PEM private key", key_file.display()))?;
|
||||
let PrivateKeyDer::Pkcs8(der) = der else {
|
||||
anyhow::bail!(
|
||||
"{}: not a PKCS#8 key; Kalshi's Ed25519 keys are",
|
||||
key_file.display()
|
||||
);
|
||||
};
|
||||
let pair = Ed25519KeyPair::from_pkcs8_maybe_unchecked(der.secret_pkcs8_der())
|
||||
.map_err(|_| anyhow::anyhow!("{}: not an Ed25519 key", key_file.display()))?;
|
||||
Ok(Self { id, pair })
|
||||
}
|
||||
|
||||
/// The handshake's three headers at `millis` since the epoch.
|
||||
pub fn headers(&self, millis: i64) -> [(&'static str, String); 3] {
|
||||
let signature = self.pair.sign(format!("{millis}{SIGNED_PATH}").as_bytes());
|
||||
[
|
||||
("KALSHI-ACCESS-KEY", self.id.clone()),
|
||||
("KALSHI-ACCESS-TIMESTAMP", millis.to_string()),
|
||||
(
|
||||
"KALSHI-ACCESS-SIGNATURE",
|
||||
base64::engine::general_purpose::STANDARD.encode(signature.as_ref()),
|
||||
),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
/// RFC 8032's first test key (section 7.1), as PKCS#8: a known key, so a
|
||||
/// known signature.
|
||||
#[cfg(test)]
|
||||
const TEST_KEY: &str = "-----BEGIN PRIVATE KEY-----
|
||||
MC4CAQAwBQYDK2VwBCIEIJ1hsZ3v/VpguoRK9JLsLMREScVpezJpGXA7rAMcrn9g
|
||||
-----END PRIVATE KEY-----
|
||||
";
|
||||
|
||||
/// [`TEST_KEY`], with the id `a-key-id`.
|
||||
#[cfg(test)]
|
||||
pub fn test_key() -> Key {
|
||||
let der = PrivateKeyDer::from_pem_slice(TEST_KEY.as_bytes()).unwrap();
|
||||
let PrivateKeyDer::Pkcs8(der) = der else {
|
||||
unreachable!("PKCS#8")
|
||||
};
|
||||
Key {
|
||||
id: "a-key-id".into(),
|
||||
pair: Ed25519KeyPair::from_pkcs8_maybe_unchecked(der.secret_pkcs8_der()).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn write(dir: &Path, name: &str, text: &str) -> std::path::PathBuf {
|
||||
let path = dir.join(name);
|
||||
std::fs::write(&path, text).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn scratch(name: &str) -> std::path::PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("kalshi-sign-{name}-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signs_the_time_method_and_path() {
|
||||
let dir = scratch("signs");
|
||||
let key = Key::from_files(
|
||||
&write(&dir, "id", "a-key-id\n"),
|
||||
&write(&dir, "key.pem", TEST_KEY),
|
||||
)
|
||||
.unwrap();
|
||||
let [id, time, signature] = key.headers(1_669_149_841_234);
|
||||
assert_eq!(id, ("KALSHI-ACCESS-KEY", "a-key-id".to_string()));
|
||||
assert_eq!(
|
||||
time,
|
||||
("KALSHI-ACCESS-TIMESTAMP", "1669149841234".to_string())
|
||||
);
|
||||
assert_eq!(signature.0, "KALSHI-ACCESS-SIGNATURE");
|
||||
// Checked against the RFC's public key for this one.
|
||||
let public = ring::signature::UnparsedPublicKey::new(
|
||||
&ring::signature::ED25519,
|
||||
[
|
||||
0xd7, 0x5a, 0x98, 0x01, 0x82, 0xb1, 0x0a, 0xb7, 0xd5, 0x4b, 0xfe, 0xd3, 0xc9, 0x64,
|
||||
0x07, 0x3a, 0x0e, 0xe1, 0x72, 0xf3, 0xda, 0xa6, 0x23, 0x25, 0xaf, 0x02, 0x1a, 0x68,
|
||||
0xf7, 0x07, 0x51, 0x1a,
|
||||
],
|
||||
);
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(&signature.1)
|
||||
.unwrap();
|
||||
public
|
||||
.verify(b"1669149841234GET/trade-api/ws/v2", &bytes)
|
||||
.unwrap();
|
||||
// Ed25519 is deterministic: the same time, the same signature.
|
||||
assert_eq!(key.headers(1_669_149_841_234)[2], signature);
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_or_wrong_file_says_which_and_never_what_it_holds() {
|
||||
let dir = scratch("errors");
|
||||
let id = write(&dir, "id", "a-key-id");
|
||||
let error = |id: &Path, key: &Path| Key::from_files(id, key).err().unwrap().to_string();
|
||||
|
||||
let missing = error(&id, &dir.join("absent.pem"));
|
||||
assert!(missing.contains("absent.pem"), "{missing}");
|
||||
|
||||
// A key flattened onto one line: the PEM reader's own error quotes
|
||||
// what it read, so ours must not carry it.
|
||||
let flat = write(&dir, "flat.pem", &TEST_KEY.replace('\n', " "));
|
||||
let flattened = error(&id, &flat);
|
||||
assert!(flattened.contains("not a PEM private key"), "{flattened}");
|
||||
assert!(flattened.ends_with("not a PEM private key"), "{flattened}");
|
||||
|
||||
let garbage = write(&dir, "garbage.pem", "not a key at all");
|
||||
let unreadable = error(&id, &garbage);
|
||||
assert!(
|
||||
unreadable.ends_with("not a PEM private key"),
|
||||
"{unreadable}"
|
||||
);
|
||||
|
||||
// An RSA key in PKCS#1, as Kalshi generates its RSA keys: refused.
|
||||
let rsa = write(
|
||||
&dir,
|
||||
"rsa.pem",
|
||||
"-----BEGIN RSA PRIVATE KEY-----\nMAA=\n-----END RSA PRIVATE KEY-----\n",
|
||||
);
|
||||
assert!(error(&id, &rsa).contains("not a PKCS#8 key"));
|
||||
|
||||
let empty = write(&dir, "empty-id", "\n");
|
||||
let key = write(&dir, "key.pem", TEST_KEY);
|
||||
assert!(error(&empty, &key).contains("no key id"));
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
//! The connection loop (`src/core/socket.rs`) against a local server, with
|
||||
//! Kalshi's codec: the signed handshake, the server's ping, and a skipped
|
||||
//! `seq`.
|
||||
|
||||
use std::net::TcpListener;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use crossbeam::channel::unbounded;
|
||||
use tungstenite::handshake::server::{Request, Response};
|
||||
use tungstenite::{Bytes, Message};
|
||||
|
||||
use super::{Kalshi, sign};
|
||||
use crate::core::socket::{Command, Notice, SILENT_AFTER, run};
|
||||
use crate::core::{BOOK_THREADS, Received};
|
||||
|
||||
const SNAPSHOT: &str = r#"{"type":"orderbook_snapshot","sid":1,"seq":1,"msg":{"market_ticker":"M","market_id":"x","clock":10,"yes_dollars_fp":[["0.4000","5.00"]]},"sending_ts_ms":1794422918369}"#;
|
||||
|
||||
fn delta(seq: u64) -> String {
|
||||
format!(
|
||||
r#"{{"type":"orderbook_delta","sid":1,"seq":{seq},"msg":{{"market_ticker":"M","market_id":"x","clock":{seq}0,"price_dollars":"0.4000","delta_fp":"1.00","side":"yes","ts_ms":1794422918370}}}}"#
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
// The handshake callback's error type is tungstenite's, not ours.
|
||||
#[allow(clippy::result_large_err)]
|
||||
fn signs_its_handshake_answers_pings_and_reconnects_on_a_skipped_seq() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let url = format!("ws://{}/trade-api/ws/v2", listener.local_addr().unwrap());
|
||||
let server = thread::spawn(move || {
|
||||
let mut seen = Vec::new();
|
||||
for connection in 0..2 {
|
||||
let (stream, _) = listener.accept().unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(5)))
|
||||
.unwrap();
|
||||
let mut headers = Vec::new();
|
||||
let mut socket =
|
||||
tungstenite::accept_hdr(stream, |request: &Request, response: Response| {
|
||||
for name in [
|
||||
"KALSHI-ACCESS-KEY",
|
||||
"KALSHI-ACCESS-TIMESTAMP",
|
||||
"KALSHI-ACCESS-SIGNATURE",
|
||||
] {
|
||||
let value = request
|
||||
.headers()
|
||||
.get(name)
|
||||
.map(|value| value.to_str().unwrap().to_string());
|
||||
headers.push(value);
|
||||
}
|
||||
Ok(response)
|
||||
})
|
||||
.unwrap();
|
||||
let subscribe: serde_json::Value =
|
||||
serde_json::from_str(socket.read().unwrap().to_text().unwrap()).unwrap();
|
||||
assert_eq!(subscribe["cmd"], "subscribe");
|
||||
assert_eq!(
|
||||
subscribe["params"]["market_tickers"],
|
||||
serde_json::json!(["M"])
|
||||
);
|
||||
socket
|
||||
.send(Message::text(
|
||||
r#"{"type":"subscribed","id":1,"msg":{"channel":"orderbook_delta","sid":1}}"#,
|
||||
))
|
||||
.unwrap();
|
||||
socket.send(Message::text(SNAPSHOT)).unwrap();
|
||||
socket.send(Message::text(delta(2))).unwrap();
|
||||
if connection == 1 {
|
||||
socket.close(None).unwrap();
|
||||
while socket.read().is_ok() {}
|
||||
seen.push((headers, false));
|
||||
break;
|
||||
}
|
||||
socket
|
||||
.send(Message::Ping(Bytes::from_static(b"heartbeat")))
|
||||
.unwrap();
|
||||
let ponged = loop {
|
||||
match socket.read().unwrap() {
|
||||
Message::Pong(body) => break body == Bytes::from_static(b"heartbeat"),
|
||||
_ => continue,
|
||||
}
|
||||
};
|
||||
// 3 never comes.
|
||||
socket.send(Message::text(delta(4))).unwrap();
|
||||
while socket.read().is_ok() {}
|
||||
seen.push((headers, ponged));
|
||||
}
|
||||
seen
|
||||
});
|
||||
|
||||
let kalshi = Kalshi {
|
||||
websocket: url,
|
||||
listing: String::new(),
|
||||
key: sign::test_key(),
|
||||
};
|
||||
let (book_threads, events): (Vec<_>, Vec<_>) = (0..BOOK_THREADS).map(|_| unbounded()).unzip();
|
||||
let (commands_tx, commands) = unbounded();
|
||||
commands_tx
|
||||
.send(Command::Subscribe(vec!["M".into()]))
|
||||
.unwrap();
|
||||
let (notices_tx, notices) = unbounded();
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let client_stop = Arc::clone(&stop);
|
||||
let client = thread::spawn(move || {
|
||||
run(
|
||||
&kalshi,
|
||||
0,
|
||||
&book_threads,
|
||||
&commands,
|
||||
¬ices_tx,
|
||||
SILENT_AFTER,
|
||||
&client_stop,
|
||||
)
|
||||
});
|
||||
let seen = server.join().unwrap();
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
client.join().unwrap();
|
||||
|
||||
for (headers, _) in &seen {
|
||||
assert_eq!(headers[0].as_deref(), Some("a-key-id"));
|
||||
assert!(
|
||||
headers[1]
|
||||
.as_deref()
|
||||
.is_some_and(|time| time.parse::<i64>().is_ok())
|
||||
);
|
||||
assert!(
|
||||
headers[2]
|
||||
.as_deref()
|
||||
.is_some_and(|signature| signature.len() == 88)
|
||||
);
|
||||
}
|
||||
assert!(seen[0].1, "the ping got its pong");
|
||||
let notices: Vec<Notice> = notices.try_iter().collect();
|
||||
assert_eq!(
|
||||
notices[..3],
|
||||
[Notice::Up(0), Notice::Down(0), Notice::Up(0)]
|
||||
);
|
||||
// The book and the change before the gap reached the book thread; the
|
||||
// change after it didn't.
|
||||
let received: Vec<_> = events
|
||||
.iter()
|
||||
.flat_map(|receiver| receiver.try_iter())
|
||||
.collect();
|
||||
// Every book thread heard it go down each time: after the skipped
|
||||
// `seq`, and when the server closed the second connection.
|
||||
let downs = received
|
||||
.iter()
|
||||
.filter(|received| matches!(received, Received::Down(0)))
|
||||
.count();
|
||||
assert_eq!(downs, 2 * BOOK_THREADS);
|
||||
let events: Vec<_> = received
|
||||
.iter()
|
||||
.filter(|received| matches!(received, Received::Event(_)))
|
||||
.collect();
|
||||
assert_eq!(events.len(), 4, "two per connection: {events:?}");
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
//! Kalshi's websocket messages, as the exchange writes them, and how a
|
||||
//! connection reads them: each frame is one message, numbered by `seq` across
|
||||
//! the connection (docs/notes.md, Kalshi's websocket).
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use engineering_challenge::Side;
|
||||
use engineering_challenge::book::parse_fixed;
|
||||
use serde::Deserialize;
|
||||
|
||||
/// A message of the `orderbook_delta` channel, read.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Event {
|
||||
Book(BookMessage),
|
||||
Change(ChangeMessage),
|
||||
}
|
||||
|
||||
/// A market's full book: its YES bids are the bids; with `use_yes_price`,
|
||||
/// its NO bids come as the asks they are, at one minus their price.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct BookMessage {
|
||||
pub ticker: String,
|
||||
/// The clock of the last change it holds (0 before the market opens).
|
||||
pub clock: u64,
|
||||
pub bids: Vec<(i64, i64)>,
|
||||
pub asks: Vec<(i64, i64)>,
|
||||
/// When the exchange sent it: a book has no time of its own.
|
||||
pub sent_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// A change of one level: its quantity moves by `delta`, which can be
|
||||
/// negative.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct ChangeMessage {
|
||||
pub ticker: String,
|
||||
pub clock: u64,
|
||||
pub side: Side,
|
||||
pub price: i64,
|
||||
pub delta: i64,
|
||||
pub at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl crate::core::Event for Event {
|
||||
fn market(&self) -> Option<&str> {
|
||||
Some(match self {
|
||||
Event::Book(book) => &book.ticker,
|
||||
Event::Change(change) => &change.ticker,
|
||||
})
|
||||
}
|
||||
|
||||
fn is_book(&self) -> bool {
|
||||
matches!(self, Event::Book(_))
|
||||
}
|
||||
}
|
||||
|
||||
/// What a frame holds, every field the channel's messages use.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Frame {
|
||||
#[serde(rename = "type")]
|
||||
pub kind: String,
|
||||
/// The command it answers, for an answer.
|
||||
pub id: Option<u64>,
|
||||
/// Counts every message of the connection's subscription but
|
||||
/// `subscribed`, from 1.
|
||||
pub seq: Option<u64>,
|
||||
pub sending_ts_ms: Option<i64>,
|
||||
#[serde(default)]
|
||||
pub msg: Body,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub struct Body {
|
||||
pub market_ticker: Option<String>,
|
||||
pub clock: Option<u64>,
|
||||
pub yes_dollars_fp: Option<Vec<(String, String)>>,
|
||||
pub no_dollars_fp: Option<Vec<(String, String)>>,
|
||||
pub price_dollars: Option<String>,
|
||||
pub delta_fp: Option<String>,
|
||||
pub side: Option<String>,
|
||||
pub ts_ms: Option<i64>,
|
||||
/// `subscribed`: the subscription's id.
|
||||
pub sid: Option<u64>,
|
||||
/// `error`: its code and text.
|
||||
pub code: Option<u32>,
|
||||
pub msg: Option<String>,
|
||||
}
|
||||
|
||||
/// The event a frame of the book channel holds; none for the others (answers
|
||||
/// to commands), `Err` with what is wrong for a book or change unreadable.
|
||||
pub fn event(frame: &Frame) -> Result<Option<Event>, String> {
|
||||
let body = &frame.msg;
|
||||
let ticker = || body.market_ticker.clone().ok_or("no market_ticker");
|
||||
let clock = || body.clock.ok_or("no clock");
|
||||
match frame.kind.as_str() {
|
||||
"orderbook_snapshot" => {
|
||||
let side = |levels: &Option<Vec<(String, String)>>| {
|
||||
levels
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|(price, quantity)| Some((parse_fixed(price)?, parse_fixed(quantity)?)))
|
||||
.collect::<Option<Vec<_>>>()
|
||||
.ok_or("a level is not two decimals")
|
||||
};
|
||||
let sent_at = frame
|
||||
.sending_ts_ms
|
||||
.and_then(DateTime::from_timestamp_millis)
|
||||
.ok_or("no sending_ts_ms")?;
|
||||
Ok(Some(Event::Book(BookMessage {
|
||||
ticker: ticker()?,
|
||||
clock: clock()?,
|
||||
bids: side(&body.yes_dollars_fp)?,
|
||||
asks: side(&body.no_dollars_fp)?,
|
||||
sent_at,
|
||||
})))
|
||||
}
|
||||
"orderbook_delta" => {
|
||||
let side = match body.side.as_deref() {
|
||||
Some("yes") => Side::Buy,
|
||||
Some("no") => Side::Sell,
|
||||
_ => return Err("side is neither yes nor no".into()),
|
||||
};
|
||||
let price = body.price_dollars.as_deref().and_then(parse_fixed);
|
||||
let delta = body.delta_fp.as_deref().and_then(parse_signed);
|
||||
let at = body.ts_ms.and_then(DateTime::from_timestamp_millis);
|
||||
let (Some(price), Some(delta), Some(at)) = (price, delta, at) else {
|
||||
return Err("price_dollars, delta_fp or ts_ms unreadable".into());
|
||||
};
|
||||
Ok(Some(Event::Change(ChangeMessage {
|
||||
ticker: ticker()?,
|
||||
clock: clock()?,
|
||||
side,
|
||||
price,
|
||||
delta,
|
||||
at,
|
||||
})))
|
||||
}
|
||||
_ => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// A signed decimal (`"-54.00"`, `"20.00"`), exactly.
|
||||
fn parse_signed(text: &str) -> Option<i64> {
|
||||
match text.strip_prefix('-') {
|
||||
Some(magnitude) => parse_fixed(magnitude).map(|value| -value),
|
||||
None => parse_fixed(text),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn read(text: &str) -> Result<Option<Event>, String> {
|
||||
event(&serde_json::from_str(text).unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_a_book_and_a_change_as_recorded() {
|
||||
let book = r#"{"type":"orderbook_snapshot","sid":1,"seq":2,"msg":{"market_ticker":"KXHIGHNY-26NOV12-T59","market_id":"01a1","clock":56884934192,"yes_dollars_fp":[["0.4800","80.00"],["0.6500","10.00"]],"no_dollars_fp":[["0.9100","80.00"],["0.6600","7.00"]]},"sending_ts_ms":1794422918369}"#;
|
||||
assert_eq!(
|
||||
read(book).unwrap(),
|
||||
Some(Event::Book(BookMessage {
|
||||
ticker: "KXHIGHNY-26NOV12-T59".into(),
|
||||
clock: 56_884_934_192,
|
||||
bids: vec![(480_000, 80_000_000), (650_000, 10_000_000)],
|
||||
asks: vec![(910_000, 80_000_000), (660_000, 7_000_000)],
|
||||
sent_at: DateTime::from_timestamp_millis(1_794_422_918_369).unwrap(),
|
||||
}))
|
||||
);
|
||||
let change = r#"{"type":"orderbook_delta","sid":1,"seq":9,"msg":{"market_ticker":"M","market_id":"01a1","clock":56885842636,"price_dollars":"0.8100","delta_fp":"-1.50","side":"no","ts":"2026-11-11T18:36:37.012Z","ts_ms":1794422197012},"sending_ts_ms":1794422197023}"#;
|
||||
assert_eq!(
|
||||
read(change).unwrap(),
|
||||
Some(Event::Change(ChangeMessage {
|
||||
ticker: "M".into(),
|
||||
clock: 56_885_842_636,
|
||||
side: Side::Sell,
|
||||
price: 810_000,
|
||||
delta: -1_500_000,
|
||||
at: DateTime::from_timestamp_millis(1_794_422_197_012).unwrap(),
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_book_has_no_levels_and_answers_are_no_events() {
|
||||
// An unopened market's book, sides omitted.
|
||||
let empty = r#"{"type":"orderbook_snapshot","sid":1,"seq":2,"msg":{"market_ticker":"KXQUICKSETTLE-11NOV26H1400-2","market_id":"","clock":0},"sending_ts_ms":1794422918369}"#;
|
||||
let Some(Event::Book(book)) = read(empty).unwrap() else {
|
||||
panic!("not a book");
|
||||
};
|
||||
assert!(book.bids.is_empty() && book.asks.is_empty());
|
||||
assert_eq!(book.clock, 0);
|
||||
for answer in [
|
||||
r#"{"type":"subscribed","id":1,"msg":{"channel":"orderbook_delta","sid":1},"sending_ts_ms":1}"#,
|
||||
r#"{"type":"ok","id":2,"sid":1,"seq":6,"sending_ts_ms":1,"msg":{}}"#,
|
||||
r#"{"type":"error","id":2,"sid":1,"seq":860,"msg":{"code":14,"msg":"Market Ticker required"}}"#,
|
||||
] {
|
||||
assert_eq!(read(answer).unwrap(), None, "{answer}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_change_says_so() {
|
||||
for bad in [
|
||||
r#"{"type":"orderbook_delta","seq":1,"msg":{"market_ticker":"M","clock":1,"price_dollars":"0.81","delta_fp":"1e3","side":"no","ts_ms":1}}"#,
|
||||
r#"{"type":"orderbook_delta","seq":1,"msg":{"market_ticker":"M","clock":1,"price_dollars":"0.81","delta_fp":"1.00","side":"maybe","ts_ms":1}}"#,
|
||||
r#"{"type":"orderbook_delta","seq":1,"msg":{"clock":1,"price_dollars":"0.81","delta_fp":"1.00","side":"no","ts_ms":1}}"#,
|
||||
] {
|
||||
assert!(read(bad).is_err(), "{bad}");
|
||||
}
|
||||
assert_eq!(parse_signed("-0.01"), Some(-10_000));
|
||||
assert_eq!(parse_signed("--1"), None);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user