feat!: a generic core, and Kalshi through it

The orchestrator, the connection loop and the book thread's shell move to
src/core/, generic over an Exchange and its loss rule (ADR 0004).
Polymarket runs through it unchanged. Kalshi joins: a signed handshake
(KALSHI_KEY_ID_FILE, KALSHI_PRIVATE_KEY_FILE), a reconnect on a skipped
sequence number, first one wins by clock and rank, and its market listing
with markets tracked from three hours before they open.

Breaking: the exchange is now the first argument,
`engineering-challenge <polymarket|kalshi|kalshi-demo> [OUTPUT]`.
This commit is contained in:
pascal committed 2026-11-11 20:54:17 +00:00
1 parent 480ee3ad2d
commit 1a5bbcd602
33 files changed
+8150 -1846

No files matched your search

+823
View File
@@ -0,0 +1,823 @@
//! Kalshi's rule for a book thread: first one wins by clock and rank
//! (roadmap/06). A market's changes carry its clock, the same on every
//! connection, and come in the same order on each; several can share a
//! clock, so a change is named by its clock and its rank among the changes of
//! that clock. The book stands at the position of the last change it holds;
//! each connection's stream at the position of the last change it delivered,
//! counted from the book it last delivered. A change passing the book's
//! position is the next one; any other is a duplicate. Nothing is held.
//! A connection that lost a message reconnects (its `seq`, in
//! `src/kalshi/mod.rs`); the book then comes fresh.
use std::collections::BTreeMap;
use chrono::{DateTime, Utc};
use engineering_challenge::book::Book;
use engineering_challenge::cause::{self, Cause};
use super::Market;
use super::wire::{BookMessage, ChangeMessage, Event};
use crate::core::feed::{self, Effect, Minute, Rule, Shell, is_up};
use crate::core::socket::Command;
/// A clock, and a rank among the changes of that clock.
type Position = (u64, u32);
/// A book's rank: it holds every change of its clock.
const BOOK: u32 = u32::MAX;
/// A symbol's state for the rule.
#[derive(Debug, Default)]
pub struct Clocked {
/// Where the book stands; none before its first book.
at: Option<Position>,
/// By connection: where its stream stands, since the book it last
/// delivered. A connection with none has delivered no book since it came
/// up or subscribed: its changes have nowhere to go.
on: BTreeMap<usize, Position>,
/// The connection a fresh book was asked from: its answer replaces ours.
asked: Option<usize>,
}
type Symbol = feed::Symbol<Clocked>;
/// What the minute line counts, since the start.
#[derive(Debug, Default)]
struct Counts {
written: u64,
duplicates: u64,
/// Changes from a connection that had delivered no book of the symbol.
unplaced: u64,
/// Fresh books after a symbol's first.
books: u64,
/// Of them, books at or past ours that differed: written as a Snapshot.
losses: u64,
/// Changes from a connection past the book, after the book went back to
/// an answer: dropped until the book reaches it.
ahead: u64,
/// Changes that took a level below zero: our book was wrong.
below_zero: u64,
resyncs: u64,
}
/// Kalshi's rule, for one book thread.
#[derive(Default)]
pub struct Clock {
counts: Counts,
}
impl Clock {
fn on_book(
&mut self,
shell: &mut Shell<Clocked>,
received_at: DateTime<Utc>,
connection: usize,
message: BookMessage,
effects: &mut Vec<Effect>,
) {
let ticker = message.ticker;
let Some(symbol) = shell.symbols.get_mut(&ticker) else {
return;
};
if !symbol.connections.contains(&connection) {
return;
}
let position = (message.clock, BOOK);
symbol.rule.on.insert(connection, position);
let book = Book {
bids: message.bids.into_iter().collect(),
asks: message.asks.into_iter().collect(),
};
let asked = symbol.rule.asked == Some(connection);
let cause = match symbol.rule.at {
None => None,
Some(at) => {
self.counts.books += 1;
// Behind ours, unasked: it only says where its connection is.
if position < at && !asked {
return;
}
if book == symbol.book {
symbol.rule.at = Some(at.max(position));
if asked {
symbol.rule.asked = None;
}
return;
}
Some(at)
}
};
// The first book, an answer, or a book at or past ours that differs:
// ours is replaced. Which connection missed what can't be told: the
// other may only be behind.
symbol.book = book;
symbol.exchange_timestamp = message.sent_at;
symbol.rule.at = Some(position);
// A first book answers whichever connection it was asked from.
if asked || cause.is_none() {
symbol.rule.asked = None;
}
effects.push(Effect::Write(symbol.snapshot(&ticker, received_at)));
if let Some(ours) = cause {
self.counts.losses += 1;
let detail = format!(" (its clock {}, ours {})", message.clock, ours.0);
log!(
"kalshi: {}",
cause::line(&ticker, received_at, Cause::Loss, &detail)
);
}
}
fn on_change(
&mut self,
shell: &mut Shell<Clocked>,
received_at: DateTime<Utc>,
connection: usize,
change: ChangeMessage,
effects: &mut Vec<Effect>,
) {
let ticker = change.ticker;
shell.count_received(&ticker, connection);
let Some(symbol) = shell.symbols.get_mut(&ticker) else {
return;
};
let Some(on) = symbol.rule.on.get_mut(&connection) else {
self.counts.unplaced += 1;
return;
};
let before = *on;
*on = match before {
(clock, rank) if clock == change.clock => (clock, rank.saturating_add(1)),
_ => (change.clock, 1),
};
let position = *on;
match symbol.rule.at {
Some(at) if position <= at => {
self.counts.duplicates += 1;
return;
}
// Its stream was already past the book, which went back to an
// answer: the changes between are still to come from the
// connection that answered, and this one would skip them.
Some(at) if before > at => {
self.counts.ahead += 1;
return;
}
_ => {}
}
symbol.rule.at = Some(position);
let quantity = symbol.book.add(change.side, change.price, change.delta);
let quantity = if quantity < 0 {
// A level can't hold less than nothing: ours was wrong.
self.counts.below_zero += 1;
log!(
"kalshi: {ticker}: a change takes a level below zero, resync on connection \
{connection}"
);
if symbol.rule.asked.is_none() {
Self::ask_fresh_book(&ticker, symbol, connection, effects);
self.counts.resyncs += 1;
}
0
} else {
quantity
};
symbol.exchange_timestamp = change.at;
self.counts.written += 1;
effects.push(Effect::Write(symbol.delta(
&ticker,
change.side,
change.price,
quantity,
received_at,
change.at,
)));
}
/// Asks `connection` for a fresh book: unsubscribe and subscribe again.
fn ask_fresh_book(
ticker: &str,
symbol: &mut Symbol,
connection: usize,
effects: &mut Vec<Effect>,
) {
symbol.rule.asked = Some(connection);
effects.push(Effect::Send(
connection,
Command::Resubscribe(ticker.to_string()),
));
}
}
impl Rule for Clock {
type Market = Market;
type Event = Event;
type Symbol = Clocked;
fn symbol(&self, _market: &Market) -> Clocked {
Clocked::default()
}
fn on_event(
&mut self,
shell: &mut Shell<Clocked>,
connection: usize,
received_at: DateTime<Utc>,
event: Event,
effects: &mut Vec<Effect>,
) {
match event {
Event::Book(book) => self.on_book(shell, received_at, connection, book, effects),
Event::Change(change) => {
self.on_change(shell, received_at, connection, change, effects)
}
}
}
/// Its stream there counts no more; a fresh book asked from it is asked
/// from another.
fn on_leave(
&mut self,
up: &[bool],
ticker: &str,
symbol: &mut Symbol,
connection: usize,
effects: &mut Vec<Effect>,
) {
symbol.rule.on.remove(&connection);
if symbol.rule.asked == Some(connection)
&& let Some(other) = symbol.other_up(up, connection)
{
Self::ask_fresh_book(ticker, symbol, other, effects);
self.counts.resyncs += 1;
}
}
/// Its streams count no more: back up, it delivers fresh books, and one
/// asked from it is its answer.
fn on_down(
&mut self,
shell: &mut Shell<Clocked>,
connection: usize,
_effects: &mut Vec<Effect>,
) {
for symbol in shell.symbols.values_mut() {
symbol.rule.on.remove(&connection);
}
}
fn on_timer(
&mut self,
_shell: &mut Shell<Clocked>,
_now: DateTime<Utc>,
_effects: &mut Vec<Effect>,
) {
}
fn wakes_soon(&self) -> bool {
false
}
fn on_removed(&mut self, _ticker: &str) {}
/// While no connection delivers it, the book may be missing changes: no
/// Snapshot of it until one does.
fn has_book(symbol: &Clocked) -> bool {
symbol.at.is_some() && !symbol.on.is_empty()
}
fn may_be_quiet(symbol: &Clocked) -> bool {
symbol.at.is_some() && symbol.asked.is_none()
}
/// Asks again for a fresh book still not come, and for the book of a
/// symbol no connection delivers: its first, or one a reconnect didn't
/// bring.
fn on_minute(
&mut self,
up: &[bool],
ticker: &str,
symbol: &mut Symbol,
_now: DateTime<Utc>,
effects: &mut Vec<Effect>,
) {
let again = symbol.rule.asked.or_else(|| {
let up = |&connection: &usize| is_up(up, connection);
symbol
.rule
.on
.is_empty()
.then(|| symbol.connections.iter().copied().find(up))
.flatten()
});
if let Some(connection) = again {
Self::ask_fresh_book(ticker, symbol, connection, effects);
self.counts.resyncs += 1;
}
}
fn log_minute(&mut self, minute: &Minute) {
let counts = &self.counts;
log!(
"kalshi: book thread {}: {} symbols; so far {} changes written, {} duplicates, \
{} before their connection's book, {} ahead of the book, {} fresh books \
({} losses), {} below zero, \
{} resyncs, {} quiet subscriptions resubscribed, {} markets reported quiet again; \
this minute {} changes received, at most {} events waited",
minute.number,
minute.symbols,
counts.written,
counts.duplicates,
counts.unplaced,
counts.ahead,
counts.books,
counts.losses,
counts.below_zero,
counts.resyncs,
minute.quiet,
minute.quiet_reported,
minute.received,
minute.peak_waiting
);
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use chrono::TimeDelta;
use engineering_challenge::util::dtos::challenge_orderbook::PriceLevelUpdateDomain;
use serde::Deserialize;
use super::*;
use crate::core::{Arrival, Received, feed::next_minute};
use crate::kalshi::wire;
use engineering_challenge::Side;
use engineering_challenge::book::Output;
type Feed = feed::Feed<Clock>;
fn at(millis: i64) -> DateTime<Utc> {
DateTime::from_timestamp_millis(1_794_422_000_000 + millis).unwrap()
}
fn feed_of(tickers: &[&str], connections: &[usize]) -> Feed {
let mut feed = Feed::new("kalshi", 0, Clock::default());
for ticker in tickers {
feed.add_market(
&Market {
ticker: ticker.to_string(),
},
connections,
at(0),
);
}
feed
}
fn book(ticker: &str, clock: u64, bids: &[(i64, i64)], asks: &[(i64, i64)]) -> Event {
Event::Book(BookMessage {
ticker: ticker.into(),
clock,
bids: bids.to_vec(),
asks: asks.to_vec(),
sent_at: at(0),
})
}
fn change(ticker: &str, clock: u64, side: Side, price: i64, delta: i64) -> Event {
Event::Change(ChangeMessage {
ticker: ticker.into(),
clock,
side,
price,
delta,
at: at(0),
})
}
impl Feed {
fn from(&mut self, connection: usize, millis: i64, event: Event) -> Vec<Effect> {
self.on_event(Arrival {
received_at: at(millis),
connection,
bytes: 0,
event,
})
}
}
fn deltas(effects: &[Effect]) -> Vec<&PriceLevelUpdateDomain> {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Write(Output::Delta(delta)) => Some(delta),
_ => None,
})
.collect()
}
fn snapshots(effects: &[Effect]) -> usize {
effects
.iter()
.filter(|effect| matches!(effect, Effect::Write(Output::Snapshot(_))))
.count()
}
fn resyncs(effects: &[Effect]) -> Vec<(usize, &str)> {
effects
.iter()
.filter_map(|effect| match effect {
Effect::Send(connection, Command::Resubscribe(ticker)) => {
Some((*connection, ticker.as_str()))
}
_ => None,
})
.collect()
}
/// A market with the same book from both connections.
fn both_on(ticker: &str) -> Feed {
let mut feed = feed_of(&[ticker], &[0, 1]);
let first = feed.from(0, 1, book(ticker, 10, &[(400_000, 5_000_000)], &[]));
assert_eq!(snapshots(&first), 1);
let second = feed.from(1, 2, book(ticker, 10, &[(400_000, 5_000_000)], &[]));
assert!(second.is_empty(), "{second:?}");
feed
}
#[test]
fn a_change_is_written_once_from_whichever_delivers_it_first() {
let mut feed = both_on("M");
let first = feed.from(1, 5, change("M", 11, Side::Buy, 400_000, 2_000_000));
let written = deltas(&first);
assert_eq!(written.len(), 1);
// The absolute quantity, at the first copy's receive time.
assert_eq!(written[0].quantity, 7_000_000);
assert_eq!(written[0].backtest_timestamp, at(5));
assert!(
deltas(&feed.from(0, 6, change("M", 11, Side::Buy, 400_000, 2_000_000))).is_empty()
);
assert_eq!(feed.rule.counts.duplicates, 1);
}
#[test]
fn every_change_of_one_clock_is_applied_once() {
let mut feed = both_on("M");
// Two identical changes share a clock: both are real.
let twice = [
change("M", 11, Side::Sell, 600_000, 1_000_000),
change("M", 11, Side::Sell, 600_000, 1_000_000),
];
let mut written = Vec::new();
for event in twice {
written.extend(feed.from(0, 5, event));
}
let quantities: Vec<i64> = deltas(&written)
.iter()
.map(|delta| delta.quantity)
.collect();
assert_eq!(quantities, [1_000_000, 2_000_000]);
for event in [
change("M", 11, Side::Sell, 600_000, 1_000_000),
change("M", 11, Side::Sell, 600_000, 1_000_000),
] {
assert!(deltas(&feed.from(1, 6, event)).is_empty());
}
// The second connection's third change of that clock is new.
let third = feed.from(1, 7, change("M", 11, Side::Sell, 600_000, 1_000_000));
assert_eq!(deltas(&third)[0].quantity, 3_000_000);
}
#[test]
fn a_connection_counts_for_a_symbol_only_from_its_book() {
let mut feed = feed_of(&["M"], &[0, 1]);
feed.from(0, 1, book("M", 10, &[], &[]));
// Connection 1 hasn't delivered its book: its change has no place.
assert!(feed.from(1, 2, change("M", 11, Side::Buy, 1, 1)).is_empty());
assert_eq!(feed.rule.counts.unplaced, 1);
// Down and up again, connection 0 too waits for its book.
feed.on_received(Received::Down(0));
feed.on_received(Received::Up(0));
assert!(feed.from(0, 3, change("M", 12, Side::Buy, 1, 1)).is_empty());
assert_eq!(feed.rule.counts.unplaced, 2);
}
#[test]
fn a_fresh_book_replaces_ours_only_at_or_past_it_and_different() {
let mut feed = both_on("M");
feed.from(0, 3, change("M", 11, Side::Buy, 400_000, 1_000_000));
// Older than ours: connection 1 is only behind.
assert!(
feed.from(1, 4, book("M", 10, &[(400_000, 5_000_000)], &[]))
.is_empty()
);
// At our clock and the same: nothing.
assert!(
feed.from(1, 5, book("M", 11, &[(400_000, 6_000_000)], &[]))
.is_empty()
);
// Past ours and the same: nothing either.
assert!(
feed.from(1, 6, book("M", 15, &[(400_000, 6_000_000)], &[]))
.is_empty()
);
assert_eq!(feed.rule.counts.losses, 0);
// Past ours and different: it replaces ours, as a Snapshot.
let different = feed.from(1, 7, book("M", 20, &[(400_000, 1_000_000)], &[]));
assert_eq!(snapshots(&different), 1);
assert_eq!(feed.rule.counts.losses, 1);
assert_eq!(feed.shell.symbols["M"].book.bids[&400_000], 1_000_000);
// Its changes up to that book are duplicates now.
assert!(deltas(&feed.from(0, 8, change("M", 16, Side::Buy, 400_000, 1))).is_empty());
}
#[test]
fn a_level_below_zero_resyncs_and_the_answer_replaces_ours() {
let mut feed = both_on("M");
let effects = feed.from(0, 3, change("M", 11, Side::Buy, 400_000, -6_000_000));
assert_eq!(deltas(&effects)[0].quantity, 0);
assert_eq!(resyncs(&effects), [(0, "M")]);
assert_eq!(feed.rule.counts.below_zero, 1);
assert!(!Clock::may_be_quiet(&feed.shell.symbols["M"].rule));
// The answer replaces ours even behind it, and differing is a loss.
let answer = feed.from(0, 4, book("M", 9, &[(400_000, 9_000_000)], &[]));
assert_eq!(snapshots(&answer), 1);
assert_eq!(feed.shell.symbols["M"].book.bids[&400_000], 9_000_000);
assert_eq!(feed.shell.symbols["M"].rule.asked, None);
// The stream that answered goes on from its book.
let after = feed.from(0, 5, change("M", 10, Side::Buy, 400_000, 1_000_000));
assert_eq!(deltas(&after)[0].quantity, 10_000_000);
}
/// The answer to a resync can be behind what the other connection has
/// delivered: the book goes back to it, and only a stream that continues
/// from there is taken until the other fits again.
#[test]
fn after_an_answer_behind_ours_a_connection_ahead_waits_for_the_book() {
let mut feed = both_on("M");
feed.from(0, 3, change("M", 11, Side::Buy, 400_000, -6_000_000));
feed.from(1, 4, change("M", 11, Side::Buy, 400_000, -6_000_000));
feed.from(1, 5, change("M", 12, Side::Buy, 400_000, 1_000_000));
// Connection 0's answer holds clock 11, not yet 12.
let answer = feed.from(0, 6, book("M", 11, &[(400_000, 2_000_000)], &[]));
assert_eq!(snapshots(&answer), 1);
// Connection 1 is past it: its next change would skip clock 12.
let ahead = feed.from(1, 7, change("M", 13, Side::Buy, 400_000, 1_000_000));
assert!(deltas(&ahead).is_empty(), "{ahead:?}");
let twelve = feed.from(0, 8, change("M", 12, Side::Buy, 400_000, 1_000_000));
assert_eq!(deltas(&twelve)[0].quantity, 3_000_000);
let thirteen = feed.from(0, 9, change("M", 13, Side::Buy, 400_000, 1_000_000));
assert_eq!(deltas(&thirteen)[0].quantity, 4_000_000);
// The book has caught up with connection 1: it counts again.
let fourteen = feed.from(1, 10, change("M", 14, Side::Buy, 400_000, 1_000_000));
assert_eq!(deltas(&fourteen)[0].quantity, 5_000_000);
assert_eq!(feed.rule.counts.ahead, 1);
}
#[test]
fn a_first_book_asked_from_one_connection_can_come_from_the_other() {
let mut feed = feed_of(&["N"], &[0, 1]);
feed.on_received(Received::Up(0));
feed.from(0, 5, change("N", 1, Side::Buy, 1, 1));
assert_eq!(resyncs(&feed.tick(next_minute(at(10)))), [(0, "N")]);
let first = feed.from(1, 61_000, book("N", 20, &[(400_000, 5_000_000)], &[]));
assert_eq!(snapshots(&first), 1);
// Connection 0's answer, older and different, is only where it stands.
let answer = feed.from(0, 61_001, book("N", 19, &[(400_000, 4_000_000)], &[]));
assert!(answer.is_empty(), "{answer:?}");
assert_eq!(feed.rule.counts.losses, 0);
assert_eq!(feed.shell.symbols["N"].book.bids[&400_000], 5_000_000);
}
#[test]
fn a_symbol_no_connection_delivers_is_asked_for_at_the_minute() {
let mut feed = both_on("M");
feed.on_received(Received::Down(0));
feed.on_received(Received::Down(1));
feed.on_received(Received::Up(0));
feed.on_received(Received::Up(1));
// Its books didn't come with the reconnects.
assert_eq!(resyncs(&feed.tick(next_minute(at(10)))), [(0, "M")]);
}
#[test]
fn a_resync_asked_from_a_connection_that_leaves_is_asked_from_another() {
let mut feed = both_on("M");
feed.from(0, 3, change("M", 11, Side::Buy, 400_000, -6_000_000));
let market = Market { ticker: "M".into() };
let moved = feed.move_market(&market, 0, 2, at(4));
assert!(resyncs(&moved).is_empty());
// Connection 2's book ends the move: connection 0 leaves.
let ended = feed.from(2, 5, book("M", 12, &[], &[]));
assert!(ended.iter().any(|effect| matches!(
effect,
Effect::Send(0, Command::Unsubscribe(tokens)) if tokens == &["M".to_string()]
)));
// It came from connection 2, not the one asked: not the answer.
assert_eq!(resyncs(&ended), [(1, "M")]);
}
#[test]
fn a_fresh_book_still_not_come_is_asked_again_at_the_minute() {
let mut feed = both_on("M");
feed.from(0, 3, change("M", 11, Side::Buy, 400_000, -6_000_000));
let minute = feed.tick(next_minute(at(10)));
assert_eq!(resyncs(&minute), [(0, "M")]);
// A symbol with no book at all is asked for one too.
let mut none = feed_of(&["N"], &[0]);
none.on_received(Received::Up(0));
// An event starts the book thread's minutes.
none.from(0, 5, change("N", 1, Side::Buy, 1, 1));
assert_eq!(resyncs(&none.tick(next_minute(at(10)))), [(0, "N")]);
// One in order isn't.
let mut fine = both_on("F");
assert!(resyncs(&fine.tick(next_minute(at(10)))).is_empty());
}
#[test]
fn no_minute_snapshot_while_no_connection_delivers_the_symbol() {
let mut feed = both_on("M");
let minute = next_minute(at(10));
assert_eq!(snapshots(&feed.tick(minute)), 1);
feed.on_received(Received::Down(0));
feed.on_received(Received::Down(1));
assert_eq!(snapshots(&feed.tick(minute + TimeDelta::minutes(1))), 0);
// Back, its book compared: the same, so no Snapshot until the minute.
assert!(
feed.from(1, 70_000, book("M", 30, &[(400_000, 5_000_000)], &[]))
.is_empty()
);
assert_eq!(snapshots(&feed.tick(minute + TimeDelta::minutes(2))), 1);
}
/// The demo's recording on two connections (tests/fixtures/README.md).
fn recording() -> Vec<(DateTime<Utc>, usize, Event)> {
#[derive(Deserialize)]
struct Line {
received_at_us: i64,
connection: usize,
text: String,
}
let path = concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/fixtures/kalshi-frames-two.jsonl"
);
std::fs::read_to_string(path)
.unwrap()
.lines()
.filter_map(|line| {
let line: Line = serde_json::from_str(line).unwrap();
let frame: wire::Frame = serde_json::from_str(&line.text).unwrap();
let event = wire::event(&frame).unwrap()?;
let received_at = DateTime::from_timestamp_micros(line.received_at_us).unwrap();
Some((received_at, line.connection, event))
})
.collect()
}
fn tickers(recording: &[(DateTime<Utc>, usize, Event)]) -> Vec<String> {
let mut tickers: Vec<String> = recording
.iter()
.map(|(_, _, event)| crate::core::Event::market(event).unwrap().to_string())
.collect();
tickers.sort();
tickers.dedup();
tickers
}
/// Each symbol's Deltas, as (side, price, quantity, receive time).
type Written = HashMap<String, Vec<(Side, i64, i64, DateTime<Utc>)>>;
/// Replays the connections in `on`.
fn replay(on: &[usize]) -> (Feed, Written) {
let recording = recording();
let tickers = tickers(&recording);
let tickers: Vec<&str> = tickers.iter().map(String::as_str).collect();
let mut feed = feed_of(&tickers, on);
let mut written: HashMap<_, Vec<_>> = HashMap::new();
for (received_at, connection, event) in recording {
if !on.contains(&connection) {
continue;
}
let arrival = Arrival {
received_at,
connection,
bytes: 0,
event,
};
for delta in deltas(&feed.on_event(arrival)) {
written.entry(delta.symbol.clone()).or_default().push((
delta.side,
delta.price_level,
delta.quantity,
delta.backtest_timestamp,
));
}
}
(feed, written)
}
/// Connection 0 asked for every book again every 3 s: each, compared
/// with ours folded from the changes, was the same.
#[test]
fn the_recorded_changes_fold_into_the_exchanges_books() {
let (feed, written) = replay(&[0]);
let counts = &feed.rule.counts;
assert!(counts.books > 700, "{counts:?}");
assert_eq!(
(counts.losses, counts.below_zero, counts.duplicates),
(0, 0, 0)
);
assert!(written.values().map(Vec::len).sum::<usize>() > 1_000);
// A NO bid arrived as the ask it is: no book is crossed. Taken at
// one minus its price again, a third of them would be.
let mut two_sided = 0;
for (ticker, symbol) in &feed.shell.symbols {
let best_bid = symbol.book.bids.keys().next_back();
let best_ask = symbol.book.asks.keys().next();
if let (Some(bid), Some(ask)) = (best_bid, best_ask) {
assert!(bid < ask, "{ticker}: bid {bid}, ask {ask}");
two_sided += 1;
}
}
assert!(two_sided >= 10, "{two_sided}");
}
/// The book and writer threads as main runs them, fed the recording:
/// every Snapshot is the one before it with the Deltas between.
#[test]
fn the_output_reconciles() {
let recording = recording();
let tickers = tickers(&recording);
let tickers: Vec<&str> = tickers.iter().map(String::as_str).collect();
let feed = feed_of(&tickers, &[0, 1]);
let (events_tx, events_rx) = crossbeam::channel::unbounded();
let (outputs_tx, outputs_rx) = crossbeam::channel::unbounded();
let (connection_tx, _connection_rx) = crossbeam::channel::unbounded();
let (_commands_tx, commands_rx) = crossbeam::channel::unbounded();
let (reports_tx, _reports_rx) = crossbeam::channel::unbounded();
for (received_at, connection, event) in recording {
let arrival = Arrival {
received_at,
connection,
bytes: 0,
event,
};
events_tx.send(Received::Event(arrival)).unwrap();
}
drop(events_tx);
let book = std::thread::spawn(move || {
feed::serve(
feed,
vec![connection_tx.clone(), connection_tx],
events_rx,
commands_rx,
outputs_tx,
reports_tx,
)
});
let path =
std::env::temp_dir().join(format!("kalshi-reconcile-{}.sbe", std::process::id()));
let written = crate::write(path.to_str().unwrap(), outputs_rx);
book.join().unwrap();
written.unwrap();
let report = engineering_challenge::reconcile::reconcile(&path, None);
std::fs::remove_file(&path).unwrap();
let report = report.unwrap();
// The opening Snapshots, and one per symbol at the minute crossed.
assert_eq!(report.snapshots, tickers.len() * 2, "{report:?}");
assert!(report.deltas > 1_000, "{report:?}");
assert_eq!(report.orphan_deltas, 0);
assert_eq!(report.mismatches, []);
}
/// Connection 1 left five markets for 5 s: connection 0 carried them.
#[test]
fn two_recorded_connections_write_the_changes_of_one_each_at_its_first_copy() {
let (one, alone) = replay(&[0]);
let (both, together) = replay(&[0, 1]);
assert_eq!(both.rule.counts.losses, 0);
assert!(both.rule.counts.duplicates > 900, "{:?}", both.rule.counts);
assert_eq!(alone.keys().len(), together.keys().len());
let mut earlier = 0;
for (symbol, changes) in &alone {
let content = |changes: &[(Side, i64, i64, DateTime<Utc>)]| {
changes
.iter()
.map(|&(side, price, quantity, _)| (side, price, quantity))
.collect::<Vec<_>>()
};
assert_eq!(content(changes), content(&together[symbol]), "{symbol}");
for (single, first) in changes.iter().zip(&together[symbol]) {
assert!(first.3 <= single.3);
earlier += usize::from(first.3 < single.3);
}
}
// Connection 1 was first for most changes in the recording.
assert!(earlier > 500, "{earlier}");
assert_eq!(one.rule.counts.written, both.rule.counts.written);
}
}
+334
View File
@@ -0,0 +1,334 @@
//! Kalshi's market listing: which markets are live, or about to be. Two fetcher threads page
//! it on the orchestrator's request and stream what they read; they hold no
//! state and no timer, as Gamma's (roadmap/06). Read without a key.
use std::time::Duration;
use chrono::{DateTime, TimeDelta, Utc};
use crossbeam::channel::{Receiver, Sender};
use serde::Deserialize;
use super::Market;
use crate::core::{Finding, Request, retried};
/// The listing's largest page.
const PAGE_SIZE: &str = "1000";
/// For every request, so SIGINT never waits on the listing longer than this.
const TIMEOUT: Duration = Duration::from_secs(10);
/// A page's tries, and the wait between them.
const TRIES: u32 = 5;
const RETRY_AFTER: Duration = Duration::from_secs(5);
/// A market not yet open is tracked this long before its opening: its book is
/// there, empty, from its first order. Three hourly rescans, so every market
/// is reached by a scan before it opens even when one scan fails.
const OPENING_AHEAD: TimeDelta = TimeDelta::hours(3);
/// A fetcher thread: answers each request until the orchestrator is gone.
pub fn run(listing: &str, requests: Receiver<Request>, findings: Sender<Finding<Market>>) {
let agent = ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.timeout_global(Some(TIMEOUT))
.build(),
);
let markets = format!("{listing}/markets");
for request in requests {
let sent = match request {
Request::Rescan => rescan(&agent, &markets, &findings),
Request::Recent { since } => recent(&agent, &markets, since, &findings),
};
if sent.is_err() {
return;
}
}
}
/// `Err` once the orchestrator is gone.
type Sent = Result<(), crossbeam::channel::SendError<Finding<Market>>>;
fn rescan(agent: &ureq::Agent, markets: &str, findings: &Sender<Finding<Market>>) -> Sent {
findings.send(Finding::ScanStarted)?;
// Unopened first: a market that opens while the scan runs is then on one
// list at least. Open first, it could be on neither, and be removed.
for status in ["unopened", "open"] {
let query = [("status", status.to_string()), mve_excluded()];
let mut cursor = String::new();
loop {
let page = match retried(TRIES, RETRY_AFTER, || page(agent, markets, &query, &cursor)) {
Ok(page) => page,
Err(error) => {
log!("kalshi: rescan failed: {error}");
return findings.send(Finding::ScanFailed);
}
};
let last = page.markets.is_empty() || page.cursor.is_empty();
findings.send(Finding::ScanPage(tracked(page.markets, Utc::now())))?;
if last {
break;
}
cursor = page.cursor;
}
}
findings.send(Finding::ScanDone)
}
fn recent(
agent: &ureq::Agent,
markets: &str,
since: DateTime<Utc>,
findings: &Sender<Finding<Market>>,
) -> Sent {
let since = since.timestamp().to_string();
// Created, whatever their status: open and unopened alike.
let created = [("min_created_ts", since.clone()), mve_excluded()];
let closed = [
("status", "closed".to_string()),
("min_close_ts", since),
mve_excluded(),
];
let result = every_page(agent, markets, &created).and_then(|created| {
findings.send(Finding::Created(tracked(created, Utc::now())))?;
every_page(agent, markets, &closed)
});
match result {
Ok(closed) => {
let tickers = closed.into_iter().map(|market| market.ticker);
findings.send(Finding::Closed(tickers.collect()))?;
findings.send(Finding::RecentDone)
}
Err(error) => {
log!("kalshi: recent check failed: {error}");
findings.send(Finding::RecentFailed)
}
}
}
/// Combos (MVEs) are not in the challenge.
fn mve_excluded() -> (&'static str, String) {
("mve_filter", "exclude".to_string())
}
fn every_page(
agent: &ureq::Agent,
markets: &str,
query: &[(&str, String)],
) -> anyhow::Result<Vec<ListedMarket>> {
let mut listed = Vec::new();
let mut cursor = String::new();
loop {
let page = retried(TRIES, RETRY_AFTER, || page(agent, markets, query, &cursor))?;
let last = page.markets.is_empty() || page.cursor.is_empty();
listed.extend(page.markets);
if last {
return Ok(listed);
}
cursor = page.cursor;
}
}
#[derive(Deserialize)]
struct Page {
markets: Vec<ListedMarket>,
/// Empty on the last page.
#[serde(default)]
cursor: String,
}
#[derive(Deserialize)]
struct ListedMarket {
ticker: String,
/// `active` once open, `initialized` before.
status: String,
open_time: Option<String>,
}
fn page(
agent: &ureq::Agent,
markets: &str,
query: &[(&str, String)],
cursor: &str,
) -> anyhow::Result<Page> {
let mut request = agent
.get(markets)
.query("limit", PAGE_SIZE)
.query_pairs(query.iter().map(|(name, value)| (*name, value.as_str())));
if !cursor.is_empty() {
request = request.query("cursor", cursor);
}
let body = request.call()?.body_mut().read_to_string()?;
Ok(serde_json::from_str(&body)?)
}
/// The markets to track among `markets` at `now`: those live, and those
/// opening within [`OPENING_AHEAD`].
fn tracked(markets: Vec<ListedMarket>, now: DateTime<Utc>) -> Vec<Market> {
markets
.into_iter()
.filter(|market| match market.status.as_str() {
"active" => true,
"initialized" => market
.open_time
.as_deref()
.and_then(|time| DateTime::parse_from_rfc3339(time).ok())
.is_some_and(|opens| opens.to_utc() - now <= OPENING_AHEAD),
_ => false,
})
.map(|market| Market {
ticker: market.ticker,
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
fn fixture(name: &str) -> Page {
let path = format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"));
serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()
}
fn time(text: &str) -> DateTime<Utc> {
DateTime::parse_from_rfc3339(text).unwrap().to_utc()
}
fn tickers(markets: &[Market]) -> Vec<&str> {
markets
.iter()
.map(|market| market.ticker.as_str())
.collect()
}
#[test]
fn open_markets_are_tracked_and_the_last_page_has_no_cursor() {
let page = fixture("kalshi-open.json");
assert!(!page.cursor.is_empty());
let live = tracked(page.markets, time("2026-11-11T19:30:00Z"));
assert_eq!(live.len(), 3);
assert!(live[0].ticker.starts_with("KX"));
let last: Page = serde_json::from_str(r#"{"markets":[],"cursor":""}"#).unwrap();
assert!(last.cursor.is_empty());
}
/// The fixture's unopened markets open at 19:19:08.
#[test]
fn an_unopened_market_is_tracked_from_three_hours_before_its_opening() {
let opening = time("2026-11-11T19:19:08Z");
let at = |now: DateTime<Utc>| tracked(fixture("kalshi-unopened.json").markets, now);
assert!(at(opening - OPENING_AHEAD - TimeDelta::seconds(1)).is_empty());
assert_eq!(at(opening - OPENING_AHEAD).len(), 3);
// Past its opening and still listed unopened: tracked, empty.
assert_eq!(
tickers(&at(opening + TimeDelta::minutes(20)))[0],
"KXEUROCUPTOTAL-26NOV121400MAXSKY-179"
);
}
/// A local listing: answers each request with the fixture its query asks
/// for (an empty last page after a cursor), and tells the queries it got.
fn listing(requests: usize) -> (String, std::thread::JoinHandle<Vec<String>>) {
use std::io::{BufRead, BufReader, Write};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let server = std::thread::spawn(move || {
let mut queries = Vec::new();
for _ in 0..requests {
let (mut stream, _) = listener.accept().unwrap();
let mut line = String::new();
BufReader::new(&stream).read_line(&mut line).unwrap();
let query = line.split(' ').nth(1).unwrap().to_string();
let name = if query.contains("cursor=") {
None
} else if query.contains("status=unopened") {
Some("kalshi-unopened.json")
} else if query.contains("status=closed") {
Some("kalshi-closed.json")
} else {
Some("kalshi-open.json")
};
let body = name.map_or(r#"{"markets":[],"cursor":""}"#.to_string(), |name| {
let path = format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"));
std::fs::read_to_string(path).unwrap()
});
write!(
stream,
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
)
.unwrap();
queries.push(query);
}
queries
});
(url, server)
}
fn found(findings: &[Finding<Market>]) -> Vec<String> {
findings
.iter()
.map(|finding| match finding {
Finding::ScanPage(markets) => format!("page {}", tickers(markets).join(" ")),
Finding::Created(markets) => format!("created {}", tickers(markets).join(" ")),
Finding::Closed(closed) => format!("closed {}", closed.join(" ")),
other => format!("{other:?}"),
})
.collect()
}
#[test]
fn a_rescan_pages_the_unopened_markets_then_the_open_ones() {
let (url, server) = listing(4);
let (requests_tx, requests) = crossbeam::channel::unbounded();
let (findings_tx, findings) = crossbeam::channel::unbounded();
requests_tx.send(Request::Rescan).unwrap();
drop(requests_tx);
run(&url, requests, findings_tx);
let queries = server.join().unwrap();
// Unopened first: a market opening meanwhile is on one list at least.
assert!(queries[0].contains("status=unopened"), "{queries:?}");
assert!(queries[1].contains("status=unopened") && queries[1].contains("cursor="));
assert!(queries[2].contains("status=open") && !queries[2].contains("cursor="));
for query in &queries {
assert!(query.contains("mve_filter=exclude") && query.contains("limit=1000"));
}
let found = found(&findings.try_iter().collect::<Vec<_>>());
assert_eq!(found.len(), 6, "{found:?}");
assert_eq!(found[0], "ScanStarted");
assert!(found[1].starts_with("page KXEUROCUPTOTAL-26NOV121400MAXSKY-179 "));
assert_eq!(found[2], "page ");
assert!(found[3].starts_with("page KXNCAAWBTOTAL-26NOV11QUINSJU-106 "));
assert_eq!(found[5], "ScanDone");
}
#[test]
fn a_recent_check_reports_the_markets_created_and_those_closed() {
let (url, server) = listing(4);
let (requests_tx, requests) = crossbeam::channel::unbounded();
let (findings_tx, findings) = crossbeam::channel::unbounded();
let since = time("2026-11-11T19:00:00Z");
requests_tx.send(Request::Recent { since }).unwrap();
drop(requests_tx);
run(&url, requests, findings_tx);
let queries = server.join().unwrap();
let since = format!("={}", since.timestamp());
assert!(
queries[0].contains(&format!("min_created_ts{since}"))
&& !queries[0].contains("status=")
);
assert!(
queries[2].contains(&format!("min_close_ts{since}"))
&& queries[2].contains("status=closed")
);
let found = found(&findings.try_iter().collect::<Vec<_>>());
assert_eq!(found.len(), 3, "{found:?}");
assert!(found[0].starts_with("created KXNCAAWBTOTAL-26NOV11QUINSJU-106 "));
assert!(found[1].starts_with("closed KXFIBAECUPSPREAD-26NOV111230SZOZIG-SZO23 "));
assert_eq!(found[2], "RecentDone");
}
#[test]
fn a_closed_market_is_not_tracked() {
let closed = fixture("kalshi-closed.json");
assert_eq!(closed.markets[0].status, "determined");
assert!(tracked(closed.markets, time("2026-11-11T19:30:00Z")).is_empty());
}
}
+286
View File
@@ -0,0 +1,286 @@
//! Kalshi: its websocket's `orderbook_delta` channel (the connection's side of
//! it, signed with an API key), its market listing, and the clock rule its
//! book threads apply (docs/notes.md, Kalshi's websocket; roadmap/06).
pub mod feed;
pub mod listing;
pub mod sign;
pub mod wire;
use chrono::Utc;
use crossbeam::channel::{Receiver, Sender};
use serde_json::json;
use tungstenite::client::IntoClientRequest;
use crate::core::{self, Finding, Request, Undecoded};
use sign::Key;
/// Tickers per subscribe or unsubscribe message.
const CHUNK: usize = 500;
/// Errors after which the subscription is gone (Kalshi's docs: a channel
/// error, a subscription buffer overflow): the connection starts over.
const TERMINAL_ERRORS: [u32; 2] = [10, 25];
/// A tracked market: one book, named by its ticker, which is its id and its
/// only symbol.
#[derive(Debug, Clone)]
pub struct Market {
pub ticker: String,
}
impl core::Market for Market {
fn id(&self) -> &str {
&self.ticker
}
fn symbols(&self) -> &[String] {
std::slice::from_ref(&self.ticker)
}
}
/// The exchange: its websocket and listing, and the key that signs the
/// websocket's handshake.
pub struct Kalshi {
pub websocket: String,
pub listing: String,
pub key: Key,
}
impl Kalshi {
pub fn production(key: Key) -> Self {
Self {
websocket: "wss://external-api-ws.kalshi.com/trade-api/ws/v2".into(),
listing: "https://external-api.kalshi.com/trade-api/v2".into(),
key,
}
}
pub fn demo(key: Key) -> Self {
Self {
websocket: "wss://external-api-ws.demo.kalshi.co/trade-api/ws/v2".into(),
listing: "https://external-api.demo.kalshi.co/trade-api/v2".into(),
key,
}
}
}
impl core::Exchange for Kalshi {
type Market = Market;
type Event = wire::Event;
type Session = Session;
type Rule = feed::Clock;
const NAME: &'static str = "kalshi";
fn request(&self) -> anyhow::Result<tungstenite::handshake::client::Request> {
let mut request = self.websocket.as_str().into_client_request()?;
for (name, value) in self.key.headers(Utc::now().timestamp_millis()) {
request.headers_mut().insert(name, value.parse()?);
}
Ok(request)
}
fn session(&self) -> Session {
Session::default()
}
fn rule(&self) -> feed::Clock {
feed::Clock::default()
}
fn fetch(&self, requests: Receiver<Request>, findings: Sender<Finding<Market>>) {
listing::run(&self.listing, requests, findings)
}
}
/// A connection's subscription: one per connection, which every subscribe
/// adds to.
#[derive(Debug, Default)]
pub struct Session {
/// The last command's id.
command: u64,
/// The subscription's id, from its `subscribed` answer.
sid: Option<u64>,
/// The last message's `seq`: the next one must follow it.
seq: Option<u64>,
}
impl Session {
fn command(&mut self, cmd: &str, params: serde_json::Value) -> String {
self.command += 1;
json!({"id": self.command, "cmd": cmd, "params": params}).to_string()
}
}
impl core::Session for Session {
type Event = wire::Event;
/// Nothing: a subscribe can't name no market, so the first one creates the
/// subscription.
fn opening(&mut self) -> Vec<String> {
Vec::new()
}
/// `use_yes_price` makes a NO bid come as the ask it is; sent, since its
/// default is announced to flip.
fn subscribe(&mut self, tickers: &[&String]) -> Vec<String> {
tickers
.chunks(CHUNK)
.map(|chunk| {
let params = json!({
"channels": ["orderbook_delta"],
"market_tickers": chunk,
"use_yes_price": true,
"skip_ticker_ack": true,
});
self.command("subscribe", params)
})
.collect()
}
fn unsubscribe(&mut self, tickers: &[&String]) -> Vec<String> {
// Every fresh connection's subscription was 1 (docs/notes.md): an
// unsubscribe sent before its answer names it so.
let sid = self.sid.unwrap_or(1);
tickers
.chunks(CHUNK)
.map(|chunk| {
let params = json!({
"sids": [sid],
"market_tickers": chunk,
"action": "delete_markets",
"skip_ticker_ack": true,
});
self.command("update_subscription", params)
})
.collect()
}
/// None: Kalshi pings a connection it has nothing else to send to, and
/// tungstenite answers.
fn keepalive(&self) -> Option<&'static str> {
None
}
fn decode(&mut self, text: &str) -> Result<Vec<wire::Event>, Undecoded> {
let frame: wire::Frame =
serde_json::from_str(text).map_err(|error| Undecoded::Unreadable(error.to_string()))?;
if let Some(seq) = frame.seq {
let expected = self.seq.map_or(1, |last| last + 1);
if seq != expected {
return Err(Undecoded::Lost(format!(
"message {seq} came, {expected} expected"
)));
}
self.seq = Some(seq);
}
match frame.kind.as_str() {
"subscribed" => self.sid = self.sid.or(frame.msg.sid),
"error" => {
let (code, text) = (frame.msg.code.unwrap_or(0), frame.msg.msg.as_deref());
let id = frame.id.unwrap_or(0);
if TERMINAL_ERRORS.contains(&code) {
return Err(Undecoded::Lost(format!("error {code} ({text:?})")));
}
log!("kalshi: command {id} refused: error {code} ({text:?})");
}
_ => {}
}
wire::event(&frame)
.map(|event| event.into_iter().collect())
.map_err(Undecoded::Unreadable)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::core::Session as _;
fn decoded(session: &mut Session, text: &str) -> Result<usize, String> {
match session.decode(text) {
Ok(events) => Ok(events.len()),
Err(Undecoded::Lost(why)) => Err(format!("lost: {why}")),
Err(Undecoded::Unreadable(why)) => Err(format!("unreadable: {why}")),
}
}
const DELTA: &str = r#"{"type":"orderbook_delta","sid":1,"seq":SEQ,"msg":{"market_ticker":"M","market_id":"x","clock":5,"price_dollars":"0.5000","delta_fp":"1.00","side":"yes","ts_ms":1}}"#;
fn delta(seq: u64) -> String {
DELTA.replace("SEQ", &seq.to_string())
}
#[test]
fn a_skipped_seq_is_a_lost_message() {
let mut session = Session::default();
let subscribed =
r#"{"type":"subscribed","id":1,"msg":{"channel":"orderbook_delta","sid":3}}"#;
assert_eq!(decoded(&mut session, subscribed), Ok(0));
assert_eq!(session.sid, Some(3));
assert_eq!(decoded(&mut session, &delta(1)), Ok(1));
// An answer counts too.
let ok = r#"{"type":"ok","id":2,"sid":3,"seq":2,"msg":{}}"#;
assert_eq!(decoded(&mut session, ok), Ok(0));
assert_eq!(decoded(&mut session, &delta(3)), Ok(1));
assert_eq!(
decoded(&mut session, &delta(5)),
Err("lost: message 5 came, 4 expected".into())
);
// A connection's first message is 1.
assert!(decoded(&mut Session::default(), &delta(2)).is_err());
}
#[test]
fn a_terminal_error_starts_over_and_others_are_logged() {
let error = |code: u32| {
format!(r#"{{"type":"error","id":4,"seq":1,"msg":{{"code":{code},"msg":"x"}}}}"#)
};
assert_eq!(decoded(&mut Session::default(), &error(14)), Ok(0));
assert!(
decoded(&mut Session::default(), &error(25))
.unwrap_err()
.starts_with("lost")
);
assert!(
decoded(&mut Session::default(), "not json")
.unwrap_err()
.starts_with("unreadable")
);
}
#[test]
fn commands_carry_their_ids_and_the_subscription() {
let mut session = Session::default();
let tickers: Vec<String> = (0..501).map(|n| format!("T{n}")).collect();
let refs: Vec<&String> = tickers.iter().collect();
let sent: Vec<serde_json::Value> = session
.subscribe(&refs)
.iter()
.map(|text| serde_json::from_str(text).unwrap())
.collect();
assert_eq!(sent.len(), 2);
assert_eq!(sent[0]["id"], 1);
assert_eq!(sent[1]["id"], 2);
assert_eq!(sent[0]["cmd"], "subscribe");
assert_eq!(
sent[0]["params"]["market_tickers"]
.as_array()
.unwrap()
.len(),
500
);
assert_eq!(sent[0]["params"]["use_yes_price"], true);
session.sid = Some(7);
let unsubscribe: serde_json::Value =
serde_json::from_str(&session.unsubscribe(&refs[..1])[0]).unwrap();
assert_eq!(unsubscribe["id"], 3);
assert_eq!(unsubscribe["cmd"], "update_subscription");
assert_eq!(unsubscribe["params"]["sids"], json!([7]));
assert_eq!(unsubscribe["params"]["action"], "delete_markets");
assert_eq!(unsubscribe["params"]["market_tickers"], json!(["T0"]));
}
}
#[cfg(test)]
mod socket_tests;
+186
View File
@@ -0,0 +1,186 @@
//! The key that signs Kalshi's websocket handshake (docs/notes.md, Kalshi's
//! websocket): an Ed25519 key, read once from the files that the environment
//! names. Neither file's content is ever logged.
use std::path::Path;
use base64::Engine;
use ring::signature::Ed25519KeyPair;
use rustls_pki_types::PrivateKeyDer;
use rustls_pki_types::pem::PemObject;
/// The variables naming the two files.
pub const KEY_ID_FILE: &str = "KALSHI_KEY_ID_FILE";
pub const PRIVATE_KEY_FILE: &str = "KALSHI_PRIVATE_KEY_FILE";
/// What the handshake signs, after the time: the method and the path.
const SIGNED_PATH: &str = "GET/trade-api/ws/v2";
pub struct Key {
id: String,
pair: Ed25519KeyPair,
}
impl Key {
/// The key from the files [`KEY_ID_FILE`] and [`PRIVATE_KEY_FILE`] name.
pub fn from_env() -> anyhow::Result<Self> {
let path = |variable: &str| {
std::env::var_os(variable)
.ok_or_else(|| anyhow::anyhow!("{variable} is not set: it names a file of the key"))
};
Self::from_files(
path(KEY_ID_FILE)?.as_ref(),
path(PRIVATE_KEY_FILE)?.as_ref(),
)
.map_err(|error| {
anyhow::anyhow!("{error} (the files {KEY_ID_FILE} and {PRIVATE_KEY_FILE} name)")
})
}
/// The key id in `id_file`, the private key (a PKCS#8 PEM) in `key_file`.
pub fn from_files(id_file: &Path, key_file: &Path) -> anyhow::Result<Self> {
let id = std::fs::read_to_string(id_file)
.map_err(|error| anyhow::anyhow!("{}: {error}", id_file.display()))?
.trim()
.to_string();
anyhow::ensure!(!id.is_empty(), "{}: no key id in it", id_file.display());
let pem = std::fs::read(key_file)
.map_err(|error| anyhow::anyhow!("{}: {error}", key_file.display()))?;
// The errors name the file, never its content: the PEM reader's own
// error quotes what it read, so it is dropped.
let der = PrivateKeyDer::from_pem_slice(&pem)
.map_err(|_| anyhow::anyhow!("{}: not a PEM private key", key_file.display()))?;
let PrivateKeyDer::Pkcs8(der) = der else {
anyhow::bail!(
"{}: not a PKCS#8 key; Kalshi's Ed25519 keys are",
key_file.display()
);
};
let pair = Ed25519KeyPair::from_pkcs8_maybe_unchecked(der.secret_pkcs8_der())
.map_err(|_| anyhow::anyhow!("{}: not an Ed25519 key", key_file.display()))?;
Ok(Self { id, pair })
}
/// The handshake's three headers at `millis` since the epoch.
pub fn headers(&self, millis: i64) -> [(&'static str, String); 3] {
let signature = self.pair.sign(format!("{millis}{SIGNED_PATH}").as_bytes());
[
("KALSHI-ACCESS-KEY", self.id.clone()),
("KALSHI-ACCESS-TIMESTAMP", millis.to_string()),
(
"KALSHI-ACCESS-SIGNATURE",
base64::engine::general_purpose::STANDARD.encode(signature.as_ref()),
),
]
}
}
/// RFC 8032's first test key (section 7.1), as PKCS#8: a known key, so a
/// known signature.
#[cfg(test)]
const TEST_KEY: &str = "-----BEGIN PRIVATE KEY-----
MC4CAQAwBQYDK2VwBCIEIJ1hsZ3v/VpguoRK9JLsLMREScVpezJpGXA7rAMcrn9g
-----END PRIVATE KEY-----
";
/// [`TEST_KEY`], with the id `a-key-id`.
#[cfg(test)]
pub fn test_key() -> Key {
let der = PrivateKeyDer::from_pem_slice(TEST_KEY.as_bytes()).unwrap();
let PrivateKeyDer::Pkcs8(der) = der else {
unreachable!("PKCS#8")
};
Key {
id: "a-key-id".into(),
pair: Ed25519KeyPair::from_pkcs8_maybe_unchecked(der.secret_pkcs8_der()).unwrap(),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn write(dir: &Path, name: &str, text: &str) -> std::path::PathBuf {
let path = dir.join(name);
std::fs::write(&path, text).unwrap();
path
}
fn scratch(name: &str) -> std::path::PathBuf {
let dir = std::env::temp_dir().join(format!("kalshi-sign-{name}-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn signs_the_time_method_and_path() {
let dir = scratch("signs");
let key = Key::from_files(
&write(&dir, "id", "a-key-id\n"),
&write(&dir, "key.pem", TEST_KEY),
)
.unwrap();
let [id, time, signature] = key.headers(1_669_149_841_234);
assert_eq!(id, ("KALSHI-ACCESS-KEY", "a-key-id".to_string()));
assert_eq!(
time,
("KALSHI-ACCESS-TIMESTAMP", "1669149841234".to_string())
);
assert_eq!(signature.0, "KALSHI-ACCESS-SIGNATURE");
// Checked against the RFC's public key for this one.
let public = ring::signature::UnparsedPublicKey::new(
&ring::signature::ED25519,
[
0xd7, 0x5a, 0x98, 0x01, 0x82, 0xb1, 0x0a, 0xb7, 0xd5, 0x4b, 0xfe, 0xd3, 0xc9, 0x64,
0x07, 0x3a, 0x0e, 0xe1, 0x72, 0xf3, 0xda, 0xa6, 0x23, 0x25, 0xaf, 0x02, 0x1a, 0x68,
0xf7, 0x07, 0x51, 0x1a,
],
);
let bytes = base64::engine::general_purpose::STANDARD
.decode(&signature.1)
.unwrap();
public
.verify(b"1669149841234GET/trade-api/ws/v2", &bytes)
.unwrap();
// Ed25519 is deterministic: the same time, the same signature.
assert_eq!(key.headers(1_669_149_841_234)[2], signature);
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn a_missing_or_wrong_file_says_which_and_never_what_it_holds() {
let dir = scratch("errors");
let id = write(&dir, "id", "a-key-id");
let error = |id: &Path, key: &Path| Key::from_files(id, key).err().unwrap().to_string();
let missing = error(&id, &dir.join("absent.pem"));
assert!(missing.contains("absent.pem"), "{missing}");
// A key flattened onto one line: the PEM reader's own error quotes
// what it read, so ours must not carry it.
let flat = write(&dir, "flat.pem", &TEST_KEY.replace('\n', " "));
let flattened = error(&id, &flat);
assert!(flattened.contains("not a PEM private key"), "{flattened}");
assert!(flattened.ends_with("not a PEM private key"), "{flattened}");
let garbage = write(&dir, "garbage.pem", "not a key at all");
let unreadable = error(&id, &garbage);
assert!(
unreadable.ends_with("not a PEM private key"),
"{unreadable}"
);
// An RSA key in PKCS#1, as Kalshi generates its RSA keys: refused.
let rsa = write(
&dir,
"rsa.pem",
"-----BEGIN RSA PRIVATE KEY-----\nMAA=\n-----END RSA PRIVATE KEY-----\n",
);
assert!(error(&id, &rsa).contains("not a PKCS#8 key"));
let empty = write(&dir, "empty-id", "\n");
let key = write(&dir, "key.pem", TEST_KEY);
assert!(error(&empty, &key).contains("no key id"));
std::fs::remove_dir_all(dir).unwrap();
}
}
+159
View File
@@ -0,0 +1,159 @@
//! The connection loop (`src/core/socket.rs`) against a local server, with
//! Kalshi's codec: the signed handshake, the server's ping, and a skipped
//! `seq`.
use std::net::TcpListener;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;
use std::time::Duration;
use crossbeam::channel::unbounded;
use tungstenite::handshake::server::{Request, Response};
use tungstenite::{Bytes, Message};
use super::{Kalshi, sign};
use crate::core::socket::{Command, Notice, SILENT_AFTER, run};
use crate::core::{BOOK_THREADS, Received};
const SNAPSHOT: &str = r#"{"type":"orderbook_snapshot","sid":1,"seq":1,"msg":{"market_ticker":"M","market_id":"x","clock":10,"yes_dollars_fp":[["0.4000","5.00"]]},"sending_ts_ms":1794422918369}"#;
fn delta(seq: u64) -> String {
format!(
r#"{{"type":"orderbook_delta","sid":1,"seq":{seq},"msg":{{"market_ticker":"M","market_id":"x","clock":{seq}0,"price_dollars":"0.4000","delta_fp":"1.00","side":"yes","ts_ms":1794422918370}}}}"#
)
}
#[test]
// The handshake callback's error type is tungstenite's, not ours.
#[allow(clippy::result_large_err)]
fn signs_its_handshake_answers_pings_and_reconnects_on_a_skipped_seq() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("ws://{}/trade-api/ws/v2", listener.local_addr().unwrap());
let server = thread::spawn(move || {
let mut seen = Vec::new();
for connection in 0..2 {
let (stream, _) = listener.accept().unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
let mut headers = Vec::new();
let mut socket =
tungstenite::accept_hdr(stream, |request: &Request, response: Response| {
for name in [
"KALSHI-ACCESS-KEY",
"KALSHI-ACCESS-TIMESTAMP",
"KALSHI-ACCESS-SIGNATURE",
] {
let value = request
.headers()
.get(name)
.map(|value| value.to_str().unwrap().to_string());
headers.push(value);
}
Ok(response)
})
.unwrap();
let subscribe: serde_json::Value =
serde_json::from_str(socket.read().unwrap().to_text().unwrap()).unwrap();
assert_eq!(subscribe["cmd"], "subscribe");
assert_eq!(
subscribe["params"]["market_tickers"],
serde_json::json!(["M"])
);
socket
.send(Message::text(
r#"{"type":"subscribed","id":1,"msg":{"channel":"orderbook_delta","sid":1}}"#,
))
.unwrap();
socket.send(Message::text(SNAPSHOT)).unwrap();
socket.send(Message::text(delta(2))).unwrap();
if connection == 1 {
socket.close(None).unwrap();
while socket.read().is_ok() {}
seen.push((headers, false));
break;
}
socket
.send(Message::Ping(Bytes::from_static(b"heartbeat")))
.unwrap();
let ponged = loop {
match socket.read().unwrap() {
Message::Pong(body) => break body == Bytes::from_static(b"heartbeat"),
_ => continue,
}
};
// 3 never comes.
socket.send(Message::text(delta(4))).unwrap();
while socket.read().is_ok() {}
seen.push((headers, ponged));
}
seen
});
let kalshi = Kalshi {
websocket: url,
listing: String::new(),
key: sign::test_key(),
};
let (book_threads, events): (Vec<_>, Vec<_>) = (0..BOOK_THREADS).map(|_| unbounded()).unzip();
let (commands_tx, commands) = unbounded();
commands_tx
.send(Command::Subscribe(vec!["M".into()]))
.unwrap();
let (notices_tx, notices) = unbounded();
let stop = Arc::new(AtomicBool::new(false));
let client_stop = Arc::clone(&stop);
let client = thread::spawn(move || {
run(
&kalshi,
0,
&book_threads,
&commands,
&notices_tx,
SILENT_AFTER,
&client_stop,
)
});
let seen = server.join().unwrap();
stop.store(true, Ordering::Relaxed);
client.join().unwrap();
for (headers, _) in &seen {
assert_eq!(headers[0].as_deref(), Some("a-key-id"));
assert!(
headers[1]
.as_deref()
.is_some_and(|time| time.parse::<i64>().is_ok())
);
assert!(
headers[2]
.as_deref()
.is_some_and(|signature| signature.len() == 88)
);
}
assert!(seen[0].1, "the ping got its pong");
let notices: Vec<Notice> = notices.try_iter().collect();
assert_eq!(
notices[..3],
[Notice::Up(0), Notice::Down(0), Notice::Up(0)]
);
// The book and the change before the gap reached the book thread; the
// change after it didn't.
let received: Vec<_> = events
.iter()
.flat_map(|receiver| receiver.try_iter())
.collect();
// Every book thread heard it go down each time: after the skipped
// `seq`, and when the server closed the second connection.
let downs = received
.iter()
.filter(|received| matches!(received, Received::Down(0)))
.count();
assert_eq!(downs, 2 * BOOK_THREADS);
let events: Vec<_> = received
.iter()
.filter(|received| matches!(received, Received::Event(_)))
.collect();
assert_eq!(events.len(), 4, "two per connection: {events:?}");
}
+213
View File
@@ -0,0 +1,213 @@
//! Kalshi's websocket messages, as the exchange writes them, and how a
//! connection reads them: each frame is one message, numbered by `seq` across
//! the connection (docs/notes.md, Kalshi's websocket).
use chrono::{DateTime, Utc};
use engineering_challenge::Side;
use engineering_challenge::book::parse_fixed;
use serde::Deserialize;
/// A message of the `orderbook_delta` channel, read.
#[derive(Debug, PartialEq, Eq)]
pub enum Event {
Book(BookMessage),
Change(ChangeMessage),
}
/// A market's full book: its YES bids are the bids; with `use_yes_price`,
/// its NO bids come as the asks they are, at one minus their price.
#[derive(Debug, PartialEq, Eq)]
pub struct BookMessage {
pub ticker: String,
/// The clock of the last change it holds (0 before the market opens).
pub clock: u64,
pub bids: Vec<(i64, i64)>,
pub asks: Vec<(i64, i64)>,
/// When the exchange sent it: a book has no time of its own.
pub sent_at: DateTime<Utc>,
}
/// A change of one level: its quantity moves by `delta`, which can be
/// negative.
#[derive(Debug, PartialEq, Eq)]
pub struct ChangeMessage {
pub ticker: String,
pub clock: u64,
pub side: Side,
pub price: i64,
pub delta: i64,
pub at: DateTime<Utc>,
}
impl crate::core::Event for Event {
fn market(&self) -> Option<&str> {
Some(match self {
Event::Book(book) => &book.ticker,
Event::Change(change) => &change.ticker,
})
}
fn is_book(&self) -> bool {
matches!(self, Event::Book(_))
}
}
/// What a frame holds, every field the channel's messages use.
#[derive(Debug, Deserialize)]
pub struct Frame {
#[serde(rename = "type")]
pub kind: String,
/// The command it answers, for an answer.
pub id: Option<u64>,
/// Counts every message of the connection's subscription but
/// `subscribed`, from 1.
pub seq: Option<u64>,
pub sending_ts_ms: Option<i64>,
#[serde(default)]
pub msg: Body,
}
#[derive(Debug, Default, Deserialize)]
pub struct Body {
pub market_ticker: Option<String>,
pub clock: Option<u64>,
pub yes_dollars_fp: Option<Vec<(String, String)>>,
pub no_dollars_fp: Option<Vec<(String, String)>>,
pub price_dollars: Option<String>,
pub delta_fp: Option<String>,
pub side: Option<String>,
pub ts_ms: Option<i64>,
/// `subscribed`: the subscription's id.
pub sid: Option<u64>,
/// `error`: its code and text.
pub code: Option<u32>,
pub msg: Option<String>,
}
/// The event a frame of the book channel holds; none for the others (answers
/// to commands), `Err` with what is wrong for a book or change unreadable.
pub fn event(frame: &Frame) -> Result<Option<Event>, String> {
let body = &frame.msg;
let ticker = || body.market_ticker.clone().ok_or("no market_ticker");
let clock = || body.clock.ok_or("no clock");
match frame.kind.as_str() {
"orderbook_snapshot" => {
let side = |levels: &Option<Vec<(String, String)>>| {
levels
.iter()
.flatten()
.map(|(price, quantity)| Some((parse_fixed(price)?, parse_fixed(quantity)?)))
.collect::<Option<Vec<_>>>()
.ok_or("a level is not two decimals")
};
let sent_at = frame
.sending_ts_ms
.and_then(DateTime::from_timestamp_millis)
.ok_or("no sending_ts_ms")?;
Ok(Some(Event::Book(BookMessage {
ticker: ticker()?,
clock: clock()?,
bids: side(&body.yes_dollars_fp)?,
asks: side(&body.no_dollars_fp)?,
sent_at,
})))
}
"orderbook_delta" => {
let side = match body.side.as_deref() {
Some("yes") => Side::Buy,
Some("no") => Side::Sell,
_ => return Err("side is neither yes nor no".into()),
};
let price = body.price_dollars.as_deref().and_then(parse_fixed);
let delta = body.delta_fp.as_deref().and_then(parse_signed);
let at = body.ts_ms.and_then(DateTime::from_timestamp_millis);
let (Some(price), Some(delta), Some(at)) = (price, delta, at) else {
return Err("price_dollars, delta_fp or ts_ms unreadable".into());
};
Ok(Some(Event::Change(ChangeMessage {
ticker: ticker()?,
clock: clock()?,
side,
price,
delta,
at,
})))
}
_ => Ok(None),
}
}
/// A signed decimal (`"-54.00"`, `"20.00"`), exactly.
fn parse_signed(text: &str) -> Option<i64> {
match text.strip_prefix('-') {
Some(magnitude) => parse_fixed(magnitude).map(|value| -value),
None => parse_fixed(text),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn read(text: &str) -> Result<Option<Event>, String> {
event(&serde_json::from_str(text).unwrap())
}
#[test]
fn reads_a_book_and_a_change_as_recorded() {
let book = r#"{"type":"orderbook_snapshot","sid":1,"seq":2,"msg":{"market_ticker":"KXHIGHNY-26NOV12-T59","market_id":"01a1","clock":56884934192,"yes_dollars_fp":[["0.4800","80.00"],["0.6500","10.00"]],"no_dollars_fp":[["0.9100","80.00"],["0.6600","7.00"]]},"sending_ts_ms":1794422918369}"#;
assert_eq!(
read(book).unwrap(),
Some(Event::Book(BookMessage {
ticker: "KXHIGHNY-26NOV12-T59".into(),
clock: 56_884_934_192,
bids: vec![(480_000, 80_000_000), (650_000, 10_000_000)],
asks: vec![(910_000, 80_000_000), (660_000, 7_000_000)],
sent_at: DateTime::from_timestamp_millis(1_794_422_918_369).unwrap(),
}))
);
let change = r#"{"type":"orderbook_delta","sid":1,"seq":9,"msg":{"market_ticker":"M","market_id":"01a1","clock":56885842636,"price_dollars":"0.8100","delta_fp":"-1.50","side":"no","ts":"2026-11-11T18:36:37.012Z","ts_ms":1794422197012},"sending_ts_ms":1794422197023}"#;
assert_eq!(
read(change).unwrap(),
Some(Event::Change(ChangeMessage {
ticker: "M".into(),
clock: 56_885_842_636,
side: Side::Sell,
price: 810_000,
delta: -1_500_000,
at: DateTime::from_timestamp_millis(1_794_422_197_012).unwrap(),
}))
);
}
#[test]
fn an_empty_book_has_no_levels_and_answers_are_no_events() {
// An unopened market's book, sides omitted.
let empty = r#"{"type":"orderbook_snapshot","sid":1,"seq":2,"msg":{"market_ticker":"KXQUICKSETTLE-11NOV26H1400-2","market_id":"","clock":0},"sending_ts_ms":1794422918369}"#;
let Some(Event::Book(book)) = read(empty).unwrap() else {
panic!("not a book");
};
assert!(book.bids.is_empty() && book.asks.is_empty());
assert_eq!(book.clock, 0);
for answer in [
r#"{"type":"subscribed","id":1,"msg":{"channel":"orderbook_delta","sid":1},"sending_ts_ms":1}"#,
r#"{"type":"ok","id":2,"sid":1,"seq":6,"sending_ts_ms":1,"msg":{}}"#,
r#"{"type":"error","id":2,"sid":1,"seq":860,"msg":{"code":14,"msg":"Market Ticker required"}}"#,
] {
assert_eq!(read(answer).unwrap(), None, "{answer}");
}
}
#[test]
fn an_unreadable_change_says_so() {
for bad in [
r#"{"type":"orderbook_delta","seq":1,"msg":{"market_ticker":"M","clock":1,"price_dollars":"0.81","delta_fp":"1e3","side":"no","ts_ms":1}}"#,
r#"{"type":"orderbook_delta","seq":1,"msg":{"market_ticker":"M","clock":1,"price_dollars":"0.81","delta_fp":"1.00","side":"maybe","ts_ms":1}}"#,
r#"{"type":"orderbook_delta","seq":1,"msg":{"clock":1,"price_dollars":"0.81","delta_fp":"1.00","side":"no","ts_ms":1}}"#,
] {
assert!(read(bad).is_err(), "{bad}");
}
assert_eq!(parse_signed("-0.01"), Some(-10_000));
assert_eq!(parse_signed("--1"), None);
}
}