This repository has been archived on 2026-11-16. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
orderbook-collector/scripts/latency/test_analyse.py
T
pascal 515e85941d feat(latency): latency experiments, LATENCY.md, Kalshi's slow-connection recycle
Extension 4. `engineering-challenge latency EXCHANGE OUT.csv` opens 16
connections to the same busy markets and writes every copy of every change;
`scripts/latency/analyse.py` gives each connection's speed, whether it
persists, what keeping the fastest would gain, lateness by address, and lag
with the clock's offset. LATENCY.md writes up four runs: on Polymarket speed
persists but is worth under a millisecond, at the median only; on Kalshi's
demo keeping the fastest pair lowers p99 by 28% and 42%, the threshold
agreed before the runs.

So the collector now recycles a Kalshi connection whose mean lateness stays,
3 minutes running, in the slowest quarter and above twice everyone's, ahead
of its hourly turn. Kalshi's rule measures each duplicate's lateness; the
orchestrator judges once a minute. A duplicate's lag in Polymarket's minute
line is now its "lateness".

Roadmap milestone 7, roadmap/07.
2026-11-12 15:05:25 +01:00

204 lines
8.3 KiB
Python

"""analyse.py on runs small enough to work out by hand.
Run from `scripts/ci.sh`: `python3 -m unittest discover -s scripts/latency`.
"""
import unittest
import analyse
S = 1_000_000_000_000 # the run's first change, µs
MIN = analyse.MINUTE_US
ADDRESS = {0: "A", 1: "B", 2: "A"}
def csv(copies, notes=()):
"""`copies`: (connection, change, received_us, exchange_ms)."""
lines = ["connection,address,symbol,identity,received_us,exchange_ms"]
lines += [f"# {note}" for note in notes]
for connection, change, received, exchange_ms in copies:
lines.append(f"{connection},{ADDRESS[connection]},s,{change},{received},{exchange_ms}")
return [line + "\n" for line in lines]
# Minute 1: c1 and c2 (connection 2 misses c2); minute 2: c3 and c4; c5 only
# ends minute 2, its own minute partial. Lateness in µs:
# conn 0 conn 1 conn 2
# c1 0 1000 3000
# c2 0 2000 -
# c3 1000 0 4000
# c4 3000 0 5000
RUN = csv(
[
(0, "c1", S, (S - 10_000) // 1000),
(1, "c1", S + 1000, 0),
(2, "c1", S + 3000, 0),
# A server's repeat on connection 1: its first copy counts.
(1, "c1", S + 50_000, 0),
(0, "c2", S + 10_000_000, 0),
(1, "c2", S + 10_002_000, 0),
(1, "c3", S + MIN, 0),
(0, "c3", S + MIN + 1000, 0),
(2, "c3", S + MIN + 4000, 0),
(1, "c4", S + MIN + 10_000_000, 0),
(0, "c4", S + MIN + 10_003_000, 0),
(2, "c4", S + MIN + 10_005_000, 0),
(0, "c5", S + 2 * MIN + 5_000_000, 0),
(1, "c5", S + 2 * MIN + 5_000_000, 0),
(2, "c5", S + 2 * MIN + 5_000_000, 0),
],
notes=["offset,start,400,3000", "offset,end,600,3000"],
)
class Analyse(unittest.TestCase):
def setUp(self):
self.run = analyse.Run(RUN)
self.minutes = self.run.minutes()
def test_full_minutes_only(self):
self.assertEqual([len(minute) for minute in self.minutes], [2, 2])
def test_the_first_copy_is_not_late_and_a_repeat_is_ignored(self):
lateness, _ = self.run.lateness(self.minutes[0], [0])
self.assertEqual(lateness, [0, 0])
lateness, _ = self.run.lateness(self.minutes[0], [1])
self.assertEqual(lateness, [1000, 2000])
def test_a_change_a_connection_lacks_is_missed_by_it(self):
changes = self.minutes[0] + self.minutes[1]
self.assertEqual([self.run.lateness(changes, [c])[1] for c in (0, 1, 2)], [0, 0, 1])
def test_speeds_and_their_persistence(self):
self.assertEqual(analyse.speeds(self.run, self.minutes[0]), {0: 0, 1: 1500, 2: 3000})
self.assertEqual(analyse.speeds(self.run, self.minutes[1]), {0: 2000, 1: 0, 2: 4500})
# Ranks (1, 2, 3) then (2, 1, 3).
self.assertEqual(analyse.persistence(self.run, self.minutes), [0.5])
def test_the_fastest_of_minute_one_against_the_others(self):
top, rest = analyse.top_against_rest(self.run, self.minutes, top=1)
self.assertEqual(top, (2000, 3000))
# [0, 0, 4000, 5000]: the median between, p99 the last.
self.assertEqual(rest, (2000, 5000))
def test_racing(self):
# Kept: connections 0 and 1, the sooner copy 0 for c3 and c4. At
# random: (0, 1) [0, 0], (0, 2) [1000, 3000] (median 2000, p99 3000),
# (1, 2) [0, 0].
self.assertEqual(
analyse.racing(self.run, self.minutes, pair=2), ((0, 0), 0, (2000 / 3, 1000), 0)
)
def test_by_address(self):
changes = self.minutes[0] + self.minutes[1]
self.assertEqual(
analyse.by_address(self.run, changes),
# A: [0, 0, 1000, 3000] and [3000, 4000, 5000]; B: [1000, 2000, 0, 0].
{"A": (3000, 5000), "B": (500, 2000)},
)
def test_lag_raw_and_corrected_by_the_mean_offset(self):
raw, corrected, offset = analyse.lag(self.run, self.minutes[0][:1])
self.assertEqual((raw, corrected, offset), ((10_000, 10_000), (10_500, 10_500), 500))
def test_speed_is_the_median_and_racing_keeps_the_fastest(self):
# Minute 1: lateness 0, 0, 0 on connection 0; 0, 0, 3000 on 1 (its
# median 0, its mean 1000); 2000 each on 2. Order 0, 1, 2, so the
# pair kept is (0, 1). Minute 2: connection 1 is now the slowest.
copies = []
for n, late in enumerate([0, 0, 3000]):
at = S + n * 1_000_000
copies += [(0, f"a{n}", at, 0), (1, f"a{n}", at + late, 0), (2, f"a{n}", at + 2000, 0)]
copies += [
(0, "b", S + MIN, 0),
(1, "b", S + MIN + 9000, 0),
(2, "b", S + MIN + 1000, 0),
(0, "end", S + 2 * MIN + 1, 0),
]
run = analyse.Run(csv(copies))
minutes = run.minutes()
self.assertEqual(analyse.speeds(run, minutes[0]), {0: 0, 1: 0, 2: 2000})
# Kept (0, 1): 0. At random: (0, 1) 0, (0, 2) 0, (1, 2) 1000.
picked, _, random, _ = analyse.racing(run, minutes, pair=2)
self.assertEqual((picked, random), ((0, 0), (1000 / 3, 1000 / 3)))
def test_p99_is_not_the_maximum(self):
self.assertEqual(analyse.quantile(range(200), 0.99), 198)
def test_the_report_of_the_hand_worked_run(self):
report = analyse.report(self.run)
# Connection 2's speeds, and the change it missed.
self.assertIn("| 2 | A | 3.00 4.50 | 1 |", report)
self.assertIn("rank correlation minute to minute 0.50, mean 0.50", report)
# Three connections: no 4 fastest against the others.
self.assertIn("The 4 fastest of minute 1 against the others: too few connections", report)
# Kept pair p99 0, pairs at random 1000 on average: 100% lower.
self.assertIn("a pair at random, 0.67 ms, 1.00 ms, 0.0 missed; p99 100% lower", report)
def test_racing_counts_what_a_pair_missed(self):
# Minute 1 ranks 0, 1, 2; after it, one change only connection 2
# delivers: missed by the kept pair (0, 1), by one pair in three.
run = analyse.Run(
csv(
[
(0, "a", S, 0),
(1, "a", S + 1000, 0),
(2, "a", S + 2000, 0),
(2, "b", S + MIN, 0),
(0, "c", S + MIN + 1000, 0),
(1, "c", S + MIN + 1000, 0),
(2, "c", S + MIN + 1000, 0),
(0, "end", S + 2 * MIN + 1, 0),
]
)
)
_, picked_missed, _, random_missed = analyse.racing(run, run.minutes(), pair=2)
self.assertEqual((picked_missed, random_missed), (1, 1 / 3))
def test_an_offset_the_probe_could_not_measure_is_left_out(self):
run = analyse.Run(csv([], notes=["offset,start,,", "offset,end,600,3000"]))
self.assertEqual(run.offsets, [600])
def test_thin_runs_are_reported_without_a_traceback(self):
# One connection over two minutes: nothing to compare.
alone = [(0, "a", S, 0), (0, "b", S + MIN, 0), (0, "end", S + 2 * MIN + 1, 0)]
report = analyse.report(analyse.Run(csv(alone)))
self.assertIn("Racing: too few connections deliver after minute 1", report)
# The kept pair, and every other, silent after minute 1 but one.
silent = [
(0, "a", S, 0),
(1, "a", S + 1000, 0),
(2, "a", S + 2000, 0),
(2, "end", S + 2 * MIN + 1, 0),
]
analyse.report(analyse.Run(csv(silent)))
def test_a_run_with_no_change_is_reported_as_such(self):
self.assertEqual(
analyse.report(analyse.Run(csv([]))), "0 connections, no change over a full minute"
)
def test_a_connection_is_missing_only_while_up(self):
run = analyse.Run(
csv(
[
# c1 before connection 2's first copy, c3 after it went
# down: neither is missed by it.
(0, "c1", S, 0),
(2, "c2", S + 1000, 0),
(0, "c2", S + 1000, 0),
(0, "c3", S + 3000, 0),
(0, "c4", S + MIN + 1, 0),
],
notes=[f"down,2,{S + 2000}"],
)
)
self.assertEqual(run.lateness(run.minutes()[0], [2])[1], 0)
# Without the note, c3 is missed.
run.down.clear()
self.assertEqual(run.lateness(run.minutes()[0], [2])[1], 1)
if __name__ == "__main__":
unittest.main()