Archived
feat(latency): latency experiments, LATENCY.md, Kalshi's slow-connection recycle
Extension 4. `engineering-challenge latency EXCHANGE OUT.csv` opens 16 connections to the same busy markets and writes every copy of every change; `scripts/latency/analyse.py` gives each connection's speed, whether it persists, what keeping the fastest would gain, lateness by address, and lag with the clock's offset. LATENCY.md writes up four runs: on Polymarket speed persists but is worth under a millisecond, at the median only; on Kalshi's demo keeping the fastest pair lowers p99 by 28% and 42%, the threshold agreed before the runs. So the collector now recycles a Kalshi connection whose mean lateness stays, 3 minutes running, in the slowest quarter and above twice everyone's, ahead of its hourly turn. Kalshi's rule measures each duplicate's lateness; the orchestrator judges once a minute. A duplicate's lag in Polymarket's minute line is now its "lateness". Roadmap milestone 7, roadmap/07.
This commit is contained in:
22 files changed
+2132
-55
No files matched your search
@@ -0,0 +1,203 @@
|
||||
"""analyse.py on runs small enough to work out by hand.
|
||||
|
||||
Run from `scripts/ci.sh`: `python3 -m unittest discover -s scripts/latency`.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
import analyse
|
||||
|
||||
S = 1_000_000_000_000 # the run's first change, µs
|
||||
MIN = analyse.MINUTE_US
|
||||
ADDRESS = {0: "A", 1: "B", 2: "A"}
|
||||
|
||||
|
||||
def csv(copies, notes=()):
|
||||
"""`copies`: (connection, change, received_us, exchange_ms)."""
|
||||
lines = ["connection,address,symbol,identity,received_us,exchange_ms"]
|
||||
lines += [f"# {note}" for note in notes]
|
||||
for connection, change, received, exchange_ms in copies:
|
||||
lines.append(f"{connection},{ADDRESS[connection]},s,{change},{received},{exchange_ms}")
|
||||
return [line + "\n" for line in lines]
|
||||
|
||||
|
||||
# Minute 1: c1 and c2 (connection 2 misses c2); minute 2: c3 and c4; c5 only
|
||||
# ends minute 2, its own minute partial. Lateness in µs:
|
||||
# conn 0 conn 1 conn 2
|
||||
# c1 0 1000 3000
|
||||
# c2 0 2000 -
|
||||
# c3 1000 0 4000
|
||||
# c4 3000 0 5000
|
||||
RUN = csv(
|
||||
[
|
||||
(0, "c1", S, (S - 10_000) // 1000),
|
||||
(1, "c1", S + 1000, 0),
|
||||
(2, "c1", S + 3000, 0),
|
||||
# A server's repeat on connection 1: its first copy counts.
|
||||
(1, "c1", S + 50_000, 0),
|
||||
(0, "c2", S + 10_000_000, 0),
|
||||
(1, "c2", S + 10_002_000, 0),
|
||||
(1, "c3", S + MIN, 0),
|
||||
(0, "c3", S + MIN + 1000, 0),
|
||||
(2, "c3", S + MIN + 4000, 0),
|
||||
(1, "c4", S + MIN + 10_000_000, 0),
|
||||
(0, "c4", S + MIN + 10_003_000, 0),
|
||||
(2, "c4", S + MIN + 10_005_000, 0),
|
||||
(0, "c5", S + 2 * MIN + 5_000_000, 0),
|
||||
(1, "c5", S + 2 * MIN + 5_000_000, 0),
|
||||
(2, "c5", S + 2 * MIN + 5_000_000, 0),
|
||||
],
|
||||
notes=["offset,start,400,3000", "offset,end,600,3000"],
|
||||
)
|
||||
|
||||
|
||||
class Analyse(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.run = analyse.Run(RUN)
|
||||
self.minutes = self.run.minutes()
|
||||
|
||||
def test_full_minutes_only(self):
|
||||
self.assertEqual([len(minute) for minute in self.minutes], [2, 2])
|
||||
|
||||
def test_the_first_copy_is_not_late_and_a_repeat_is_ignored(self):
|
||||
lateness, _ = self.run.lateness(self.minutes[0], [0])
|
||||
self.assertEqual(lateness, [0, 0])
|
||||
lateness, _ = self.run.lateness(self.minutes[0], [1])
|
||||
self.assertEqual(lateness, [1000, 2000])
|
||||
|
||||
def test_a_change_a_connection_lacks_is_missed_by_it(self):
|
||||
changes = self.minutes[0] + self.minutes[1]
|
||||
self.assertEqual([self.run.lateness(changes, [c])[1] for c in (0, 1, 2)], [0, 0, 1])
|
||||
|
||||
def test_speeds_and_their_persistence(self):
|
||||
self.assertEqual(analyse.speeds(self.run, self.minutes[0]), {0: 0, 1: 1500, 2: 3000})
|
||||
self.assertEqual(analyse.speeds(self.run, self.minutes[1]), {0: 2000, 1: 0, 2: 4500})
|
||||
# Ranks (1, 2, 3) then (2, 1, 3).
|
||||
self.assertEqual(analyse.persistence(self.run, self.minutes), [0.5])
|
||||
|
||||
def test_the_fastest_of_minute_one_against_the_others(self):
|
||||
top, rest = analyse.top_against_rest(self.run, self.minutes, top=1)
|
||||
self.assertEqual(top, (2000, 3000))
|
||||
# [0, 0, 4000, 5000]: the median between, p99 the last.
|
||||
self.assertEqual(rest, (2000, 5000))
|
||||
|
||||
def test_racing(self):
|
||||
# Kept: connections 0 and 1, the sooner copy 0 for c3 and c4. At
|
||||
# random: (0, 1) [0, 0], (0, 2) [1000, 3000] (median 2000, p99 3000),
|
||||
# (1, 2) [0, 0].
|
||||
self.assertEqual(
|
||||
analyse.racing(self.run, self.minutes, pair=2), ((0, 0), 0, (2000 / 3, 1000), 0)
|
||||
)
|
||||
|
||||
def test_by_address(self):
|
||||
changes = self.minutes[0] + self.minutes[1]
|
||||
self.assertEqual(
|
||||
analyse.by_address(self.run, changes),
|
||||
# A: [0, 0, 1000, 3000] and [3000, 4000, 5000]; B: [1000, 2000, 0, 0].
|
||||
{"A": (3000, 5000), "B": (500, 2000)},
|
||||
)
|
||||
|
||||
def test_lag_raw_and_corrected_by_the_mean_offset(self):
|
||||
raw, corrected, offset = analyse.lag(self.run, self.minutes[0][:1])
|
||||
self.assertEqual((raw, corrected, offset), ((10_000, 10_000), (10_500, 10_500), 500))
|
||||
|
||||
def test_speed_is_the_median_and_racing_keeps_the_fastest(self):
|
||||
# Minute 1: lateness 0, 0, 0 on connection 0; 0, 0, 3000 on 1 (its
|
||||
# median 0, its mean 1000); 2000 each on 2. Order 0, 1, 2, so the
|
||||
# pair kept is (0, 1). Minute 2: connection 1 is now the slowest.
|
||||
copies = []
|
||||
for n, late in enumerate([0, 0, 3000]):
|
||||
at = S + n * 1_000_000
|
||||
copies += [(0, f"a{n}", at, 0), (1, f"a{n}", at + late, 0), (2, f"a{n}", at + 2000, 0)]
|
||||
copies += [
|
||||
(0, "b", S + MIN, 0),
|
||||
(1, "b", S + MIN + 9000, 0),
|
||||
(2, "b", S + MIN + 1000, 0),
|
||||
(0, "end", S + 2 * MIN + 1, 0),
|
||||
]
|
||||
run = analyse.Run(csv(copies))
|
||||
minutes = run.minutes()
|
||||
self.assertEqual(analyse.speeds(run, minutes[0]), {0: 0, 1: 0, 2: 2000})
|
||||
# Kept (0, 1): 0. At random: (0, 1) 0, (0, 2) 0, (1, 2) 1000.
|
||||
picked, _, random, _ = analyse.racing(run, minutes, pair=2)
|
||||
self.assertEqual((picked, random), ((0, 0), (1000 / 3, 1000 / 3)))
|
||||
|
||||
def test_p99_is_not_the_maximum(self):
|
||||
self.assertEqual(analyse.quantile(range(200), 0.99), 198)
|
||||
|
||||
def test_the_report_of_the_hand_worked_run(self):
|
||||
report = analyse.report(self.run)
|
||||
# Connection 2's speeds, and the change it missed.
|
||||
self.assertIn("| 2 | A | 3.00 4.50 | 1 |", report)
|
||||
self.assertIn("rank correlation minute to minute 0.50, mean 0.50", report)
|
||||
# Three connections: no 4 fastest against the others.
|
||||
self.assertIn("The 4 fastest of minute 1 against the others: too few connections", report)
|
||||
# Kept pair p99 0, pairs at random 1000 on average: 100% lower.
|
||||
self.assertIn("a pair at random, 0.67 ms, 1.00 ms, 0.0 missed; p99 100% lower", report)
|
||||
|
||||
def test_racing_counts_what_a_pair_missed(self):
|
||||
# Minute 1 ranks 0, 1, 2; after it, one change only connection 2
|
||||
# delivers: missed by the kept pair (0, 1), by one pair in three.
|
||||
run = analyse.Run(
|
||||
csv(
|
||||
[
|
||||
(0, "a", S, 0),
|
||||
(1, "a", S + 1000, 0),
|
||||
(2, "a", S + 2000, 0),
|
||||
(2, "b", S + MIN, 0),
|
||||
(0, "c", S + MIN + 1000, 0),
|
||||
(1, "c", S + MIN + 1000, 0),
|
||||
(2, "c", S + MIN + 1000, 0),
|
||||
(0, "end", S + 2 * MIN + 1, 0),
|
||||
]
|
||||
)
|
||||
)
|
||||
_, picked_missed, _, random_missed = analyse.racing(run, run.minutes(), pair=2)
|
||||
self.assertEqual((picked_missed, random_missed), (1, 1 / 3))
|
||||
|
||||
def test_an_offset_the_probe_could_not_measure_is_left_out(self):
|
||||
run = analyse.Run(csv([], notes=["offset,start,,", "offset,end,600,3000"]))
|
||||
self.assertEqual(run.offsets, [600])
|
||||
|
||||
def test_thin_runs_are_reported_without_a_traceback(self):
|
||||
# One connection over two minutes: nothing to compare.
|
||||
alone = [(0, "a", S, 0), (0, "b", S + MIN, 0), (0, "end", S + 2 * MIN + 1, 0)]
|
||||
report = analyse.report(analyse.Run(csv(alone)))
|
||||
self.assertIn("Racing: too few connections deliver after minute 1", report)
|
||||
# The kept pair, and every other, silent after minute 1 but one.
|
||||
silent = [
|
||||
(0, "a", S, 0),
|
||||
(1, "a", S + 1000, 0),
|
||||
(2, "a", S + 2000, 0),
|
||||
(2, "end", S + 2 * MIN + 1, 0),
|
||||
]
|
||||
analyse.report(analyse.Run(csv(silent)))
|
||||
|
||||
def test_a_run_with_no_change_is_reported_as_such(self):
|
||||
self.assertEqual(
|
||||
analyse.report(analyse.Run(csv([]))), "0 connections, no change over a full minute"
|
||||
)
|
||||
|
||||
def test_a_connection_is_missing_only_while_up(self):
|
||||
run = analyse.Run(
|
||||
csv(
|
||||
[
|
||||
# c1 before connection 2's first copy, c3 after it went
|
||||
# down: neither is missed by it.
|
||||
(0, "c1", S, 0),
|
||||
(2, "c2", S + 1000, 0),
|
||||
(0, "c2", S + 1000, 0),
|
||||
(0, "c3", S + 3000, 0),
|
||||
(0, "c4", S + MIN + 1, 0),
|
||||
],
|
||||
notes=[f"down,2,{S + 2000}"],
|
||||
)
|
||||
)
|
||||
self.assertEqual(run.lateness(run.minutes()[0], [2])[1], 0)
|
||||
# Without the note, c3 is missed.
|
||||
run.down.clear()
|
||||
self.assertEqual(run.lateness(run.minutes()[0], [2])[1], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user