feat(latency): latency experiments, LATENCY.md, Kalshi's slow-connection recycle

Extension 4. `engineering-challenge latency EXCHANGE OUT.csv` opens 16
connections to the same busy markets and writes every copy of every change;
`scripts/latency/analyse.py` gives each connection's speed, whether it
persists, what keeping the fastest would gain, lateness by address, and lag
with the clock's offset. LATENCY.md writes up four runs: on Polymarket speed
persists but is worth under a millisecond, at the median only; on Kalshi's
demo keeping the fastest pair lowers p99 by 28% and 42%, the threshold
agreed before the runs.

So the collector now recycles a Kalshi connection whose mean lateness stays,
3 minutes running, in the slowest quarter and above twice everyone's, ahead
of its hourly turn. Kalshi's rule measures each duplicate's lateness; the
orchestrator judges once a minute. A duplicate's lag in Polymarket's minute
line is now its "lateness".

Roadmap milestone 7, roadmap/07.
This commit is contained in:
pascal committed 2026-11-12 15:05:25 +01:00
1 parent 1a5bbcd602
commit 515e85941d
22 files changed
+2132 -55

No files matched your search

+258
View File
@@ -0,0 +1,258 @@
"""The latency probe's CSV, analysed (roadmap/07): lateness, missed changes,
each connection's speed per minute, whether speed persists, what racing
would gain, lateness by address, and lag. Standard library only.
Usage: python3 scripts/latency/analyse.py RUN.csv
Terms (GLOSSARY.md): a copy's lateness is how long after the change's first
copy it came; a connection's speed is its median lateness over a minute;
lag is a receive time minus the exchange's timestamp.
"""
import csv
import itertools
import statistics
import sys
from pathlib import Path
MINUTE_US = 60_000_000
# The fastest of the first minute, against the others (persistence).
TOP = 4
# Connections a symbol has in the collector: first one wins (ADR 0003).
PAIR = 2
class Run:
"""A run's changes, each with every connection's first copy of it."""
def __init__(self, lines):
self.connections = {} # connection -> address
self.down = {} # connection -> when it went down, µs
self.offsets = [] # the clock's offsets, µs
# (symbol, identity) -> [exchange_ms, {connection: received_us}]
self.changes = {}
rows = []
for line in lines:
if line.startswith("#"):
self._note(line[1:].strip())
else:
rows.append(line)
for row in csv.DictReader(rows):
connection = int(row["connection"])
self.connections[connection] = row["address"]
key = (row["symbol"], row["identity"])
change = self.changes.setdefault(key, [int(row["exchange_ms"]), {}])
# A server's repeat on one connection: its first copy counts.
change[1].setdefault(connection, int(row["received_us"]))
self.up_from = {} # connection -> its first copy, µs
for _, copies in self.changes.values():
for connection, received in copies.items():
self.up_from[connection] = min(self.up_from.get(connection, received), received)
def _note(self, note):
fields = note.split(",")
if fields[0] == "down":
self.down[int(fields[1])] = int(fields[2])
elif fields[0] == "offset" and fields[2]:
self.offsets.append(int(fields[2]))
def first(self, change):
return min(change[1].values())
def minutes(self):
"""Each full minute's changes, from the run's first change."""
firsts = [(self.first(change), change) for change in self.changes.values()]
if not firsts:
return []
start = min(first for first, _ in firsts)
last = max(first for first, _ in firsts)
count = (last - start) // MINUTE_US # the last, partial, left out
minutes = [[] for _ in range(count)]
for first, change in firsts:
minute = (first - start) // MINUTE_US
if minute < count:
minutes[minute].append(change)
return minutes
def up(self, connection, at):
"""From its first copy (its subscribe done) until it went down."""
never = float("inf")
return self.up_from.get(connection, never) <= at < self.down.get(connection, never)
def lateness(self, changes, connections):
"""The lateness of the first copy among `connections`, change by
change, behind the first of all; and how many changes none of them
delivered while up."""
lateness, missed = [], 0
for change in changes:
first = self.first(change)
copies = [change[1][c] for c in connections if c in change[1]]
if copies:
lateness.append(min(copies) - first)
elif any(self.up(c, first) for c in connections):
missed += 1
return lateness, missed
def quantile(values, q):
"""As the collector's log computes it: the value at index n * q."""
ordered = sorted(values)
return ordered[min(int(len(ordered) * q), len(ordered) - 1)]
def speeds(run, minute):
"""Each connection's speed over a minute's changes, µs."""
result = {}
for connection in run.connections:
lateness, _ = run.lateness(minute, [connection])
if lateness:
result[connection] = statistics.median(lateness)
return result
def ranked(speeds_by_connection):
return sorted(speeds_by_connection, key=lambda c: (speeds_by_connection[c], c))
def persistence(run, minutes):
"""The rank correlation of the connections' speeds between each minute
and the next."""
per_minute = [speeds(run, minute) for minute in minutes]
correlations = []
for this, following in itertools.pairwise(per_minute):
both = sorted(set(this) & set(following))
if len(both) > 2:
correlations.append(
statistics.correlation(
[this[c] for c in both],
[following[c] for c in both],
method="ranked",
)
)
return correlations
def summary(lateness):
if not lateness:
return None
return statistics.median(lateness), quantile(lateness, 0.99)
def top_against_rest(run, minutes, top=TOP):
"""The `top` fastest of the first minute against the others, over the
minutes after it: (median, p99) of their copies' lateness each."""
order = ranked(speeds(run, minutes[0]))
top, rest = order[:top], order[top:]
later = [change for minute in minutes[1:] for change in minute]
def copies(connections):
lateness = []
for connection in connections:
lateness += run.lateness(later, [connection])[0]
return summary(lateness)
return copies(top), copies(rest)
def racing(run, minutes, pair=PAIR):
"""The `pair` fastest of the first minute kept, against a pair at
random (every pair, averaged), over the minutes after it: the (median,
p99) of the pair's lateness (first one wins: the sooner of its two
copies), and its missed changes. None if too few connections deliver
after the first minute to compare."""
order = ranked(speeds(run, minutes[0]))
later = [change for minute in minutes[1:] for change in minute]
picked_lateness, picked_missed = run.lateness(later, order[:pair])
pairs = [run.lateness(later, list(two)) for two in itertools.combinations(order, pair)]
random = [s for s in (summary(lateness) for lateness, _ in pairs) if s]
picked = summary(picked_lateness)
if len(order) < pair or picked is None or not random:
return None
return (
picked,
picked_missed,
(statistics.mean(m for m, _ in random), statistics.mean(p for _, p in random)),
statistics.mean(missed for _, missed in pairs),
)
def by_address(run, changes):
"""(median, p99) of lateness by address, every copy counted."""
addresses = {}
for connection, address in run.connections.items():
addresses.setdefault(address, []).extend(run.lateness(changes, [connection])[0])
return {address: summary(lateness) for address, lateness in sorted(addresses.items())}
def lag(run, changes):
"""(median, p99) of the first copies' lag, µs, raw and corrected by the
clock's mean offset (positive: the local clock is behind)."""
raw = [run.first(change) - change[0] * 1000 for change in changes]
offset = statistics.mean(run.offsets) if run.offsets else 0
return summary(raw), summary([value + offset for value in raw]), offset
def ms(micros):
return f"{micros / 1000:.2f}"
def report(run):
minutes = run.minutes()
changes = [change for minute in minutes for change in minute]
if not changes:
return f"{len(run.connections)} connections, no change over a full minute"
lines = [
f"{len(run.connections)} connections, {len(changes)} changes "
f"over {len(minutes)} full minutes",
"",
"| Connection | Address | Speed per minute (ms) | Missed |",
"|---|---|---|---|",
]
per_minute = [speeds(run, minute) for minute in minutes]
for connection, address in sorted(run.connections.items()):
row = " ".join(ms(s[connection]) if connection in s else "-" for s in per_minute)
_, missed = run.lateness(changes, [connection])
lines.append(f"| {connection} | {address} | {row} | {missed} |")
if len(minutes) >= 2:
correlations = persistence(run, minutes)
if correlations:
each = " ".join(f"{c:.2f}" for c in correlations)
mean = statistics.mean(correlations)
lines += ["", f"Persistence: rank correlation minute to minute {each}, mean {mean:.2f}"]
top, rest = top_against_rest(run, minutes)
if top and rest:
lines.append(
f"The {TOP} fastest of minute 1 after it: {ms(top[0])} ms median, "
f"{ms(top[1])} ms p99; the others: {ms(rest[0])} ms, {ms(rest[1])} ms"
)
else:
lines.append(f"The {TOP} fastest of minute 1 against the others: too few connections")
raced = racing(run, minutes)
if raced:
picked, picked_missed, random, random_missed = raced
gain = (random[1] - picked[1]) / random[1] if random[1] else 0
lines.append(
f"Racing: the {PAIR} fastest of minute 1, {ms(picked[0])} ms median, "
f"{ms(picked[1])} ms p99, {picked_missed} missed; a pair at random, "
f"{ms(random[0])} ms, {ms(random[1])} ms, {random_missed:.1f} missed; "
f"p99 {gain:.0%} lower"
)
else:
lines.append("Racing: too few connections deliver after minute 1")
lines += ["", "| Address | Median (ms) | p99 (ms) |", "|---|---|---|"]
for address, lateness in by_address(run, changes).items():
cells = f"{ms(lateness[0])} | {ms(lateness[1])}" if lateness else "- | -"
lines.append(f"| {address} | {cells} |")
raw, corrected, offset = lag(run, changes)
lines += [
"",
f"Lag of the first copies: raw {ms(raw[0])} ms median, {ms(raw[1])} ms p99; "
f"corrected by the clock's offset ({ms(offset)} ms): "
f"{ms(corrected[0])} ms, {ms(corrected[1])} ms",
]
return "\n".join(lines)
if __name__ == "__main__":
with Path(sys.argv[1]).open() as file:
print(report(Run(file)))