Archived
feat(latency): latency experiments, LATENCY.md, Kalshi's slow-connection recycle
Extension 4. `engineering-challenge latency EXCHANGE OUT.csv` opens 16 connections to the same busy markets and writes every copy of every change; `scripts/latency/analyse.py` gives each connection's speed, whether it persists, what keeping the fastest would gain, lateness by address, and lag with the clock's offset. LATENCY.md writes up four runs: on Polymarket speed persists but is worth under a millisecond, at the median only; on Kalshi's demo keeping the fastest pair lowers p99 by 28% and 42%, the threshold agreed before the runs. So the collector now recycles a Kalshi connection whose mean lateness stays, 3 minutes running, in the slowest quarter and above twice everyone's, ahead of its hourly turn. Kalshi's rule measures each duplicate's lateness; the orchestrator judges once a minute. A duplicate's lag in Polymarket's minute line is now its "lateness". Roadmap milestone 7, roadmap/07.
This commit is contained in:
22 files changed
+2132
-55
No files matched your search
@@ -0,0 +1,258 @@
|
||||
"""The latency probe's CSV, analysed (roadmap/07): lateness, missed changes,
|
||||
each connection's speed per minute, whether speed persists, what racing
|
||||
would gain, lateness by address, and lag. Standard library only.
|
||||
|
||||
Usage: python3 scripts/latency/analyse.py RUN.csv
|
||||
|
||||
Terms (GLOSSARY.md): a copy's lateness is how long after the change's first
|
||||
copy it came; a connection's speed is its median lateness over a minute;
|
||||
lag is a receive time minus the exchange's timestamp.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import itertools
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MINUTE_US = 60_000_000
|
||||
# The fastest of the first minute, against the others (persistence).
|
||||
TOP = 4
|
||||
# Connections a symbol has in the collector: first one wins (ADR 0003).
|
||||
PAIR = 2
|
||||
|
||||
|
||||
class Run:
|
||||
"""A run's changes, each with every connection's first copy of it."""
|
||||
|
||||
def __init__(self, lines):
|
||||
self.connections = {} # connection -> address
|
||||
self.down = {} # connection -> when it went down, µs
|
||||
self.offsets = [] # the clock's offsets, µs
|
||||
# (symbol, identity) -> [exchange_ms, {connection: received_us}]
|
||||
self.changes = {}
|
||||
rows = []
|
||||
for line in lines:
|
||||
if line.startswith("#"):
|
||||
self._note(line[1:].strip())
|
||||
else:
|
||||
rows.append(line)
|
||||
for row in csv.DictReader(rows):
|
||||
connection = int(row["connection"])
|
||||
self.connections[connection] = row["address"]
|
||||
key = (row["symbol"], row["identity"])
|
||||
change = self.changes.setdefault(key, [int(row["exchange_ms"]), {}])
|
||||
# A server's repeat on one connection: its first copy counts.
|
||||
change[1].setdefault(connection, int(row["received_us"]))
|
||||
self.up_from = {} # connection -> its first copy, µs
|
||||
for _, copies in self.changes.values():
|
||||
for connection, received in copies.items():
|
||||
self.up_from[connection] = min(self.up_from.get(connection, received), received)
|
||||
|
||||
def _note(self, note):
|
||||
fields = note.split(",")
|
||||
if fields[0] == "down":
|
||||
self.down[int(fields[1])] = int(fields[2])
|
||||
elif fields[0] == "offset" and fields[2]:
|
||||
self.offsets.append(int(fields[2]))
|
||||
|
||||
def first(self, change):
|
||||
return min(change[1].values())
|
||||
|
||||
def minutes(self):
|
||||
"""Each full minute's changes, from the run's first change."""
|
||||
firsts = [(self.first(change), change) for change in self.changes.values()]
|
||||
if not firsts:
|
||||
return []
|
||||
start = min(first for first, _ in firsts)
|
||||
last = max(first for first, _ in firsts)
|
||||
count = (last - start) // MINUTE_US # the last, partial, left out
|
||||
minutes = [[] for _ in range(count)]
|
||||
for first, change in firsts:
|
||||
minute = (first - start) // MINUTE_US
|
||||
if minute < count:
|
||||
minutes[minute].append(change)
|
||||
return minutes
|
||||
|
||||
def up(self, connection, at):
|
||||
"""From its first copy (its subscribe done) until it went down."""
|
||||
never = float("inf")
|
||||
return self.up_from.get(connection, never) <= at < self.down.get(connection, never)
|
||||
|
||||
def lateness(self, changes, connections):
|
||||
"""The lateness of the first copy among `connections`, change by
|
||||
change, behind the first of all; and how many changes none of them
|
||||
delivered while up."""
|
||||
lateness, missed = [], 0
|
||||
for change in changes:
|
||||
first = self.first(change)
|
||||
copies = [change[1][c] for c in connections if c in change[1]]
|
||||
if copies:
|
||||
lateness.append(min(copies) - first)
|
||||
elif any(self.up(c, first) for c in connections):
|
||||
missed += 1
|
||||
return lateness, missed
|
||||
|
||||
|
||||
def quantile(values, q):
|
||||
"""As the collector's log computes it: the value at index n * q."""
|
||||
ordered = sorted(values)
|
||||
return ordered[min(int(len(ordered) * q), len(ordered) - 1)]
|
||||
|
||||
|
||||
def speeds(run, minute):
|
||||
"""Each connection's speed over a minute's changes, µs."""
|
||||
result = {}
|
||||
for connection in run.connections:
|
||||
lateness, _ = run.lateness(minute, [connection])
|
||||
if lateness:
|
||||
result[connection] = statistics.median(lateness)
|
||||
return result
|
||||
|
||||
|
||||
def ranked(speeds_by_connection):
|
||||
return sorted(speeds_by_connection, key=lambda c: (speeds_by_connection[c], c))
|
||||
|
||||
|
||||
def persistence(run, minutes):
|
||||
"""The rank correlation of the connections' speeds between each minute
|
||||
and the next."""
|
||||
per_minute = [speeds(run, minute) for minute in minutes]
|
||||
correlations = []
|
||||
for this, following in itertools.pairwise(per_minute):
|
||||
both = sorted(set(this) & set(following))
|
||||
if len(both) > 2:
|
||||
correlations.append(
|
||||
statistics.correlation(
|
||||
[this[c] for c in both],
|
||||
[following[c] for c in both],
|
||||
method="ranked",
|
||||
)
|
||||
)
|
||||
return correlations
|
||||
|
||||
|
||||
def summary(lateness):
|
||||
if not lateness:
|
||||
return None
|
||||
return statistics.median(lateness), quantile(lateness, 0.99)
|
||||
|
||||
|
||||
def top_against_rest(run, minutes, top=TOP):
|
||||
"""The `top` fastest of the first minute against the others, over the
|
||||
minutes after it: (median, p99) of their copies' lateness each."""
|
||||
order = ranked(speeds(run, minutes[0]))
|
||||
top, rest = order[:top], order[top:]
|
||||
later = [change for minute in minutes[1:] for change in minute]
|
||||
|
||||
def copies(connections):
|
||||
lateness = []
|
||||
for connection in connections:
|
||||
lateness += run.lateness(later, [connection])[0]
|
||||
return summary(lateness)
|
||||
|
||||
return copies(top), copies(rest)
|
||||
|
||||
|
||||
def racing(run, minutes, pair=PAIR):
|
||||
"""The `pair` fastest of the first minute kept, against a pair at
|
||||
random (every pair, averaged), over the minutes after it: the (median,
|
||||
p99) of the pair's lateness (first one wins: the sooner of its two
|
||||
copies), and its missed changes. None if too few connections deliver
|
||||
after the first minute to compare."""
|
||||
order = ranked(speeds(run, minutes[0]))
|
||||
later = [change for minute in minutes[1:] for change in minute]
|
||||
picked_lateness, picked_missed = run.lateness(later, order[:pair])
|
||||
pairs = [run.lateness(later, list(two)) for two in itertools.combinations(order, pair)]
|
||||
random = [s for s in (summary(lateness) for lateness, _ in pairs) if s]
|
||||
picked = summary(picked_lateness)
|
||||
if len(order) < pair or picked is None or not random:
|
||||
return None
|
||||
return (
|
||||
picked,
|
||||
picked_missed,
|
||||
(statistics.mean(m for m, _ in random), statistics.mean(p for _, p in random)),
|
||||
statistics.mean(missed for _, missed in pairs),
|
||||
)
|
||||
|
||||
|
||||
def by_address(run, changes):
|
||||
"""(median, p99) of lateness by address, every copy counted."""
|
||||
addresses = {}
|
||||
for connection, address in run.connections.items():
|
||||
addresses.setdefault(address, []).extend(run.lateness(changes, [connection])[0])
|
||||
return {address: summary(lateness) for address, lateness in sorted(addresses.items())}
|
||||
|
||||
|
||||
def lag(run, changes):
|
||||
"""(median, p99) of the first copies' lag, µs, raw and corrected by the
|
||||
clock's mean offset (positive: the local clock is behind)."""
|
||||
raw = [run.first(change) - change[0] * 1000 for change in changes]
|
||||
offset = statistics.mean(run.offsets) if run.offsets else 0
|
||||
return summary(raw), summary([value + offset for value in raw]), offset
|
||||
|
||||
|
||||
def ms(micros):
|
||||
return f"{micros / 1000:.2f}"
|
||||
|
||||
|
||||
def report(run):
|
||||
minutes = run.minutes()
|
||||
changes = [change for minute in minutes for change in minute]
|
||||
if not changes:
|
||||
return f"{len(run.connections)} connections, no change over a full minute"
|
||||
lines = [
|
||||
f"{len(run.connections)} connections, {len(changes)} changes "
|
||||
f"over {len(minutes)} full minutes",
|
||||
"",
|
||||
"| Connection | Address | Speed per minute (ms) | Missed |",
|
||||
"|---|---|---|---|",
|
||||
]
|
||||
per_minute = [speeds(run, minute) for minute in minutes]
|
||||
for connection, address in sorted(run.connections.items()):
|
||||
row = " ".join(ms(s[connection]) if connection in s else "-" for s in per_minute)
|
||||
_, missed = run.lateness(changes, [connection])
|
||||
lines.append(f"| {connection} | {address} | {row} | {missed} |")
|
||||
if len(minutes) >= 2:
|
||||
correlations = persistence(run, minutes)
|
||||
if correlations:
|
||||
each = " ".join(f"{c:.2f}" for c in correlations)
|
||||
mean = statistics.mean(correlations)
|
||||
lines += ["", f"Persistence: rank correlation minute to minute {each}, mean {mean:.2f}"]
|
||||
top, rest = top_against_rest(run, minutes)
|
||||
if top and rest:
|
||||
lines.append(
|
||||
f"The {TOP} fastest of minute 1 after it: {ms(top[0])} ms median, "
|
||||
f"{ms(top[1])} ms p99; the others: {ms(rest[0])} ms, {ms(rest[1])} ms"
|
||||
)
|
||||
else:
|
||||
lines.append(f"The {TOP} fastest of minute 1 against the others: too few connections")
|
||||
raced = racing(run, minutes)
|
||||
if raced:
|
||||
picked, picked_missed, random, random_missed = raced
|
||||
gain = (random[1] - picked[1]) / random[1] if random[1] else 0
|
||||
lines.append(
|
||||
f"Racing: the {PAIR} fastest of minute 1, {ms(picked[0])} ms median, "
|
||||
f"{ms(picked[1])} ms p99, {picked_missed} missed; a pair at random, "
|
||||
f"{ms(random[0])} ms, {ms(random[1])} ms, {random_missed:.1f} missed; "
|
||||
f"p99 {gain:.0%} lower"
|
||||
)
|
||||
else:
|
||||
lines.append("Racing: too few connections deliver after minute 1")
|
||||
lines += ["", "| Address | Median (ms) | p99 (ms) |", "|---|---|---|"]
|
||||
for address, lateness in by_address(run, changes).items():
|
||||
cells = f"{ms(lateness[0])} | {ms(lateness[1])}" if lateness else "- | -"
|
||||
lines.append(f"| {address} | {cells} |")
|
||||
raw, corrected, offset = lag(run, changes)
|
||||
lines += [
|
||||
"",
|
||||
f"Lag of the first copies: raw {ms(raw[0])} ms median, {ms(raw[1])} ms p99; "
|
||||
f"corrected by the clock's offset ({ms(offset)} ms): "
|
||||
f"{ms(corrected[0])} ms, {ms(corrected[1])} ms",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with Path(sys.argv[1]).open() as file:
|
||||
print(report(Run(file)))
|
||||
Reference in new issue
Block a user