Archived
Extension 4. `engineering-challenge latency EXCHANGE OUT.csv` opens 16 connections to the same busy markets and writes every copy of every change; `scripts/latency/analyse.py` gives each connection's speed, whether it persists, what keeping the fastest would gain, lateness by address, and lag with the clock's offset. LATENCY.md writes up four runs: on Polymarket speed persists but is worth under a millisecond, at the median only; on Kalshi's demo keeping the fastest pair lowers p99 by 28% and 42%, the threshold agreed before the runs. So the collector now recycles a Kalshi connection whose mean lateness stays, 3 minutes running, in the slowest quarter and above twice everyone's, ahead of its hourly turn. Kalshi's rule measures each duplicate's lateness; the orchestrator judges once a minute. A duplicate's lag in Polymarket's minute line is now its "lateness". Roadmap milestone 7, roadmap/07.
259 lines
10 KiB
Python
259 lines
10 KiB
Python
"""The latency probe's CSV, analysed (roadmap/07): lateness, missed changes,
|
|
each connection's speed per minute, whether speed persists, what racing
|
|
would gain, lateness by address, and lag. Standard library only.
|
|
|
|
Usage: python3 scripts/latency/analyse.py RUN.csv
|
|
|
|
Terms (GLOSSARY.md): a copy's lateness is how long after the change's first
|
|
copy it came; a connection's speed is its median lateness over a minute;
|
|
lag is a receive time minus the exchange's timestamp.
|
|
"""
|
|
|
|
import csv
|
|
import itertools
|
|
import statistics
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
MINUTE_US = 60_000_000
|
|
# The fastest of the first minute, against the others (persistence).
|
|
TOP = 4
|
|
# Connections a symbol has in the collector: first one wins (ADR 0003).
|
|
PAIR = 2
|
|
|
|
|
|
class Run:
|
|
"""A run's changes, each with every connection's first copy of it."""
|
|
|
|
def __init__(self, lines):
|
|
self.connections = {} # connection -> address
|
|
self.down = {} # connection -> when it went down, µs
|
|
self.offsets = [] # the clock's offsets, µs
|
|
# (symbol, identity) -> [exchange_ms, {connection: received_us}]
|
|
self.changes = {}
|
|
rows = []
|
|
for line in lines:
|
|
if line.startswith("#"):
|
|
self._note(line[1:].strip())
|
|
else:
|
|
rows.append(line)
|
|
for row in csv.DictReader(rows):
|
|
connection = int(row["connection"])
|
|
self.connections[connection] = row["address"]
|
|
key = (row["symbol"], row["identity"])
|
|
change = self.changes.setdefault(key, [int(row["exchange_ms"]), {}])
|
|
# A server's repeat on one connection: its first copy counts.
|
|
change[1].setdefault(connection, int(row["received_us"]))
|
|
self.up_from = {} # connection -> its first copy, µs
|
|
for _, copies in self.changes.values():
|
|
for connection, received in copies.items():
|
|
self.up_from[connection] = min(self.up_from.get(connection, received), received)
|
|
|
|
def _note(self, note):
|
|
fields = note.split(",")
|
|
if fields[0] == "down":
|
|
self.down[int(fields[1])] = int(fields[2])
|
|
elif fields[0] == "offset" and fields[2]:
|
|
self.offsets.append(int(fields[2]))
|
|
|
|
def first(self, change):
|
|
return min(change[1].values())
|
|
|
|
def minutes(self):
|
|
"""Each full minute's changes, from the run's first change."""
|
|
firsts = [(self.first(change), change) for change in self.changes.values()]
|
|
if not firsts:
|
|
return []
|
|
start = min(first for first, _ in firsts)
|
|
last = max(first for first, _ in firsts)
|
|
count = (last - start) // MINUTE_US # the last, partial, left out
|
|
minutes = [[] for _ in range(count)]
|
|
for first, change in firsts:
|
|
minute = (first - start) // MINUTE_US
|
|
if minute < count:
|
|
minutes[minute].append(change)
|
|
return minutes
|
|
|
|
def up(self, connection, at):
|
|
"""From its first copy (its subscribe done) until it went down."""
|
|
never = float("inf")
|
|
return self.up_from.get(connection, never) <= at < self.down.get(connection, never)
|
|
|
|
def lateness(self, changes, connections):
|
|
"""The lateness of the first copy among `connections`, change by
|
|
change, behind the first of all; and how many changes none of them
|
|
delivered while up."""
|
|
lateness, missed = [], 0
|
|
for change in changes:
|
|
first = self.first(change)
|
|
copies = [change[1][c] for c in connections if c in change[1]]
|
|
if copies:
|
|
lateness.append(min(copies) - first)
|
|
elif any(self.up(c, first) for c in connections):
|
|
missed += 1
|
|
return lateness, missed
|
|
|
|
|
|
def quantile(values, q):
|
|
"""As the collector's log computes it: the value at index n * q."""
|
|
ordered = sorted(values)
|
|
return ordered[min(int(len(ordered) * q), len(ordered) - 1)]
|
|
|
|
|
|
def speeds(run, minute):
|
|
"""Each connection's speed over a minute's changes, µs."""
|
|
result = {}
|
|
for connection in run.connections:
|
|
lateness, _ = run.lateness(minute, [connection])
|
|
if lateness:
|
|
result[connection] = statistics.median(lateness)
|
|
return result
|
|
|
|
|
|
def ranked(speeds_by_connection):
|
|
return sorted(speeds_by_connection, key=lambda c: (speeds_by_connection[c], c))
|
|
|
|
|
|
def persistence(run, minutes):
|
|
"""The rank correlation of the connections' speeds between each minute
|
|
and the next."""
|
|
per_minute = [speeds(run, minute) for minute in minutes]
|
|
correlations = []
|
|
for this, following in itertools.pairwise(per_minute):
|
|
both = sorted(set(this) & set(following))
|
|
if len(both) > 2:
|
|
correlations.append(
|
|
statistics.correlation(
|
|
[this[c] for c in both],
|
|
[following[c] for c in both],
|
|
method="ranked",
|
|
)
|
|
)
|
|
return correlations
|
|
|
|
|
|
def summary(lateness):
|
|
if not lateness:
|
|
return None
|
|
return statistics.median(lateness), quantile(lateness, 0.99)
|
|
|
|
|
|
def top_against_rest(run, minutes, top=TOP):
|
|
"""The `top` fastest of the first minute against the others, over the
|
|
minutes after it: (median, p99) of their copies' lateness each."""
|
|
order = ranked(speeds(run, minutes[0]))
|
|
top, rest = order[:top], order[top:]
|
|
later = [change for minute in minutes[1:] for change in minute]
|
|
|
|
def copies(connections):
|
|
lateness = []
|
|
for connection in connections:
|
|
lateness += run.lateness(later, [connection])[0]
|
|
return summary(lateness)
|
|
|
|
return copies(top), copies(rest)
|
|
|
|
|
|
def racing(run, minutes, pair=PAIR):
|
|
"""The `pair` fastest of the first minute kept, against a pair at
|
|
random (every pair, averaged), over the minutes after it: the (median,
|
|
p99) of the pair's lateness (first one wins: the sooner of its two
|
|
copies), and its missed changes. None if too few connections deliver
|
|
after the first minute to compare."""
|
|
order = ranked(speeds(run, minutes[0]))
|
|
later = [change for minute in minutes[1:] for change in minute]
|
|
picked_lateness, picked_missed = run.lateness(later, order[:pair])
|
|
pairs = [run.lateness(later, list(two)) for two in itertools.combinations(order, pair)]
|
|
random = [s for s in (summary(lateness) for lateness, _ in pairs) if s]
|
|
picked = summary(picked_lateness)
|
|
if len(order) < pair or picked is None or not random:
|
|
return None
|
|
return (
|
|
picked,
|
|
picked_missed,
|
|
(statistics.mean(m for m, _ in random), statistics.mean(p for _, p in random)),
|
|
statistics.mean(missed for _, missed in pairs),
|
|
)
|
|
|
|
|
|
def by_address(run, changes):
|
|
"""(median, p99) of lateness by address, every copy counted."""
|
|
addresses = {}
|
|
for connection, address in run.connections.items():
|
|
addresses.setdefault(address, []).extend(run.lateness(changes, [connection])[0])
|
|
return {address: summary(lateness) for address, lateness in sorted(addresses.items())}
|
|
|
|
|
|
def lag(run, changes):
|
|
"""(median, p99) of the first copies' lag, µs, raw and corrected by the
|
|
clock's mean offset (positive: the local clock is behind)."""
|
|
raw = [run.first(change) - change[0] * 1000 for change in changes]
|
|
offset = statistics.mean(run.offsets) if run.offsets else 0
|
|
return summary(raw), summary([value + offset for value in raw]), offset
|
|
|
|
|
|
def ms(micros):
|
|
return f"{micros / 1000:.2f}"
|
|
|
|
|
|
def report(run):
|
|
minutes = run.minutes()
|
|
changes = [change for minute in minutes for change in minute]
|
|
if not changes:
|
|
return f"{len(run.connections)} connections, no change over a full minute"
|
|
lines = [
|
|
f"{len(run.connections)} connections, {len(changes)} changes "
|
|
f"over {len(minutes)} full minutes",
|
|
"",
|
|
"| Connection | Address | Speed per minute (ms) | Missed |",
|
|
"|---|---|---|---|",
|
|
]
|
|
per_minute = [speeds(run, minute) for minute in minutes]
|
|
for connection, address in sorted(run.connections.items()):
|
|
row = " ".join(ms(s[connection]) if connection in s else "-" for s in per_minute)
|
|
_, missed = run.lateness(changes, [connection])
|
|
lines.append(f"| {connection} | {address} | {row} | {missed} |")
|
|
if len(minutes) >= 2:
|
|
correlations = persistence(run, minutes)
|
|
if correlations:
|
|
each = " ".join(f"{c:.2f}" for c in correlations)
|
|
mean = statistics.mean(correlations)
|
|
lines += ["", f"Persistence: rank correlation minute to minute {each}, mean {mean:.2f}"]
|
|
top, rest = top_against_rest(run, minutes)
|
|
if top and rest:
|
|
lines.append(
|
|
f"The {TOP} fastest of minute 1 after it: {ms(top[0])} ms median, "
|
|
f"{ms(top[1])} ms p99; the others: {ms(rest[0])} ms, {ms(rest[1])} ms"
|
|
)
|
|
else:
|
|
lines.append(f"The {TOP} fastest of minute 1 against the others: too few connections")
|
|
raced = racing(run, minutes)
|
|
if raced:
|
|
picked, picked_missed, random, random_missed = raced
|
|
gain = (random[1] - picked[1]) / random[1] if random[1] else 0
|
|
lines.append(
|
|
f"Racing: the {PAIR} fastest of minute 1, {ms(picked[0])} ms median, "
|
|
f"{ms(picked[1])} ms p99, {picked_missed} missed; a pair at random, "
|
|
f"{ms(random[0])} ms, {ms(random[1])} ms, {random_missed:.1f} missed; "
|
|
f"p99 {gain:.0%} lower"
|
|
)
|
|
else:
|
|
lines.append("Racing: too few connections deliver after minute 1")
|
|
lines += ["", "| Address | Median (ms) | p99 (ms) |", "|---|---|---|"]
|
|
for address, lateness in by_address(run, changes).items():
|
|
cells = f"{ms(lateness[0])} | {ms(lateness[1])}" if lateness else "- | -"
|
|
lines.append(f"| {address} | {cells} |")
|
|
raw, corrected, offset = lag(run, changes)
|
|
lines += [
|
|
"",
|
|
f"Lag of the first copies: raw {ms(raw[0])} ms median, {ms(raw[1])} ms p99; "
|
|
f"corrected by the clock's offset ({ms(offset)} ms): "
|
|
f"{ms(corrected[0])} ms, {ms(corrected[1])} ms",
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
with Path(sys.argv[1]).open() as file:
|
|
print(report(Run(file)))
|