"""The latency probe's CSV, analysed (roadmap/07): lateness, missed changes, each connection's speed per minute, whether speed persists, what racing would gain, lateness by address, and lag. Standard library only. Usage: python3 scripts/latency/analyse.py RUN.csv Terms (GLOSSARY.md): a copy's lateness is how long after the change's first copy it came; a connection's speed is its median lateness over a minute; lag is a receive time minus the exchange's timestamp. """ import csv import itertools import statistics import sys from pathlib import Path MINUTE_US = 60_000_000 # The fastest of the first minute, against the others (persistence). TOP = 4 # Connections a symbol has in the collector: first one wins (ADR 0003). PAIR = 2 class Run: """A run's changes, each with every connection's first copy of it.""" def __init__(self, lines): self.connections = {} # connection -> address self.down = {} # connection -> when it went down, µs self.offsets = [] # the clock's offsets, µs # (symbol, identity) -> [exchange_ms, {connection: received_us}] self.changes = {} rows = [] for line in lines: if line.startswith("#"): self._note(line[1:].strip()) else: rows.append(line) for row in csv.DictReader(rows): connection = int(row["connection"]) self.connections[connection] = row["address"] key = (row["symbol"], row["identity"]) change = self.changes.setdefault(key, [int(row["exchange_ms"]), {}]) # A server's repeat on one connection: its first copy counts. change[1].setdefault(connection, int(row["received_us"])) self.up_from = {} # connection -> its first copy, µs for _, copies in self.changes.values(): for connection, received in copies.items(): self.up_from[connection] = min(self.up_from.get(connection, received), received) def _note(self, note): fields = note.split(",") if fields[0] == "down": self.down[int(fields[1])] = int(fields[2]) elif fields[0] == "offset" and fields[2]: self.offsets.append(int(fields[2])) def first(self, change): return min(change[1].values()) def minutes(self): """Each full minute's changes, from the run's first change.""" firsts = [(self.first(change), change) for change in self.changes.values()] if not firsts: return [] start = min(first for first, _ in firsts) last = max(first for first, _ in firsts) count = (last - start) // MINUTE_US # the last, partial, left out minutes = [[] for _ in range(count)] for first, change in firsts: minute = (first - start) // MINUTE_US if minute < count: minutes[minute].append(change) return minutes def up(self, connection, at): """From its first copy (its subscribe done) until it went down.""" never = float("inf") return self.up_from.get(connection, never) <= at < self.down.get(connection, never) def lateness(self, changes, connections): """The lateness of the first copy among `connections`, change by change, behind the first of all; and how many changes none of them delivered while up.""" lateness, missed = [], 0 for change in changes: first = self.first(change) copies = [change[1][c] for c in connections if c in change[1]] if copies: lateness.append(min(copies) - first) elif any(self.up(c, first) for c in connections): missed += 1 return lateness, missed def quantile(values, q): """As the collector's log computes it: the value at index n * q.""" ordered = sorted(values) return ordered[min(int(len(ordered) * q), len(ordered) - 1)] def speeds(run, minute): """Each connection's speed over a minute's changes, µs.""" result = {} for connection in run.connections: lateness, _ = run.lateness(minute, [connection]) if lateness: result[connection] = statistics.median(lateness) return result def ranked(speeds_by_connection): return sorted(speeds_by_connection, key=lambda c: (speeds_by_connection[c], c)) def persistence(run, minutes): """The rank correlation of the connections' speeds between each minute and the next.""" per_minute = [speeds(run, minute) for minute in minutes] correlations = [] for this, following in itertools.pairwise(per_minute): both = sorted(set(this) & set(following)) if len(both) > 2: correlations.append( statistics.correlation( [this[c] for c in both], [following[c] for c in both], method="ranked", ) ) return correlations def summary(lateness): if not lateness: return None return statistics.median(lateness), quantile(lateness, 0.99) def top_against_rest(run, minutes, top=TOP): """The `top` fastest of the first minute against the others, over the minutes after it: (median, p99) of their copies' lateness each.""" order = ranked(speeds(run, minutes[0])) top, rest = order[:top], order[top:] later = [change for minute in minutes[1:] for change in minute] def copies(connections): lateness = [] for connection in connections: lateness += run.lateness(later, [connection])[0] return summary(lateness) return copies(top), copies(rest) def racing(run, minutes, pair=PAIR): """The `pair` fastest of the first minute kept, against a pair at random (every pair, averaged), over the minutes after it: the (median, p99) of the pair's lateness (first one wins: the sooner of its two copies), and its missed changes. None if too few connections deliver after the first minute to compare.""" order = ranked(speeds(run, minutes[0])) later = [change for minute in minutes[1:] for change in minute] picked_lateness, picked_missed = run.lateness(later, order[:pair]) pairs = [run.lateness(later, list(two)) for two in itertools.combinations(order, pair)] random = [s for s in (summary(lateness) for lateness, _ in pairs) if s] picked = summary(picked_lateness) if len(order) < pair or picked is None or not random: return None return ( picked, picked_missed, (statistics.mean(m for m, _ in random), statistics.mean(p for _, p in random)), statistics.mean(missed for _, missed in pairs), ) def by_address(run, changes): """(median, p99) of lateness by address, every copy counted.""" addresses = {} for connection, address in run.connections.items(): addresses.setdefault(address, []).extend(run.lateness(changes, [connection])[0]) return {address: summary(lateness) for address, lateness in sorted(addresses.items())} def lag(run, changes): """(median, p99) of the first copies' lag, µs, raw and corrected by the clock's mean offset (positive: the local clock is behind).""" raw = [run.first(change) - change[0] * 1000 for change in changes] offset = statistics.mean(run.offsets) if run.offsets else 0 return summary(raw), summary([value + offset for value in raw]), offset def ms(micros): return f"{micros / 1000:.2f}" def report(run): minutes = run.minutes() changes = [change for minute in minutes for change in minute] if not changes: return f"{len(run.connections)} connections, no change over a full minute" lines = [ f"{len(run.connections)} connections, {len(changes)} changes " f"over {len(minutes)} full minutes", "", "| Connection | Address | Speed per minute (ms) | Missed |", "|---|---|---|---|", ] per_minute = [speeds(run, minute) for minute in minutes] for connection, address in sorted(run.connections.items()): row = " ".join(ms(s[connection]) if connection in s else "-" for s in per_minute) _, missed = run.lateness(changes, [connection]) lines.append(f"| {connection} | {address} | {row} | {missed} |") if len(minutes) >= 2: correlations = persistence(run, minutes) if correlations: each = " ".join(f"{c:.2f}" for c in correlations) mean = statistics.mean(correlations) lines += ["", f"Persistence: rank correlation minute to minute {each}, mean {mean:.2f}"] top, rest = top_against_rest(run, minutes) if top and rest: lines.append( f"The {TOP} fastest of minute 1 after it: {ms(top[0])} ms median, " f"{ms(top[1])} ms p99; the others: {ms(rest[0])} ms, {ms(rest[1])} ms" ) else: lines.append(f"The {TOP} fastest of minute 1 against the others: too few connections") raced = racing(run, minutes) if raced: picked, picked_missed, random, random_missed = raced gain = (random[1] - picked[1]) / random[1] if random[1] else 0 lines.append( f"Racing: the {PAIR} fastest of minute 1, {ms(picked[0])} ms median, " f"{ms(picked[1])} ms p99, {picked_missed} missed; a pair at random, " f"{ms(random[0])} ms, {ms(random[1])} ms, {random_missed:.1f} missed; " f"p99 {gain:.0%} lower" ) else: lines.append("Racing: too few connections deliver after minute 1") lines += ["", "| Address | Median (ms) | p99 (ms) |", "|---|---|---|"] for address, lateness in by_address(run, changes).items(): cells = f"{ms(lateness[0])} | {ms(lateness[1])}" if lateness else "- | -" lines.append(f"| {address} | {cells} |") raw, corrected, offset = lag(run, changes) lines += [ "", f"Lag of the first copies: raw {ms(raw[0])} ms median, {ms(raw[1])} ms p99; " f"corrected by the clock's offset ({ms(offset)} ms): " f"{ms(corrected[0])} ms, {ms(corrected[1])} ms", ] return "\n".join(lines) if __name__ == "__main__": with Path(sys.argv[1]).open() as file: print(report(Run(file)))