init commit.

This commit is contained in:
Calder White committed 2026-10-22 00:07:38 -04:00
commit c24decd482
17 files changed
+2211

No files matched your search

+1
View File
@@ -0,0 +1 @@
/target/
+12
View File
@@ -0,0 +1,12 @@
{
"recommendations": [
"rust-lang.rust-analyzer",
"tamasfe.even-better-toml",
"redhat.vscode-xml",
"hangxingliu.vscode-systemd-support",
"hashicorp.terraform",
"redhat.ansible",
"redhat.vscode-yaml",
"mkhl.shfmt"
]
}
+49
View File
@@ -0,0 +1,49 @@
{
"editor.formatOnSave": true,
"[rust]": {
"editor.defaultFormatter": "rust-lang.rust-analyzer"
},
"[toml]": {
"editor.defaultFormatter": "tamasfe.even-better-toml",
"editor.formatOnSave": true
},
"[xml]": {
"editor.defaultFormatter": "redhat.vscode-xml",
"editor.tabSize": 2,
"editor.insertSpaces": true,
"editor.detectIndentation": false
},
"xml.format.preserveAttributeLineBreaks": true,
"[terraform]": {
"editor.defaultFormatter": "hashicorp.terraform",
"editor.formatOnSave": true
},
"[terraform-vars]": {
"editor.defaultFormatter": "hashicorp.terraform",
"editor.formatOnSave": true
},
"[ansible]": {
"editor.defaultFormatter": "redhat.vscode-yaml",
"editor.formatOnSave": true,
"editor.insertSpaces": true,
"editor.tabSize": 2,
"yaml.format.enable": true
},
"[shellscript]": {
"editor.defaultFormatter": "mkhl.shfmt",
"editor.formatOnSave": true,
"editor.insertSpaces": true,
"editor.tabSize": 2
},
"files.associations": {
// *.tfvars.example doesn't match the extension's *.tfvars glob
"*.tfvars.example": "terraform-vars",
"**/.bashrc": "shellscript",
"infra/ansible/**/*.yml": "ansible",
"infra/ansible/**/*.yaml": "ansible"
},
"files.watcherExclude": {
"**/target/**": true,
"**/.terraform/**": true
}
}
Generated
+1099
View File
File diff suppressed because it is too large. Load diff
+17
View File
@@ -0,0 +1,17 @@
[package]
name = "engineering-challenge"
version = "0.1.0"
edition = "2024"
build = "src/util/build.rs"
[dependencies]
anyhow = "1"
chrono = "0.4"
crossbeam = "0.8.4"
tempfile = "3"
[build-dependencies]
ergo-sbe = "=0.1.16"
indexmap = { version = "2", features = ["serde"] }
serde = { version = "1", features = ["derive"] }
serde_yaml_ng = "0.9"
+21
View File
@@ -0,0 +1,21 @@
# Read before getting started!!
First, read `src/main.rs`. Then, some background on this repo:
This repo uses the [FIX SBE standard](https://fixtrading.org/packages/simple-binary-encoding-technical-specification-v2-rc2/)
for writing binary messages. The rust code abstracts this away for you, and you can simply use the helpers, as demonstrated in `main.rs`.
### More background, in case you are curious!
FIX SBE is defined in XML, which I find extremely cumbersome to edit, especially
when updating schemas frequently. To solve this, I have also introduced some
helper code that takes our own custom YAML format, and turns it into FIX SBE
XML, which in turn is used to generate rust code to encode and decode the
messages. **You don't need to edit any of these files**. This explanation is
simply to help you understand the origins of the various files in this repo.
The yaml for the schema you will be using is located in
`sbe_schemas/engineering_challenge/challenge_orderbook.yaml`.
We also use a custom timestamp field, so that the Rust struct can use
`chrono::DateTime` in the generated struct, and in SBE schema, it is a uint64
representing the timestamp as the number of microseconds since unix epoch.
@@ -0,0 +1,52 @@
id: 1
version: 1
description: Voluntas engineering challenge orderbook update schema.
messages:
Snapshot:
id: 1
description: A full snapshot of the orderbook
fields:
received_at: { id: 1, type: timestamp }
exchange_timestamp: { id: 2, type: timestamp }
bids:
id: 4
description: Bid price levels
fields:
price_level:
id: 5
type: int64
description: The price level, in fixed point; 1_000_000 = 1 USDC
quantity:
id: 6
type: int64
description: The absolute available quantity, in fixed point; 1_000_000 = 1 share/contract
asks:
id: 7
description: Ask price levels
fields:
price_level:
id: 8
type: int64
description: The price level, in fixed point; 1_000_000 = 1 USDC
quantity:
id: 9
type: int64
description: The absolute available quantity, in fixed point; 1_000_000 = 1 share/contract
symbol: { id: 3, type: string }
PriceLevelUpdate:
id: 2
description: An update to a single price level, on a single orderbook
fields:
backtest_timestamp: { id: 1, type: timestamp }
exchange_timestamp: { id: 2, type: timestamp }
price_level:
id: 4
type: int64
description: The price level being updated, in fixed point; 1_000_000 = 1 USDC
quantity:
id: 5
type: int64
description: The absolute available quantity, in fixed point; 1_000_000 = 1 share/contract
symbol: { id: 3, type: string }
+55
View File
@@ -0,0 +1,55 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Shared SBE wire encodings. These definitions are included by every schema so
message files contain only schema-specific types and message declarations.
Changing an existing encoding changes every schema that uses it. Add a new
named encoding instead when a different wire representation is required.
-->
<types>
<!-- Standard 8-byte SBE message header. These names are required by the tooling. -->
<composite name="messageHeader" description="Message identifiers and length of message root">
<type name="blockLength" primitiveType="uint16"/>
<type name="templateId" primitiveType="uint16"/>
<type name="schemaId" primitiveType="uint16"/>
<type name="version" primitiveType="uint16"/>
</composite>
<!-- Repeating-group dimensions written before each group instance. -->
<composite name="groupSizeEncoding" description="Repeating group dimensions">
<type name="blockLength" primitiveType="uint16"/>
<type name="numInGroup" primitiveType="uint16"/>
</composite>
<!-- Variable-length UTF-8 string with a uint32 byte-length prefix. -->
<composite name="varStringEncoding" description="Variable-length UTF-8 string">
<type name="length" primitiveType="uint32" maxValue="1073741824"/>
<type name="varData" primitiveType="uint8" length="0" characterEncoding="UTF-8"/>
</composite>
<!-- UTC microseconds since the UNIX epoch; the constant unit occupies no wire bytes. -->
<composite name="UTCTimestampMicrosEncoding" semanticType="UTCTimestamp"
description="UTC microseconds since the UNIX epoch">
<type name="time" primitiveType="uint64"/>
<type name="unit" primitiveType="uint8" presence="constant">6</type>
</composite>
<!-- Same 8 bytes, but u64::MAX is reserved to mean "unknown". SBE has no
presence="optional" for a composite as a whole, so the nullability rides
on `time` and you test that member — the pattern real-logic points at in
issue #509. Picked by `type: timestamp, optional: true` in the yaml. -->
<composite name="UTCTimestampMicrosNullableEncoding" semanticType="UTCTimestamp"
description="UTC microseconds since the UNIX epoch, or null">
<type name="time" primitiveType="uint64" presence="optional"/>
<type name="unit" primitiveType="uint8" presence="constant">6</type>
</composite>
<type name="bytes32" primitiveType="uint8" length="32"/>
<type name="address" primitiveType="uint8" length="20"/>
<!-- FIX-standard boolean: one byte on the wire. -->
<enum name="BooleanType" encodingType="uint8">
<validValue name="False">0</validValue>
<validValue name="True">1</validValue>
</enum>
</types>
+1
View File
@@ -0,0 +1 @@
pub mod util;
+64
View File
@@ -0,0 +1,64 @@
use chrono::DateTime;
use engineering_challenge::util::dtos::challenge_orderbook::{
AnyMessage, PriceLevelUpdateDomain, SnapshotAsksEntryDomain, SnapshotBidsEntryDomain,
SnapshotDomain,
};
use engineering_challenge::util::{AlignedSbeReader, AlignedSbeWriter, OrderbookSchema};
const CHUNK_SIZE_64_MEBIBYTES: usize = 64 << 20;
/// This is a dummy example where we define a snapshot and price level update,
/// write them to a file using [`AlignedSbeWriter`], and then read them back
/// from the same file using [`AlignedSbeReader`], asserting that what we read
/// back is the exact same as what we wrote.
///
/// You should replace this main function with your actual solution, and split
/// up your code into modules in `src/` in a way that makes sense to you.
fn main() -> anyhow::Result<()> {
let timestamp = DateTime::from_timestamp_micros(1_700_000_000_000_000).unwrap();
let symbol = "EXAMPLE".to_owned();
let snapshot = SnapshotDomain {
received_at: timestamp,
exchange_timestamp: timestamp,
bids: vec![SnapshotBidsEntryDomain {
price_level: 400_000,
quantity: 2_000_000,
}],
asks: vec![SnapshotAsksEntryDomain {
price_level: 600_000,
quantity: 3_500_000,
}],
symbol: symbol.clone(),
};
let update = PriceLevelUpdateDomain {
backtest_timestamp: timestamp,
exchange_timestamp: timestamp,
price_level: 400_000,
quantity: 1_250_000,
symbol: symbol.clone(),
};
let file = tempfile::NamedTempFile::new()?;
let mut writer = AlignedSbeWriter::create(file.path(), CHUNK_SIZE_64_MEBIBYTES)?;
writer.write_msg(&snapshot)?;
writer.write_msg(&update)?;
writer.flush()?;
drop(writer);
let mut reader = AlignedSbeReader::<OrderbookSchema>::open(file.path())?;
let Some(AnyMessage::Snapshot(decoded)) = reader.read_msg()? else {
anyhow::bail!("expected snapshot");
};
assert_eq!(SnapshotDomain::try_from_decoder(decoded)?, snapshot);
let Some(AnyMessage::PriceLevelUpdate(decoded)) = reader.read_msg()? else {
anyhow::bail!("expected price level update");
};
assert_eq!(PriceLevelUpdateDomain::try_from_decoder(decoded)?, update);
assert!(reader.read_msg()?.is_none());
println!("Verification success!");
Ok(())
}
+251
View File
@@ -0,0 +1,251 @@
//! Defines the [`AlignedSbeReader`], which allows us to quickly read SBE
//! messages written by the
//! [`AlignedSbeWriter`](crate::util::aligned_sbe_writer::AlignedSbeWriter)
use std::{fs::File, marker::PhantomData, os::unix::fs::FileExt, path::Path};
use anyhow::Context;
use crossbeam::channel::{Receiver, Sender, bounded};
/// Any SBEs we want to read with the AlignedSbeReader must implement this
/// trait.
pub trait AlignedSbeSchema: 'static {
const SCHEMA_ID: u16;
const SCHEMA_VERSION: u16;
type Message<'buf>
where
Self: 'buf;
fn try_decode<'buf>(buffer: &'buf [u8], offset: usize) -> anyhow::Result<Self::Message<'buf>>;
fn encoded_length(message: &Self::Message<'_>) -> anyhow::Result<usize>;
fn expected_block_length(message: &Self::Message<'_>) -> anyhow::Result<usize>;
}
const DEFAULT_CHUNK_SIZE: usize = 64 << 20; // 64MiB
// Every schema in `dtos` uses the standard eight-byte SBE message header.
const HEADER_LENGTH: usize = 8;
/// See [`AlignedSbeReader::create_prefetch_task`] for more details.
pub type PrefetchTask = Box<dyn FnOnce() + Send + 'static>;
/// The [`AlignedSbeReader`] is designed to solve the issue of reading partial
/// messages from SBE flat files. Some background:
///
/// We want to read aeron-spec SBEs from disk at extremely fast speeds -- 100M
/// messages/sec on a single reader thread (with another prefetch thread/task in
/// the background).
///
/// Additionally, we wanted a very simple API to do so. To achieve this, we
/// prefetch our reads, and read in big chunks ([`DEFAULT_CHUNK_SIZE`]). However, when
/// you read in chunks, messages might tear across the chunk boundary. This is
/// undesireable, because now we need to coordinate our prefetch with our
/// reader, in case we need to read the next chunk to finish the currenet
/// message that is truncated on the current chunk.
///
/// This is annoying AF, so to solve the problem, we moved upstream: **Write the
/// encoded SBE messages all-or-nothing, aligned to the chunk size.** If we are
/// writing messages and we find that the next message will tear across two
/// chunks, we simply pad the rest of the current chunk with zeros, and write
/// the message, in full, starting at the next chunk.
///
/// This way the reader can then see the zeros, know the chunk is done, and skip
/// to the next one.
pub struct AlignedSbeReader<T: AlignedSbeSchema> {
file: Option<File>,
file_len: u64,
chunk_size: usize,
buffer: Vec<u8>,
file_offset: u64,
buffer_pos: usize,
eof: bool,
ready_chunks: Option<Receiver<std::io::Result<Option<Vec<u8>>>>>,
recycled_chunks: Option<Sender<Vec<u8>>>,
_schema: PhantomData<T>,
}
impl<T: AlignedSbeSchema> AlignedSbeReader<T> {
/// Create a new [`AlignedSbeReader`] from the file at `path`. Chunk size is
/// set as a constant in this file, [`DEFAULT_CHUNK_SIZE`].
pub fn open(path: impl AsRef<Path>) -> anyhow::Result<Self> {
Self::open_with_chunk_size(path, DEFAULT_CHUNK_SIZE)
}
/// Create a new [`AlignedSbeReader`] from the file at `path`, with
/// `chunk_size` for chunk size.
fn open_with_chunk_size(path: impl AsRef<Path>, chunk_size: usize) -> anyhow::Result<Self> {
let file = File::open(path)?;
let file_len = file.metadata()?.len();
Ok(Self {
file: Some(file),
file_len,
chunk_size,
buffer: vec![0; chunk_size],
file_offset: 0,
buffer_pos: 0,
eof: false,
ready_chunks: None,
recycled_chunks: None,
_schema: PhantomData,
})
}
/// Create a task to be passed up to the owner of this reader, so it can run
/// chunk prefetching on a separate thread (optionally). When this method is
/// run, this consumes certain fields of `self` so they may only be used by
/// the prefetch task.
pub fn create_prefetch_task(&mut self) -> anyhow::Result<PrefetchTask> {
anyhow::ensure!(
self.file_offset == 0 && self.buffer_pos == 0,
"prefetch must start before reading"
);
let file = self.file.take().context("prefetch already started")?;
let file_len = self.file_len;
let chunk_size = self.chunk_size;
let spare_buffers = [std::mem::take(&mut self.buffer), vec![0; self.chunk_size]];
let (ready_chunks_tx, ready_chunks) = bounded(1);
let (recycled_chunks, recycled_chunks_rx) = bounded(2);
self.ready_chunks = Some(ready_chunks);
self.recycled_chunks = Some(recycled_chunks);
Ok(Box::new(move || {
let mut spare_buffers = spare_buffers.into_iter();
let mut file_offset = 0u64;
loop {
let Some(end) = file_offset.checked_add(chunk_size as u64) else {
return;
};
if end > file_len {
let _ = ready_chunks_tx.send(Ok(None));
return;
}
let mut buffer = match spare_buffers.next() {
Some(buffer) => buffer,
None => match recycled_chunks_rx.recv() {
Ok(buffer) => buffer,
Err(_) => return,
},
};
if let Err(error) = file.read_exact_at(&mut buffer, file_offset) {
let _ = ready_chunks_tx.send(Err(error));
return;
}
if ready_chunks_tx.send(Ok(Some(buffer))).is_err() {
return;
}
file_offset = end;
}
}))
}
/// Signal to the prefetch task that it should stop running.
pub fn signal_stop_prefetch(&mut self) {
self.ready_chunks = None;
self.recycled_chunks = None;
}
/// Returns whether the next chunk could be loaded or not.
fn load_chunk(&mut self) -> anyhow::Result<bool> {
if self.eof {
return Ok(false);
}
if let Some(ready_chunks) = &self.ready_chunks {
if !self.buffer.is_empty() {
let buffer = std::mem::take(&mut self.buffer);
if let Some(recycled_chunks) = &self.recycled_chunks {
let _ = recycled_chunks.send(buffer);
}
}
return match ready_chunks
.recv()
.context("SBE prefetch thread stopped")??
{
Some(buffer) => {
self.buffer = buffer;
self.buffer_pos = 0;
Ok(true)
}
None => {
self.eof = true;
Ok(false)
}
};
}
let end = self
.file_offset
.checked_add(self.chunk_size as u64)
.context("offset overflow")?;
if end > self.file_len {
return Ok(false);
}
self.file
.as_ref()
.context("file moved to prefetch task")?
.read_exact_at(&mut self.buffer, self.file_offset)?;
self.file_offset = end;
self.buffer_pos = 0;
Ok(true)
}
/// Decode the next message from our chunk buffer.
///
/// Returns `Ok(None)` once we reach the end of the file, and `Err(...)` if
/// there is an erroring reading the file, decoding the next message, etc.
pub fn read_msg(&mut self) -> anyhow::Result<Option<T::Message<'_>>> {
loop {
let header_end = self
.buffer_pos
.checked_add(HEADER_LENGTH)
.context("pos overflow")?;
if header_end <= self.buffer.len()
&& self.buffer[self.buffer_pos..header_end] != [0; HEADER_LENGTH]
{
break;
}
if !self.load_chunk()? {
return Ok(None);
}
}
// All generated schemas share this header layout:
// blockLength, templateId, schemaId, version -- all little-endian u16.
let header_end = self.buffer_pos + HEADER_LENGTH;
let header = &self.buffer[self.buffer_pos..header_end];
let block_length = usize::from(u16::from_le_bytes([header[0], header[1]]));
let schema_id = u16::from_le_bytes([header[4], header[5]]);
let schema_version = u16::from_le_bytes([header[6], header[7]]);
anyhow::ensure!(schema_id == T::SCHEMA_ID);
anyhow::ensure!(schema_version == T::SCHEMA_VERSION);
let decoded = T::try_decode(&self.buffer, self.buffer_pos)?;
anyhow::ensure!(block_length == T::expected_block_length(&decoded)?);
let message_len = T::encoded_length(&decoded)?;
let end = self
.buffer_pos
.checked_add(message_len)
.context("overflow")?;
self.buffer_pos = end;
Ok(Some(decoded))
}
}
+76
View File
@@ -0,0 +1,76 @@
//! [`AlignedSbeWriter`] writes SBE messages in fixed-size, zero-padded file chunks.
use std::{fs::File, io::Write, path::Path};
use super::dtos::SbeEncode;
// Every schema in `dtos` uses the standard eight-byte SBE message header.
const HEADER_LENGTH: usize = 8;
/// Writes SBE messages to a flat file in fixed size, zero padded chunks.
///
/// See [`AlignedSbeReader`](crate::util::AlignedSbeReader) for more details on why
/// this struct exists.
///
/// Every chunk reserves enough trailing zeroes for an empty SBE header, so a
/// reader can stop decoding at the sentinel without carrying a frame across a
/// chunk boundary.
///
/// Call [`Self::flush`] to write the final buffered chunk before dropping the
/// writer.
pub struct AlignedSbeWriter {
file: File,
buf: Vec<u8>,
chunk: usize,
}
impl AlignedSbeWriter {
/// Create an [`AlignedSbeWriter`], writing to `path` with `chunk` chunk
/// size. Clears the file at `path`, if it exists.
pub fn create(path: impl AsRef<Path>, chunk: usize) -> anyhow::Result<Self> {
anyhow::ensure!(
chunk > HEADER_LENGTH,
"chunk must leave room for SBE data and an SBE header"
);
Ok(Self {
file: File::create(path)?,
buf: Vec::with_capacity(chunk),
chunk,
})
}
/// Encodes a complete message directly into the chunk buffer. Rotates to a
/// new chunk if it does not fit in the existing chunk.
pub fn write_msg<T: SbeEncode + ?Sized>(&mut self, message: &T) -> anyhow::Result<()> {
let len = message.encoded_length_with_header()?;
let payload_limit = self.chunk - HEADER_LENGTH;
anyhow::ensure!(len <= payload_limit, "SBE message does not fit in a chunk");
if len > payload_limit - self.buf.len() {
self.flush()?;
}
let start = self.buf.len();
self.buf.resize(start + len, 0);
let result = message.encode(&mut self.buf[start..]).and_then(|written| {
anyhow::ensure!(
written == len,
"SBE encoded length mismatch: expected {len} bytes, wrote {written}"
);
Ok(())
});
if result.is_err() {
self.buf.truncate(start);
}
result
}
/// Flush the current buffer to disk, and clear it.
pub fn flush(&mut self) -> anyhow::Result<()> {
if !self.buf.is_empty() {
self.buf.resize(self.chunk, 0);
self.file.write_all(&self.buf)?;
self.buf.clear();
}
Ok(())
}
}
+32
View File
@@ -0,0 +1,32 @@
//! Build helpers to auto-generate the rust code for the SBE schemas.
pub mod sbe_yaml;
use std::fs;
use ergo_sbe::miette::{IntoDiagnostic, Result, miette};
use ergo_sbe::{ConversionSelector, DomainVarData, GenerationConfig, generate_str_to_out_dir};
const SCHEMA: &str = "sbe_schemas/engineering_challenge/challenge_orderbook.yaml";
const COMMON_TYPES: &str = "sbe_schemas/includes/common-types.xml";
fn main() -> Result<()> {
println!("cargo::rerun-if-changed={SCHEMA}");
println!("cargo::rerun-if-changed={COMMON_TYPES}");
let yaml = fs::read_to_string(SCHEMA).into_diagnostic()?;
let schema = sbe_yaml::parse(&yaml).map_err(|error| miette!("{error}"))?;
let common_types = fs::read_to_string(COMMON_TYPES).into_diagnostic()?;
let xml = schema
.to_xml("challenge_orderbook_messages", &common_types)
.map_err(|error| miette!("{error}"))?;
let config = GenerationConfig::new("challenge_orderbook")
.with_domain_objects(DomainVarData::Strings)
.with_domain_type(
ConversionSelector::named_type("UTCTimestampMicrosEncoding"),
"chrono::DateTime<chrono::Utc>",
);
generate_str_to_out_dir(&xml, config)?;
Ok(())
}
+82
View File
@@ -0,0 +1,82 @@
//! SBE helper types, such as [`SbeEncode`].
use chrono::{DateTime, Utc};
macro_rules! schemas {
($($schema:ident),* $(,)?) => {$(
pub mod $schema {
#![allow(
dead_code,
unused_variables,
unused_mut,
unused_assignments,
unused_comparisons,
unused_unsafe,
clippy::all
)]
include!(concat!(env!("OUT_DIR"), "/", stringify!($schema), ".rs"));
}
impl $schema::TryFromSbe<$schema::UTCTimestampMicrosEncoding> for DateTime<Utc> {
type Error = &'static str;
fn try_from_sbe(wire: $schema::UTCTimestampMicrosEncoding) -> Result<Self, Self::Error> {
i64::try_from(wire.time())
.ok()
.and_then(DateTime::from_timestamp_micros)
.ok_or("UTC microseconds outside chrono's range")
}
}
impl $schema::TryToSbe<$schema::UTCTimestampMicrosEncoding> for DateTime<Utc> {
type Error = &'static str;
fn try_to_sbe(&self) -> Result<$schema::UTCTimestampMicrosEncoding, Self::Error> {
u64::try_from(self.timestamp_micros())
.map($schema::UTCTimestampMicrosEncoding::new)
.map_err(|_| "UTC timestamp predates the Unix epoch")
}
}
)*};
}
/// This trait allows us to encode multiple SBE schemas using the same function.
pub trait SbeEncode {
/// Returns the exact encoded length, including the SBE message header.
fn encoded_length_with_header(&self) -> anyhow::Result<usize>;
/// Encodes the message, including its header, into `buf` and returns the bytes written.
/// The buffer must be at least `encoded_length_with_header()` bytes long.
/// An encoding error may leave the buffer partially written.
fn encode(&self, buf: &mut [u8]) -> anyhow::Result<usize>;
fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
let mut buf = vec![0u8; self.encoded_length_with_header()?];
let len = self.encode(&mut buf)?;
debug_assert_eq!(len, buf.len());
Ok(buf)
}
}
macro_rules! sbe_encode {
($($message:ty),* $(,)?) => {$(
impl SbeEncode for $message {
#[inline]
fn encoded_length_with_header(&self) -> anyhow::Result<usize> {
Ok(<$message>::encoded_length_with_header(self)?)
}
#[inline]
fn encode(&self, buf: &mut [u8]) -> anyhow::Result<usize> {
Ok(<$message>::encode(self, buf)?)
}
}
)*};
}
schemas!(challenge_orderbook);
sbe_encode!(
challenge_orderbook::SnapshotDomain,
challenge_orderbook::PriceLevelUpdateDomain,
);
+8
View File
@@ -0,0 +1,8 @@
mod aligned_sbe_reader;
mod aligned_sbe_writer;
pub mod dtos;
mod orderbook_schema;
pub use aligned_sbe_reader::{AlignedSbeReader, AlignedSbeSchema, PrefetchTask};
pub use aligned_sbe_writer::AlignedSbeWriter;
pub use orderbook_schema::OrderbookSchema;
+29
View File
@@ -0,0 +1,29 @@
use super::AlignedSbeSchema;
use super::dtos::challenge_orderbook::{
AnyMessage, PriceLevelUpdateSchema, SCHEMA_ID, SCHEMA_VERSION, SnapshotSchema,
};
pub struct OrderbookSchema;
impl AlignedSbeSchema for OrderbookSchema {
const SCHEMA_ID: u16 = SCHEMA_ID;
const SCHEMA_VERSION: u16 = SCHEMA_VERSION;
type Message<'buf> = AnyMessage<'buf>;
fn try_decode<'buf>(buffer: &'buf [u8], offset: usize) -> anyhow::Result<Self::Message<'buf>> {
Ok(AnyMessage::try_decode(buffer, offset)?)
}
fn encoded_length(message: &Self::Message<'_>) -> anyhow::Result<usize> {
Ok(message.encoded_length_with_header()?)
}
fn expected_block_length(message: &Self::Message<'_>) -> anyhow::Result<usize> {
Ok(match message {
AnyMessage::Snapshot(_) => SnapshotSchema::BLOCK_LENGTH,
AnyMessage::PriceLevelUpdate(_) => PriceLevelUpdateSchema::BLOCK_LENGTH,
_ => anyhow::bail!("unknown message"),
})
}
}
+362
View File
@@ -0,0 +1,362 @@
//! Custom transpiler from yaml to FIX SBE XML.
use std::fmt::{self, Write as _};
use std::marker::PhantomData;
use std::ops::Deref;
use indexmap::IndexMap;
use serde::Deserialize;
use serde::de::{Deserializer, Error as _, MapAccess, Visitor};
pub type Error = Box<dyn std::error::Error + Send + Sync>;
#[derive(Debug)]
pub struct Unique<V>(IndexMap<String, V>);
impl<V> Default for Unique<V> {
fn default() -> Self {
Self(IndexMap::new())
}
}
impl<V> Deref for Unique<V> {
type Target = IndexMap<String, V>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<'de, V: Deserialize<'de>> Deserialize<'de> for Unique<V> {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct UniqueVisitor<V>(PhantomData<V>);
impl<'de, V: Deserialize<'de>> Visitor<'de> for UniqueVisitor<V> {
type Value = Unique<V>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("a mapping with unique keys")
}
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Self::Value, A::Error> {
let mut entries = IndexMap::new();
while let Some((key, value)) = map.next_entry::<String, V>()? {
if entries.insert(key.clone(), value).is_some() {
return Err(A::Error::custom(format!("duplicate key `{key}`")));
}
}
Ok(Unique(entries))
}
}
deserializer.deserialize_map(UniqueVisitor(PhantomData))
}
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Schema {
pub id: u16,
pub version: u16,
pub description: Option<String>,
#[serde(default)]
pub imports: Vec<String>,
#[serde(default)]
pub enums: Unique<Unique<u8>>,
#[serde(default)]
pub composites: Unique<Composite>,
pub messages: Unique<Message>,
}
/// A shared YAML file contains type definitions, without a schema ID or messages.
#[derive(Debug, Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SharedTypes {
#[serde(default)]
pub enums: Unique<Unique<u8>>,
#[serde(default)]
pub composites: Unique<Composite>,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Composite {
pub description: Option<String>,
pub fields: Unique<CompositeMember>,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CompositeMember {
/// Only used when a message expands this composite with `fields_from`.
pub id: Option<u16>,
pub r#type: String,
#[serde(default)]
pub optional: bool,
pub since: Option<u16>,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Message {
pub id: u16,
pub description: Option<String>,
pub fields: Option<Unique<Member>>,
/// Expand a shared composite as ordinary fields, preserving flat message APIs.
pub fields_from: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Member {
pub id: u16,
/// Documentation only; emitted as an XML attribute without changing the wire layout.
pub description: Option<String>,
pub r#type: Option<String>,
pub fields: Option<Unique<Member>>,
#[serde(default)]
pub optional: bool,
pub since: Option<u16>,
}
pub fn parse(yaml: &str) -> Result<Schema, Error> {
Ok(serde_yaml_ng::from_str(yaml)?)
}
pub fn parse_shared_types(yaml: &str) -> Result<SharedTypes, Error> {
Ok(serde_yaml_ng::from_str(yaml)?)
}
impl Schema {
pub fn import_types(&mut self, types: SharedTypes) -> Result<(), Error> {
for name in types.enums.keys().chain(types.composites.keys()) {
if self.enums.contains_key(name) || self.composites.contains_key(name) {
return Err(format!("duplicate type `{name}`").into());
}
}
self.enums.0.extend(types.enums.0);
self.composites.0.extend(types.composites.0);
Ok(())
}
pub fn to_xml(&self, package: &str, common_types: &str) -> Result<String, Error> {
let mut xml = String::new();
writeln!(xml, r#"<?xml version="1.0" encoding="UTF-8"?>"#)?;
writeln!(
xml,
r#"<sbe:messageSchema xmlns:sbe="http://fixprotocol.io/2016/sbe""#
)?;
writeln!(xml, r#" package="{}""#, escape(package))?;
writeln!(xml, r#" id="{}""#, self.id)?;
writeln!(xml, r#" version="{}""#, self.version)?;
write!(xml, r#" byteOrder="littleEndian""#)?;
if let Some(description) = &self.description {
write!(
xml,
"\n description=\"{}\"",
escape(description)
)?;
}
writeln!(xml, ">")?;
writeln!(xml, "{}", types_element(common_types)?)?;
if !self.enums.is_empty() || !self.composites.is_empty() {
writeln!(xml, " <types>")?;
for (name, values) in self.enums.iter() {
writeln!(xml, r#" <enum name="{name}" encodingType="uint8">"#)?;
for (value, number) in values.iter() {
writeln!(
xml,
r#" <validValue name="{value}">{number}</validValue>"#
)?;
}
writeln!(xml, " </enum>")?;
}
for (name, composite) in self.composites.iter() {
if self.enums.contains_key(name) {
return Err(format!("duplicate type `{name}`").into());
}
if composite.fields.is_empty() {
return Err(format!("composite `{name}` must have fields").into());
}
write!(xml, r#" <composite name="{name}""#)?;
if let Some(description) = &composite.description {
write!(xml, r#" description="{}""#, escape(description))?;
}
writeln!(xml, ">")?;
for (field, member) in composite.fields.iter() {
let path = format!("{name}.{field}");
let (kind, presence) = composite_member_type(&path, member)?;
let since = since_attribute(member.since);
if PRIMITIVES.contains(&kind) {
writeln!(
xml,
r#" <type name="{field}" primitiveType="{kind}"{presence}{since}/>"#
)?;
} else {
writeln!(xml, r#" <ref name="{field}" type="{kind}"{since}/>"#)?;
}
}
writeln!(xml, " </composite>")?;
}
writeln!(xml, " </types>")?;
}
for (name, message) in self.messages.iter() {
write!(xml, r#" <sbe:message name="{name}" id="{}""#, message.id)?;
if let Some(description) = &message.description {
write!(xml, r#" description="{}""#, escape(description))?;
}
writeln!(xml, ">")?;
match (&message.fields, &message.fields_from) {
(Some(fields), None) => write_members(&mut xml, name, fields, 2)?,
(None, Some(composite_name)) => {
let composite = self.composites.get(composite_name).ok_or_else(|| {
format!("{name}: unknown composite `{composite_name}` in `fields_from`")
})?;
for (field, member) in composite.fields.iter() {
let path = format!("{name}.{field}");
let id = member.id.ok_or_else(|| {
format!(
"{path}: `fields_from` requires an `id` for each composite member"
)
})?;
let (kind, presence) = composite_member_type(&path, member)?;
let since = since_attribute(member.since);
writeln!(
xml,
r#" <field name="{field}" id="{id}" type="{kind}"{presence}{since}/>"#
)?;
}
}
_ => {
return Err(
format!("{name}: needs exactly one of `fields` or `fields_from`").into(),
);
}
}
writeln!(xml, " </sbe:message>")?;
}
writeln!(xml, "</sbe:messageSchema>")?;
Ok(xml)
}
}
const PRIMITIVES: &[&str] = &[
"char", "uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64", "float",
"double",
];
fn since_attribute(version: Option<u16>) -> String {
version
.map(|v| format!(r#" sinceVersion="{v}""#))
.unwrap_or_default()
}
fn composite_member_type<'a>(
path: &str,
member: &'a CompositeMember,
) -> Result<(&'a str, &'static str), Error> {
if matches!(member.r#type.as_str(), "string" | "bytes") {
return Err(format!("{path}: composites require fixed-size fields").into());
}
field_type(path, &member.r#type, member.optional)
}
fn field_type<'a>(
path: &str,
kind: &'a str,
optional: bool,
) -> Result<(&'a str, &'static str), Error> {
Ok(match (kind, optional) {
("bool", false) => ("BooleanType", ""),
("timestamp", false) => ("UTCTimestampMicrosEncoding", ""),
("timestamp", true) => ("UTCTimestampMicrosNullableEncoding", ""),
(kind, true) if NULLABLE_PRIMITIVES.contains(&kind) => (kind, r#" presence="optional""#),
(kind, true) => {
return Err(format!(
"{path}: `{kind}` has no null to spare, so `optional` would be \
silently dropped. Enums and `bool` are already nullable — match \
on `NullVal`; for anything else use a 0-or-1 group."
)
.into());
}
(kind, false) => (kind, ""),
})
}
/// SBE reserves the max value of these as the null, so `optional` works as-is.
const NULLABLE_PRIMITIVES: &[&str] = &[
"uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64", "float", "double",
];
fn write_members(
xml: &mut String,
path: &str,
members: &Unique<Member>,
depth: usize,
) -> Result<(), Error> {
let indent = " ".repeat(depth);
for (name, member) in members.iter() {
let path = format!("{path}.{name}");
let id = member.id;
let since = since_attribute(member.since);
let description = member
.description
.as_ref()
.map(|text| format!(r#" description="{}""#, escape(text)))
.unwrap_or_default();
match (member.r#type.as_deref(), &member.fields) {
(Some(_), Some(_)) | (None, None) => {
return Err(format!("{path}: needs exactly one of `type` or `fields`").into());
}
(None, Some(fields)) => {
if member.optional {
return Err(format!("{path}: a group cannot be `optional`").into());
}
writeln!(
xml,
r#"{indent}<group name="{name}" id="{id}" dimensionType="groupSizeEncoding"{since}{description}>"#
)?;
write_members(xml, &path, fields, depth + 1)?;
writeln!(xml, "{indent}</group>")?;
}
(Some("string"), None) => {
if member.optional {
return Err(format!("{path}: a string cannot be `optional`").into());
}
writeln!(
xml,
r#"{indent}<data name="{name}" id="{id}" type="varStringEncoding"{since}{description}/>"#
)?;
}
(Some(kind), None) => {
// `optional` is only ever a reserved value in the encoding, so it
// is spelled differently per type: primitives get SBE's own null
// (the type's max), a timestamp swaps to the nullable composite,
// and anything else has nowhere to put one.
let (kind, presence) = field_type(&path, kind, member.optional)?;
writeln!(
xml,
r#"{indent}<field name="{name}" id="{id}" type="{kind}"{presence}{since}{description}/>"#
)?;
}
}
}
Ok(())
}
fn types_element(xml: &str) -> Result<&str, Error> {
let start = xml
.find("<types>")
.ok_or("common types must be a <types> element")?;
let end = xml
.rfind("</types>")
.ok_or("common types must be a <types> element")?;
Ok(&xml[start..end + "</types>".len()])
}
fn escape(text: &str) -> String {
text.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
}