Archived
init commit.
This commit is contained in:
commit
c24decd482
17 files changed
+2211
No files matched your search
@@ -0,0 +1 @@
|
||||
pub mod util;
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
use chrono::DateTime;
|
||||
use engineering_challenge::util::dtos::challenge_orderbook::{
|
||||
AnyMessage, PriceLevelUpdateDomain, SnapshotAsksEntryDomain, SnapshotBidsEntryDomain,
|
||||
SnapshotDomain,
|
||||
};
|
||||
use engineering_challenge::util::{AlignedSbeReader, AlignedSbeWriter, OrderbookSchema};
|
||||
|
||||
const CHUNK_SIZE_64_MEBIBYTES: usize = 64 << 20;
|
||||
|
||||
/// This is a dummy example where we define a snapshot and price level update,
|
||||
/// write them to a file using [`AlignedSbeWriter`], and then read them back
|
||||
/// from the same file using [`AlignedSbeReader`], asserting that what we read
|
||||
/// back is the exact same as what we wrote.
|
||||
///
|
||||
/// You should replace this main function with your actual solution, and split
|
||||
/// up your code into modules in `src/` in a way that makes sense to you.
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let timestamp = DateTime::from_timestamp_micros(1_700_000_000_000_000).unwrap();
|
||||
let symbol = "EXAMPLE".to_owned();
|
||||
let snapshot = SnapshotDomain {
|
||||
received_at: timestamp,
|
||||
exchange_timestamp: timestamp,
|
||||
bids: vec![SnapshotBidsEntryDomain {
|
||||
price_level: 400_000,
|
||||
quantity: 2_000_000,
|
||||
}],
|
||||
asks: vec![SnapshotAsksEntryDomain {
|
||||
price_level: 600_000,
|
||||
quantity: 3_500_000,
|
||||
}],
|
||||
symbol: symbol.clone(),
|
||||
};
|
||||
let update = PriceLevelUpdateDomain {
|
||||
backtest_timestamp: timestamp,
|
||||
exchange_timestamp: timestamp,
|
||||
price_level: 400_000,
|
||||
quantity: 1_250_000,
|
||||
symbol: symbol.clone(),
|
||||
};
|
||||
|
||||
let file = tempfile::NamedTempFile::new()?;
|
||||
let mut writer = AlignedSbeWriter::create(file.path(), CHUNK_SIZE_64_MEBIBYTES)?;
|
||||
writer.write_msg(&snapshot)?;
|
||||
writer.write_msg(&update)?;
|
||||
writer.flush()?;
|
||||
drop(writer);
|
||||
|
||||
let mut reader = AlignedSbeReader::<OrderbookSchema>::open(file.path())?;
|
||||
|
||||
let Some(AnyMessage::Snapshot(decoded)) = reader.read_msg()? else {
|
||||
anyhow::bail!("expected snapshot");
|
||||
};
|
||||
assert_eq!(SnapshotDomain::try_from_decoder(decoded)?, snapshot);
|
||||
|
||||
let Some(AnyMessage::PriceLevelUpdate(decoded)) = reader.read_msg()? else {
|
||||
anyhow::bail!("expected price level update");
|
||||
};
|
||||
assert_eq!(PriceLevelUpdateDomain::try_from_decoder(decoded)?, update);
|
||||
assert!(reader.read_msg()?.is_none());
|
||||
|
||||
println!("Verification success!");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
//! Defines the [`AlignedSbeReader`], which allows us to quickly read SBE
|
||||
//! messages written by the
|
||||
//! [`AlignedSbeWriter`](crate::util::aligned_sbe_writer::AlignedSbeWriter)
|
||||
|
||||
use std::{fs::File, marker::PhantomData, os::unix::fs::FileExt, path::Path};
|
||||
|
||||
use anyhow::Context;
|
||||
use crossbeam::channel::{Receiver, Sender, bounded};
|
||||
|
||||
/// Any SBEs we want to read with the AlignedSbeReader must implement this
|
||||
/// trait.
|
||||
pub trait AlignedSbeSchema: 'static {
|
||||
const SCHEMA_ID: u16;
|
||||
const SCHEMA_VERSION: u16;
|
||||
|
||||
type Message<'buf>
|
||||
where
|
||||
Self: 'buf;
|
||||
|
||||
fn try_decode<'buf>(buffer: &'buf [u8], offset: usize) -> anyhow::Result<Self::Message<'buf>>;
|
||||
|
||||
fn encoded_length(message: &Self::Message<'_>) -> anyhow::Result<usize>;
|
||||
|
||||
fn expected_block_length(message: &Self::Message<'_>) -> anyhow::Result<usize>;
|
||||
}
|
||||
|
||||
const DEFAULT_CHUNK_SIZE: usize = 64 << 20; // 64MiB
|
||||
|
||||
// Every schema in `dtos` uses the standard eight-byte SBE message header.
|
||||
const HEADER_LENGTH: usize = 8;
|
||||
|
||||
/// See [`AlignedSbeReader::create_prefetch_task`] for more details.
|
||||
pub type PrefetchTask = Box<dyn FnOnce() + Send + 'static>;
|
||||
|
||||
/// The [`AlignedSbeReader`] is designed to solve the issue of reading partial
|
||||
/// messages from SBE flat files. Some background:
|
||||
///
|
||||
/// We want to read aeron-spec SBEs from disk at extremely fast speeds -- 100M
|
||||
/// messages/sec on a single reader thread (with another prefetch thread/task in
|
||||
/// the background).
|
||||
///
|
||||
/// Additionally, we wanted a very simple API to do so. To achieve this, we
|
||||
/// prefetch our reads, and read in big chunks ([`DEFAULT_CHUNK_SIZE`]). However, when
|
||||
/// you read in chunks, messages might tear across the chunk boundary. This is
|
||||
/// undesireable, because now we need to coordinate our prefetch with our
|
||||
/// reader, in case we need to read the next chunk to finish the currenet
|
||||
/// message that is truncated on the current chunk.
|
||||
///
|
||||
/// This is annoying AF, so to solve the problem, we moved upstream: **Write the
|
||||
/// encoded SBE messages all-or-nothing, aligned to the chunk size.** If we are
|
||||
/// writing messages and we find that the next message will tear across two
|
||||
/// chunks, we simply pad the rest of the current chunk with zeros, and write
|
||||
/// the message, in full, starting at the next chunk.
|
||||
///
|
||||
/// This way the reader can then see the zeros, know the chunk is done, and skip
|
||||
/// to the next one.
|
||||
pub struct AlignedSbeReader<T: AlignedSbeSchema> {
|
||||
file: Option<File>,
|
||||
file_len: u64,
|
||||
chunk_size: usize,
|
||||
buffer: Vec<u8>,
|
||||
|
||||
file_offset: u64,
|
||||
buffer_pos: usize,
|
||||
eof: bool,
|
||||
ready_chunks: Option<Receiver<std::io::Result<Option<Vec<u8>>>>>,
|
||||
recycled_chunks: Option<Sender<Vec<u8>>>,
|
||||
_schema: PhantomData<T>,
|
||||
}
|
||||
|
||||
impl<T: AlignedSbeSchema> AlignedSbeReader<T> {
|
||||
/// Create a new [`AlignedSbeReader`] from the file at `path`. Chunk size is
|
||||
/// set as a constant in this file, [`DEFAULT_CHUNK_SIZE`].
|
||||
pub fn open(path: impl AsRef<Path>) -> anyhow::Result<Self> {
|
||||
Self::open_with_chunk_size(path, DEFAULT_CHUNK_SIZE)
|
||||
}
|
||||
|
||||
/// Create a new [`AlignedSbeReader`] from the file at `path`, with
|
||||
/// `chunk_size` for chunk size.
|
||||
fn open_with_chunk_size(path: impl AsRef<Path>, chunk_size: usize) -> anyhow::Result<Self> {
|
||||
let file = File::open(path)?;
|
||||
let file_len = file.metadata()?.len();
|
||||
|
||||
Ok(Self {
|
||||
file: Some(file),
|
||||
file_len,
|
||||
chunk_size,
|
||||
buffer: vec![0; chunk_size],
|
||||
file_offset: 0,
|
||||
buffer_pos: 0,
|
||||
eof: false,
|
||||
ready_chunks: None,
|
||||
recycled_chunks: None,
|
||||
_schema: PhantomData,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a task to be passed up to the owner of this reader, so it can run
|
||||
/// chunk prefetching on a separate thread (optionally). When this method is
|
||||
/// run, this consumes certain fields of `self` so they may only be used by
|
||||
/// the prefetch task.
|
||||
pub fn create_prefetch_task(&mut self) -> anyhow::Result<PrefetchTask> {
|
||||
anyhow::ensure!(
|
||||
self.file_offset == 0 && self.buffer_pos == 0,
|
||||
"prefetch must start before reading"
|
||||
);
|
||||
|
||||
let file = self.file.take().context("prefetch already started")?;
|
||||
let file_len = self.file_len;
|
||||
let chunk_size = self.chunk_size;
|
||||
let spare_buffers = [std::mem::take(&mut self.buffer), vec![0; self.chunk_size]];
|
||||
let (ready_chunks_tx, ready_chunks) = bounded(1);
|
||||
let (recycled_chunks, recycled_chunks_rx) = bounded(2);
|
||||
self.ready_chunks = Some(ready_chunks);
|
||||
self.recycled_chunks = Some(recycled_chunks);
|
||||
|
||||
Ok(Box::new(move || {
|
||||
let mut spare_buffers = spare_buffers.into_iter();
|
||||
let mut file_offset = 0u64;
|
||||
|
||||
loop {
|
||||
let Some(end) = file_offset.checked_add(chunk_size as u64) else {
|
||||
return;
|
||||
};
|
||||
if end > file_len {
|
||||
let _ = ready_chunks_tx.send(Ok(None));
|
||||
return;
|
||||
}
|
||||
|
||||
let mut buffer = match spare_buffers.next() {
|
||||
Some(buffer) => buffer,
|
||||
None => match recycled_chunks_rx.recv() {
|
||||
Ok(buffer) => buffer,
|
||||
Err(_) => return,
|
||||
},
|
||||
};
|
||||
if let Err(error) = file.read_exact_at(&mut buffer, file_offset) {
|
||||
let _ = ready_chunks_tx.send(Err(error));
|
||||
return;
|
||||
}
|
||||
if ready_chunks_tx.send(Ok(Some(buffer))).is_err() {
|
||||
return;
|
||||
}
|
||||
file_offset = end;
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
/// Signal to the prefetch task that it should stop running.
|
||||
pub fn signal_stop_prefetch(&mut self) {
|
||||
self.ready_chunks = None;
|
||||
self.recycled_chunks = None;
|
||||
}
|
||||
|
||||
/// Returns whether the next chunk could be loaded or not.
|
||||
fn load_chunk(&mut self) -> anyhow::Result<bool> {
|
||||
if self.eof {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if let Some(ready_chunks) = &self.ready_chunks {
|
||||
if !self.buffer.is_empty() {
|
||||
let buffer = std::mem::take(&mut self.buffer);
|
||||
if let Some(recycled_chunks) = &self.recycled_chunks {
|
||||
let _ = recycled_chunks.send(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
return match ready_chunks
|
||||
.recv()
|
||||
.context("SBE prefetch thread stopped")??
|
||||
{
|
||||
Some(buffer) => {
|
||||
self.buffer = buffer;
|
||||
self.buffer_pos = 0;
|
||||
Ok(true)
|
||||
}
|
||||
None => {
|
||||
self.eof = true;
|
||||
Ok(false)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let end = self
|
||||
.file_offset
|
||||
.checked_add(self.chunk_size as u64)
|
||||
.context("offset overflow")?;
|
||||
if end > self.file_len {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
self.file
|
||||
.as_ref()
|
||||
.context("file moved to prefetch task")?
|
||||
.read_exact_at(&mut self.buffer, self.file_offset)?;
|
||||
|
||||
self.file_offset = end;
|
||||
self.buffer_pos = 0;
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Decode the next message from our chunk buffer.
|
||||
///
|
||||
/// Returns `Ok(None)` once we reach the end of the file, and `Err(...)` if
|
||||
/// there is an erroring reading the file, decoding the next message, etc.
|
||||
pub fn read_msg(&mut self) -> anyhow::Result<Option<T::Message<'_>>> {
|
||||
loop {
|
||||
let header_end = self
|
||||
.buffer_pos
|
||||
.checked_add(HEADER_LENGTH)
|
||||
.context("pos overflow")?;
|
||||
|
||||
if header_end <= self.buffer.len()
|
||||
&& self.buffer[self.buffer_pos..header_end] != [0; HEADER_LENGTH]
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if !self.load_chunk()? {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
// All generated schemas share this header layout:
|
||||
// blockLength, templateId, schemaId, version -- all little-endian u16.
|
||||
let header_end = self.buffer_pos + HEADER_LENGTH;
|
||||
let header = &self.buffer[self.buffer_pos..header_end];
|
||||
let block_length = usize::from(u16::from_le_bytes([header[0], header[1]]));
|
||||
let schema_id = u16::from_le_bytes([header[4], header[5]]);
|
||||
let schema_version = u16::from_le_bytes([header[6], header[7]]);
|
||||
|
||||
anyhow::ensure!(schema_id == T::SCHEMA_ID);
|
||||
anyhow::ensure!(schema_version == T::SCHEMA_VERSION);
|
||||
|
||||
let decoded = T::try_decode(&self.buffer, self.buffer_pos)?;
|
||||
|
||||
anyhow::ensure!(block_length == T::expected_block_length(&decoded)?);
|
||||
|
||||
let message_len = T::encoded_length(&decoded)?;
|
||||
let end = self
|
||||
.buffer_pos
|
||||
.checked_add(message_len)
|
||||
.context("overflow")?;
|
||||
|
||||
self.buffer_pos = end;
|
||||
|
||||
Ok(Some(decoded))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
//! [`AlignedSbeWriter`] writes SBE messages in fixed-size, zero-padded file chunks.
|
||||
|
||||
use std::{fs::File, io::Write, path::Path};
|
||||
|
||||
use super::dtos::SbeEncode;
|
||||
|
||||
// Every schema in `dtos` uses the standard eight-byte SBE message header.
|
||||
const HEADER_LENGTH: usize = 8;
|
||||
|
||||
/// Writes SBE messages to a flat file in fixed size, zero padded chunks.
|
||||
///
|
||||
/// See [`AlignedSbeReader`](crate::util::AlignedSbeReader) for more details on why
|
||||
/// this struct exists.
|
||||
///
|
||||
/// Every chunk reserves enough trailing zeroes for an empty SBE header, so a
|
||||
/// reader can stop decoding at the sentinel without carrying a frame across a
|
||||
/// chunk boundary.
|
||||
///
|
||||
/// Call [`Self::flush`] to write the final buffered chunk before dropping the
|
||||
/// writer.
|
||||
pub struct AlignedSbeWriter {
|
||||
file: File,
|
||||
buf: Vec<u8>,
|
||||
chunk: usize,
|
||||
}
|
||||
|
||||
impl AlignedSbeWriter {
|
||||
/// Create an [`AlignedSbeWriter`], writing to `path` with `chunk` chunk
|
||||
/// size. Clears the file at `path`, if it exists.
|
||||
pub fn create(path: impl AsRef<Path>, chunk: usize) -> anyhow::Result<Self> {
|
||||
anyhow::ensure!(
|
||||
chunk > HEADER_LENGTH,
|
||||
"chunk must leave room for SBE data and an SBE header"
|
||||
);
|
||||
Ok(Self {
|
||||
file: File::create(path)?,
|
||||
buf: Vec::with_capacity(chunk),
|
||||
chunk,
|
||||
})
|
||||
}
|
||||
|
||||
/// Encodes a complete message directly into the chunk buffer. Rotates to a
|
||||
/// new chunk if it does not fit in the existing chunk.
|
||||
pub fn write_msg<T: SbeEncode + ?Sized>(&mut self, message: &T) -> anyhow::Result<()> {
|
||||
let len = message.encoded_length_with_header()?;
|
||||
let payload_limit = self.chunk - HEADER_LENGTH;
|
||||
anyhow::ensure!(len <= payload_limit, "SBE message does not fit in a chunk");
|
||||
if len > payload_limit - self.buf.len() {
|
||||
self.flush()?;
|
||||
}
|
||||
|
||||
let start = self.buf.len();
|
||||
self.buf.resize(start + len, 0);
|
||||
let result = message.encode(&mut self.buf[start..]).and_then(|written| {
|
||||
anyhow::ensure!(
|
||||
written == len,
|
||||
"SBE encoded length mismatch: expected {len} bytes, wrote {written}"
|
||||
);
|
||||
Ok(())
|
||||
});
|
||||
if result.is_err() {
|
||||
self.buf.truncate(start);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Flush the current buffer to disk, and clear it.
|
||||
pub fn flush(&mut self) -> anyhow::Result<()> {
|
||||
if !self.buf.is_empty() {
|
||||
self.buf.resize(self.chunk, 0);
|
||||
self.file.write_all(&self.buf)?;
|
||||
self.buf.clear();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//! Build helpers to auto-generate the rust code for the SBE schemas.
|
||||
|
||||
pub mod sbe_yaml;
|
||||
|
||||
use std::fs;
|
||||
|
||||
use ergo_sbe::miette::{IntoDiagnostic, Result, miette};
|
||||
use ergo_sbe::{ConversionSelector, DomainVarData, GenerationConfig, generate_str_to_out_dir};
|
||||
|
||||
const SCHEMA: &str = "sbe_schemas/engineering_challenge/challenge_orderbook.yaml";
|
||||
const COMMON_TYPES: &str = "sbe_schemas/includes/common-types.xml";
|
||||
|
||||
fn main() -> Result<()> {
|
||||
println!("cargo::rerun-if-changed={SCHEMA}");
|
||||
println!("cargo::rerun-if-changed={COMMON_TYPES}");
|
||||
|
||||
let yaml = fs::read_to_string(SCHEMA).into_diagnostic()?;
|
||||
let schema = sbe_yaml::parse(&yaml).map_err(|error| miette!("{error}"))?;
|
||||
let common_types = fs::read_to_string(COMMON_TYPES).into_diagnostic()?;
|
||||
let xml = schema
|
||||
.to_xml("challenge_orderbook_messages", &common_types)
|
||||
.map_err(|error| miette!("{error}"))?;
|
||||
|
||||
let config = GenerationConfig::new("challenge_orderbook")
|
||||
.with_domain_objects(DomainVarData::Strings)
|
||||
.with_domain_type(
|
||||
ConversionSelector::named_type("UTCTimestampMicrosEncoding"),
|
||||
"chrono::DateTime<chrono::Utc>",
|
||||
);
|
||||
generate_str_to_out_dir(&xml, config)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//! SBE helper types, such as [`SbeEncode`].
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
macro_rules! schemas {
|
||||
($($schema:ident),* $(,)?) => {$(
|
||||
pub mod $schema {
|
||||
#![allow(
|
||||
dead_code,
|
||||
unused_variables,
|
||||
unused_mut,
|
||||
unused_assignments,
|
||||
unused_comparisons,
|
||||
unused_unsafe,
|
||||
clippy::all
|
||||
)]
|
||||
include!(concat!(env!("OUT_DIR"), "/", stringify!($schema), ".rs"));
|
||||
}
|
||||
|
||||
impl $schema::TryFromSbe<$schema::UTCTimestampMicrosEncoding> for DateTime<Utc> {
|
||||
type Error = &'static str;
|
||||
|
||||
fn try_from_sbe(wire: $schema::UTCTimestampMicrosEncoding) -> Result<Self, Self::Error> {
|
||||
i64::try_from(wire.time())
|
||||
.ok()
|
||||
.and_then(DateTime::from_timestamp_micros)
|
||||
.ok_or("UTC microseconds outside chrono's range")
|
||||
}
|
||||
}
|
||||
|
||||
impl $schema::TryToSbe<$schema::UTCTimestampMicrosEncoding> for DateTime<Utc> {
|
||||
type Error = &'static str;
|
||||
|
||||
fn try_to_sbe(&self) -> Result<$schema::UTCTimestampMicrosEncoding, Self::Error> {
|
||||
u64::try_from(self.timestamp_micros())
|
||||
.map($schema::UTCTimestampMicrosEncoding::new)
|
||||
.map_err(|_| "UTC timestamp predates the Unix epoch")
|
||||
}
|
||||
}
|
||||
)*};
|
||||
}
|
||||
|
||||
/// This trait allows us to encode multiple SBE schemas using the same function.
|
||||
pub trait SbeEncode {
|
||||
/// Returns the exact encoded length, including the SBE message header.
|
||||
fn encoded_length_with_header(&self) -> anyhow::Result<usize>;
|
||||
|
||||
/// Encodes the message, including its header, into `buf` and returns the bytes written.
|
||||
/// The buffer must be at least `encoded_length_with_header()` bytes long.
|
||||
/// An encoding error may leave the buffer partially written.
|
||||
fn encode(&self, buf: &mut [u8]) -> anyhow::Result<usize>;
|
||||
|
||||
fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
|
||||
let mut buf = vec![0u8; self.encoded_length_with_header()?];
|
||||
let len = self.encode(&mut buf)?;
|
||||
debug_assert_eq!(len, buf.len());
|
||||
Ok(buf)
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! sbe_encode {
|
||||
($($message:ty),* $(,)?) => {$(
|
||||
impl SbeEncode for $message {
|
||||
#[inline]
|
||||
fn encoded_length_with_header(&self) -> anyhow::Result<usize> {
|
||||
Ok(<$message>::encoded_length_with_header(self)?)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn encode(&self, buf: &mut [u8]) -> anyhow::Result<usize> {
|
||||
Ok(<$message>::encode(self, buf)?)
|
||||
}
|
||||
}
|
||||
)*};
|
||||
}
|
||||
|
||||
schemas!(challenge_orderbook);
|
||||
|
||||
sbe_encode!(
|
||||
challenge_orderbook::SnapshotDomain,
|
||||
challenge_orderbook::PriceLevelUpdateDomain,
|
||||
);
|
||||
@@ -0,0 +1,8 @@
|
||||
mod aligned_sbe_reader;
|
||||
mod aligned_sbe_writer;
|
||||
pub mod dtos;
|
||||
mod orderbook_schema;
|
||||
|
||||
pub use aligned_sbe_reader::{AlignedSbeReader, AlignedSbeSchema, PrefetchTask};
|
||||
pub use aligned_sbe_writer::AlignedSbeWriter;
|
||||
pub use orderbook_schema::OrderbookSchema;
|
||||
@@ -0,0 +1,29 @@
|
||||
use super::AlignedSbeSchema;
|
||||
use super::dtos::challenge_orderbook::{
|
||||
AnyMessage, PriceLevelUpdateSchema, SCHEMA_ID, SCHEMA_VERSION, SnapshotSchema,
|
||||
};
|
||||
|
||||
pub struct OrderbookSchema;
|
||||
|
||||
impl AlignedSbeSchema for OrderbookSchema {
|
||||
const SCHEMA_ID: u16 = SCHEMA_ID;
|
||||
const SCHEMA_VERSION: u16 = SCHEMA_VERSION;
|
||||
|
||||
type Message<'buf> = AnyMessage<'buf>;
|
||||
|
||||
fn try_decode<'buf>(buffer: &'buf [u8], offset: usize) -> anyhow::Result<Self::Message<'buf>> {
|
||||
Ok(AnyMessage::try_decode(buffer, offset)?)
|
||||
}
|
||||
|
||||
fn encoded_length(message: &Self::Message<'_>) -> anyhow::Result<usize> {
|
||||
Ok(message.encoded_length_with_header()?)
|
||||
}
|
||||
|
||||
fn expected_block_length(message: &Self::Message<'_>) -> anyhow::Result<usize> {
|
||||
Ok(match message {
|
||||
AnyMessage::Snapshot(_) => SnapshotSchema::BLOCK_LENGTH,
|
||||
AnyMessage::PriceLevelUpdate(_) => PriceLevelUpdateSchema::BLOCK_LENGTH,
|
||||
_ => anyhow::bail!("unknown message"),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
//! Custom transpiler from yaml to FIX SBE XML.
|
||||
|
||||
use std::fmt::{self, Write as _};
|
||||
use std::marker::PhantomData;
|
||||
use std::ops::Deref;
|
||||
|
||||
use indexmap::IndexMap;
|
||||
use serde::Deserialize;
|
||||
use serde::de::{Deserializer, Error as _, MapAccess, Visitor};
|
||||
|
||||
pub type Error = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Unique<V>(IndexMap<String, V>);
|
||||
|
||||
impl<V> Default for Unique<V> {
|
||||
fn default() -> Self {
|
||||
Self(IndexMap::new())
|
||||
}
|
||||
}
|
||||
|
||||
impl<V> Deref for Unique<V> {
|
||||
type Target = IndexMap<String, V>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de, V: Deserialize<'de>> Deserialize<'de> for Unique<V> {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct UniqueVisitor<V>(PhantomData<V>);
|
||||
|
||||
impl<'de, V: Deserialize<'de>> Visitor<'de> for UniqueVisitor<V> {
|
||||
type Value = Unique<V>;
|
||||
|
||||
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.write_str("a mapping with unique keys")
|
||||
}
|
||||
|
||||
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Self::Value, A::Error> {
|
||||
let mut entries = IndexMap::new();
|
||||
while let Some((key, value)) = map.next_entry::<String, V>()? {
|
||||
if entries.insert(key.clone(), value).is_some() {
|
||||
return Err(A::Error::custom(format!("duplicate key `{key}`")));
|
||||
}
|
||||
}
|
||||
Ok(Unique(entries))
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_map(UniqueVisitor(PhantomData))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Schema {
|
||||
pub id: u16,
|
||||
pub version: u16,
|
||||
pub description: Option<String>,
|
||||
#[serde(default)]
|
||||
pub imports: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub enums: Unique<Unique<u8>>,
|
||||
#[serde(default)]
|
||||
pub composites: Unique<Composite>,
|
||||
pub messages: Unique<Message>,
|
||||
}
|
||||
|
||||
/// A shared YAML file contains type definitions, without a schema ID or messages.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SharedTypes {
|
||||
#[serde(default)]
|
||||
pub enums: Unique<Unique<u8>>,
|
||||
#[serde(default)]
|
||||
pub composites: Unique<Composite>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Composite {
|
||||
pub description: Option<String>,
|
||||
pub fields: Unique<CompositeMember>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CompositeMember {
|
||||
/// Only used when a message expands this composite with `fields_from`.
|
||||
pub id: Option<u16>,
|
||||
pub r#type: String,
|
||||
#[serde(default)]
|
||||
pub optional: bool,
|
||||
pub since: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Message {
|
||||
pub id: u16,
|
||||
pub description: Option<String>,
|
||||
pub fields: Option<Unique<Member>>,
|
||||
/// Expand a shared composite as ordinary fields, preserving flat message APIs.
|
||||
pub fields_from: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Member {
|
||||
pub id: u16,
|
||||
/// Documentation only; emitted as an XML attribute without changing the wire layout.
|
||||
pub description: Option<String>,
|
||||
pub r#type: Option<String>,
|
||||
pub fields: Option<Unique<Member>>,
|
||||
#[serde(default)]
|
||||
pub optional: bool,
|
||||
pub since: Option<u16>,
|
||||
}
|
||||
|
||||
pub fn parse(yaml: &str) -> Result<Schema, Error> {
|
||||
Ok(serde_yaml_ng::from_str(yaml)?)
|
||||
}
|
||||
|
||||
pub fn parse_shared_types(yaml: &str) -> Result<SharedTypes, Error> {
|
||||
Ok(serde_yaml_ng::from_str(yaml)?)
|
||||
}
|
||||
|
||||
impl Schema {
|
||||
pub fn import_types(&mut self, types: SharedTypes) -> Result<(), Error> {
|
||||
for name in types.enums.keys().chain(types.composites.keys()) {
|
||||
if self.enums.contains_key(name) || self.composites.contains_key(name) {
|
||||
return Err(format!("duplicate type `{name}`").into());
|
||||
}
|
||||
}
|
||||
self.enums.0.extend(types.enums.0);
|
||||
self.composites.0.extend(types.composites.0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn to_xml(&self, package: &str, common_types: &str) -> Result<String, Error> {
|
||||
let mut xml = String::new();
|
||||
writeln!(xml, r#"<?xml version="1.0" encoding="UTF-8"?>"#)?;
|
||||
writeln!(
|
||||
xml,
|
||||
r#"<sbe:messageSchema xmlns:sbe="http://fixprotocol.io/2016/sbe""#
|
||||
)?;
|
||||
writeln!(xml, r#" package="{}""#, escape(package))?;
|
||||
writeln!(xml, r#" id="{}""#, self.id)?;
|
||||
writeln!(xml, r#" version="{}""#, self.version)?;
|
||||
write!(xml, r#" byteOrder="littleEndian""#)?;
|
||||
if let Some(description) = &self.description {
|
||||
write!(
|
||||
xml,
|
||||
"\n description=\"{}\"",
|
||||
escape(description)
|
||||
)?;
|
||||
}
|
||||
writeln!(xml, ">")?;
|
||||
writeln!(xml, "{}", types_element(common_types)?)?;
|
||||
if !self.enums.is_empty() || !self.composites.is_empty() {
|
||||
writeln!(xml, " <types>")?;
|
||||
for (name, values) in self.enums.iter() {
|
||||
writeln!(xml, r#" <enum name="{name}" encodingType="uint8">"#)?;
|
||||
for (value, number) in values.iter() {
|
||||
writeln!(
|
||||
xml,
|
||||
r#" <validValue name="{value}">{number}</validValue>"#
|
||||
)?;
|
||||
}
|
||||
writeln!(xml, " </enum>")?;
|
||||
}
|
||||
for (name, composite) in self.composites.iter() {
|
||||
if self.enums.contains_key(name) {
|
||||
return Err(format!("duplicate type `{name}`").into());
|
||||
}
|
||||
if composite.fields.is_empty() {
|
||||
return Err(format!("composite `{name}` must have fields").into());
|
||||
}
|
||||
write!(xml, r#" <composite name="{name}""#)?;
|
||||
if let Some(description) = &composite.description {
|
||||
write!(xml, r#" description="{}""#, escape(description))?;
|
||||
}
|
||||
writeln!(xml, ">")?;
|
||||
for (field, member) in composite.fields.iter() {
|
||||
let path = format!("{name}.{field}");
|
||||
let (kind, presence) = composite_member_type(&path, member)?;
|
||||
let since = since_attribute(member.since);
|
||||
if PRIMITIVES.contains(&kind) {
|
||||
writeln!(
|
||||
xml,
|
||||
r#" <type name="{field}" primitiveType="{kind}"{presence}{since}/>"#
|
||||
)?;
|
||||
} else {
|
||||
writeln!(xml, r#" <ref name="{field}" type="{kind}"{since}/>"#)?;
|
||||
}
|
||||
}
|
||||
writeln!(xml, " </composite>")?;
|
||||
}
|
||||
writeln!(xml, " </types>")?;
|
||||
}
|
||||
for (name, message) in self.messages.iter() {
|
||||
write!(xml, r#" <sbe:message name="{name}" id="{}""#, message.id)?;
|
||||
if let Some(description) = &message.description {
|
||||
write!(xml, r#" description="{}""#, escape(description))?;
|
||||
}
|
||||
writeln!(xml, ">")?;
|
||||
match (&message.fields, &message.fields_from) {
|
||||
(Some(fields), None) => write_members(&mut xml, name, fields, 2)?,
|
||||
(None, Some(composite_name)) => {
|
||||
let composite = self.composites.get(composite_name).ok_or_else(|| {
|
||||
format!("{name}: unknown composite `{composite_name}` in `fields_from`")
|
||||
})?;
|
||||
for (field, member) in composite.fields.iter() {
|
||||
let path = format!("{name}.{field}");
|
||||
let id = member.id.ok_or_else(|| {
|
||||
format!(
|
||||
"{path}: `fields_from` requires an `id` for each composite member"
|
||||
)
|
||||
})?;
|
||||
let (kind, presence) = composite_member_type(&path, member)?;
|
||||
let since = since_attribute(member.since);
|
||||
writeln!(
|
||||
xml,
|
||||
r#" <field name="{field}" id="{id}" type="{kind}"{presence}{since}/>"#
|
||||
)?;
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(
|
||||
format!("{name}: needs exactly one of `fields` or `fields_from`").into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
writeln!(xml, " </sbe:message>")?;
|
||||
}
|
||||
writeln!(xml, "</sbe:messageSchema>")?;
|
||||
Ok(xml)
|
||||
}
|
||||
}
|
||||
|
||||
const PRIMITIVES: &[&str] = &[
|
||||
"char", "uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64", "float",
|
||||
"double",
|
||||
];
|
||||
|
||||
fn since_attribute(version: Option<u16>) -> String {
|
||||
version
|
||||
.map(|v| format!(r#" sinceVersion="{v}""#))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn composite_member_type<'a>(
|
||||
path: &str,
|
||||
member: &'a CompositeMember,
|
||||
) -> Result<(&'a str, &'static str), Error> {
|
||||
if matches!(member.r#type.as_str(), "string" | "bytes") {
|
||||
return Err(format!("{path}: composites require fixed-size fields").into());
|
||||
}
|
||||
field_type(path, &member.r#type, member.optional)
|
||||
}
|
||||
|
||||
fn field_type<'a>(
|
||||
path: &str,
|
||||
kind: &'a str,
|
||||
optional: bool,
|
||||
) -> Result<(&'a str, &'static str), Error> {
|
||||
Ok(match (kind, optional) {
|
||||
("bool", false) => ("BooleanType", ""),
|
||||
("timestamp", false) => ("UTCTimestampMicrosEncoding", ""),
|
||||
("timestamp", true) => ("UTCTimestampMicrosNullableEncoding", ""),
|
||||
(kind, true) if NULLABLE_PRIMITIVES.contains(&kind) => (kind, r#" presence="optional""#),
|
||||
(kind, true) => {
|
||||
return Err(format!(
|
||||
"{path}: `{kind}` has no null to spare, so `optional` would be \
|
||||
silently dropped. Enums and `bool` are already nullable — match \
|
||||
on `NullVal`; for anything else use a 0-or-1 group."
|
||||
)
|
||||
.into());
|
||||
}
|
||||
(kind, false) => (kind, ""),
|
||||
})
|
||||
}
|
||||
|
||||
/// SBE reserves the max value of these as the null, so `optional` works as-is.
|
||||
const NULLABLE_PRIMITIVES: &[&str] = &[
|
||||
"uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64", "float", "double",
|
||||
];
|
||||
|
||||
fn write_members(
|
||||
xml: &mut String,
|
||||
path: &str,
|
||||
members: &Unique<Member>,
|
||||
depth: usize,
|
||||
) -> Result<(), Error> {
|
||||
let indent = " ".repeat(depth);
|
||||
for (name, member) in members.iter() {
|
||||
let path = format!("{path}.{name}");
|
||||
let id = member.id;
|
||||
let since = since_attribute(member.since);
|
||||
let description = member
|
||||
.description
|
||||
.as_ref()
|
||||
.map(|text| format!(r#" description="{}""#, escape(text)))
|
||||
.unwrap_or_default();
|
||||
match (member.r#type.as_deref(), &member.fields) {
|
||||
(Some(_), Some(_)) | (None, None) => {
|
||||
return Err(format!("{path}: needs exactly one of `type` or `fields`").into());
|
||||
}
|
||||
(None, Some(fields)) => {
|
||||
if member.optional {
|
||||
return Err(format!("{path}: a group cannot be `optional`").into());
|
||||
}
|
||||
writeln!(
|
||||
xml,
|
||||
r#"{indent}<group name="{name}" id="{id}" dimensionType="groupSizeEncoding"{since}{description}>"#
|
||||
)?;
|
||||
write_members(xml, &path, fields, depth + 1)?;
|
||||
writeln!(xml, "{indent}</group>")?;
|
||||
}
|
||||
(Some("string"), None) => {
|
||||
if member.optional {
|
||||
return Err(format!("{path}: a string cannot be `optional`").into());
|
||||
}
|
||||
writeln!(
|
||||
xml,
|
||||
r#"{indent}<data name="{name}" id="{id}" type="varStringEncoding"{since}{description}/>"#
|
||||
)?;
|
||||
}
|
||||
(Some(kind), None) => {
|
||||
// `optional` is only ever a reserved value in the encoding, so it
|
||||
// is spelled differently per type: primitives get SBE's own null
|
||||
// (the type's max), a timestamp swaps to the nullable composite,
|
||||
// and anything else has nowhere to put one.
|
||||
let (kind, presence) = field_type(&path, kind, member.optional)?;
|
||||
writeln!(
|
||||
xml,
|
||||
r#"{indent}<field name="{name}" id="{id}" type="{kind}"{presence}{since}{description}/>"#
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn types_element(xml: &str) -> Result<&str, Error> {
|
||||
let start = xml
|
||||
.find("<types>")
|
||||
.ok_or("common types must be a <types> element")?;
|
||||
let end = xml
|
||||
.rfind("</types>")
|
||||
.ok_or("common types must be a <types> element")?;
|
||||
Ok(&xml[start..end + "</types>".len()])
|
||||
}
|
||||
|
||||
fn escape(text: &str) -> String {
|
||||
text.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('>', ">")
|
||||
.replace('"', """)
|
||||
}
|
||||
Reference in new issue
Block a user