init commit.

This commit is contained in:
Calder White committed 2026-10-22 00:07:38 -04:00
commit c24decd482
17 files changed
+2211

No files matched your search

+1
View File
@@ -0,0 +1 @@
pub mod util;
+64
View File
@@ -0,0 +1,64 @@
use chrono::DateTime;
use engineering_challenge::util::dtos::challenge_orderbook::{
AnyMessage, PriceLevelUpdateDomain, SnapshotAsksEntryDomain, SnapshotBidsEntryDomain,
SnapshotDomain,
};
use engineering_challenge::util::{AlignedSbeReader, AlignedSbeWriter, OrderbookSchema};
const CHUNK_SIZE_64_MEBIBYTES: usize = 64 << 20;
/// This is a dummy example where we define a snapshot and price level update,
/// write them to a file using [`AlignedSbeWriter`], and then read them back
/// from the same file using [`AlignedSbeReader`], asserting that what we read
/// back is the exact same as what we wrote.
///
/// You should replace this main function with your actual solution, and split
/// up your code into modules in `src/` in a way that makes sense to you.
fn main() -> anyhow::Result<()> {
let timestamp = DateTime::from_timestamp_micros(1_700_000_000_000_000).unwrap();
let symbol = "EXAMPLE".to_owned();
let snapshot = SnapshotDomain {
received_at: timestamp,
exchange_timestamp: timestamp,
bids: vec![SnapshotBidsEntryDomain {
price_level: 400_000,
quantity: 2_000_000,
}],
asks: vec![SnapshotAsksEntryDomain {
price_level: 600_000,
quantity: 3_500_000,
}],
symbol: symbol.clone(),
};
let update = PriceLevelUpdateDomain {
backtest_timestamp: timestamp,
exchange_timestamp: timestamp,
price_level: 400_000,
quantity: 1_250_000,
symbol: symbol.clone(),
};
let file = tempfile::NamedTempFile::new()?;
let mut writer = AlignedSbeWriter::create(file.path(), CHUNK_SIZE_64_MEBIBYTES)?;
writer.write_msg(&snapshot)?;
writer.write_msg(&update)?;
writer.flush()?;
drop(writer);
let mut reader = AlignedSbeReader::<OrderbookSchema>::open(file.path())?;
let Some(AnyMessage::Snapshot(decoded)) = reader.read_msg()? else {
anyhow::bail!("expected snapshot");
};
assert_eq!(SnapshotDomain::try_from_decoder(decoded)?, snapshot);
let Some(AnyMessage::PriceLevelUpdate(decoded)) = reader.read_msg()? else {
anyhow::bail!("expected price level update");
};
assert_eq!(PriceLevelUpdateDomain::try_from_decoder(decoded)?, update);
assert!(reader.read_msg()?.is_none());
println!("Verification success!");
Ok(())
}
+251
View File
@@ -0,0 +1,251 @@
//! Defines the [`AlignedSbeReader`], which allows us to quickly read SBE
//! messages written by the
//! [`AlignedSbeWriter`](crate::util::aligned_sbe_writer::AlignedSbeWriter)
use std::{fs::File, marker::PhantomData, os::unix::fs::FileExt, path::Path};
use anyhow::Context;
use crossbeam::channel::{Receiver, Sender, bounded};
/// Any SBEs we want to read with the AlignedSbeReader must implement this
/// trait.
pub trait AlignedSbeSchema: 'static {
const SCHEMA_ID: u16;
const SCHEMA_VERSION: u16;
type Message<'buf>
where
Self: 'buf;
fn try_decode<'buf>(buffer: &'buf [u8], offset: usize) -> anyhow::Result<Self::Message<'buf>>;
fn encoded_length(message: &Self::Message<'_>) -> anyhow::Result<usize>;
fn expected_block_length(message: &Self::Message<'_>) -> anyhow::Result<usize>;
}
const DEFAULT_CHUNK_SIZE: usize = 64 << 20; // 64MiB
// Every schema in `dtos` uses the standard eight-byte SBE message header.
const HEADER_LENGTH: usize = 8;
/// See [`AlignedSbeReader::create_prefetch_task`] for more details.
pub type PrefetchTask = Box<dyn FnOnce() + Send + 'static>;
/// The [`AlignedSbeReader`] is designed to solve the issue of reading partial
/// messages from SBE flat files. Some background:
///
/// We want to read aeron-spec SBEs from disk at extremely fast speeds -- 100M
/// messages/sec on a single reader thread (with another prefetch thread/task in
/// the background).
///
/// Additionally, we wanted a very simple API to do so. To achieve this, we
/// prefetch our reads, and read in big chunks ([`DEFAULT_CHUNK_SIZE`]). However, when
/// you read in chunks, messages might tear across the chunk boundary. This is
/// undesireable, because now we need to coordinate our prefetch with our
/// reader, in case we need to read the next chunk to finish the currenet
/// message that is truncated on the current chunk.
///
/// This is annoying AF, so to solve the problem, we moved upstream: **Write the
/// encoded SBE messages all-or-nothing, aligned to the chunk size.** If we are
/// writing messages and we find that the next message will tear across two
/// chunks, we simply pad the rest of the current chunk with zeros, and write
/// the message, in full, starting at the next chunk.
///
/// This way the reader can then see the zeros, know the chunk is done, and skip
/// to the next one.
pub struct AlignedSbeReader<T: AlignedSbeSchema> {
file: Option<File>,
file_len: u64,
chunk_size: usize,
buffer: Vec<u8>,
file_offset: u64,
buffer_pos: usize,
eof: bool,
ready_chunks: Option<Receiver<std::io::Result<Option<Vec<u8>>>>>,
recycled_chunks: Option<Sender<Vec<u8>>>,
_schema: PhantomData<T>,
}
impl<T: AlignedSbeSchema> AlignedSbeReader<T> {
/// Create a new [`AlignedSbeReader`] from the file at `path`. Chunk size is
/// set as a constant in this file, [`DEFAULT_CHUNK_SIZE`].
pub fn open(path: impl AsRef<Path>) -> anyhow::Result<Self> {
Self::open_with_chunk_size(path, DEFAULT_CHUNK_SIZE)
}
/// Create a new [`AlignedSbeReader`] from the file at `path`, with
/// `chunk_size` for chunk size.
fn open_with_chunk_size(path: impl AsRef<Path>, chunk_size: usize) -> anyhow::Result<Self> {
let file = File::open(path)?;
let file_len = file.metadata()?.len();
Ok(Self {
file: Some(file),
file_len,
chunk_size,
buffer: vec![0; chunk_size],
file_offset: 0,
buffer_pos: 0,
eof: false,
ready_chunks: None,
recycled_chunks: None,
_schema: PhantomData,
})
}
/// Create a task to be passed up to the owner of this reader, so it can run
/// chunk prefetching on a separate thread (optionally). When this method is
/// run, this consumes certain fields of `self` so they may only be used by
/// the prefetch task.
pub fn create_prefetch_task(&mut self) -> anyhow::Result<PrefetchTask> {
anyhow::ensure!(
self.file_offset == 0 && self.buffer_pos == 0,
"prefetch must start before reading"
);
let file = self.file.take().context("prefetch already started")?;
let file_len = self.file_len;
let chunk_size = self.chunk_size;
let spare_buffers = [std::mem::take(&mut self.buffer), vec![0; self.chunk_size]];
let (ready_chunks_tx, ready_chunks) = bounded(1);
let (recycled_chunks, recycled_chunks_rx) = bounded(2);
self.ready_chunks = Some(ready_chunks);
self.recycled_chunks = Some(recycled_chunks);
Ok(Box::new(move || {
let mut spare_buffers = spare_buffers.into_iter();
let mut file_offset = 0u64;
loop {
let Some(end) = file_offset.checked_add(chunk_size as u64) else {
return;
};
if end > file_len {
let _ = ready_chunks_tx.send(Ok(None));
return;
}
let mut buffer = match spare_buffers.next() {
Some(buffer) => buffer,
None => match recycled_chunks_rx.recv() {
Ok(buffer) => buffer,
Err(_) => return,
},
};
if let Err(error) = file.read_exact_at(&mut buffer, file_offset) {
let _ = ready_chunks_tx.send(Err(error));
return;
}
if ready_chunks_tx.send(Ok(Some(buffer))).is_err() {
return;
}
file_offset = end;
}
}))
}
/// Signal to the prefetch task that it should stop running.
pub fn signal_stop_prefetch(&mut self) {
self.ready_chunks = None;
self.recycled_chunks = None;
}
/// Returns whether the next chunk could be loaded or not.
fn load_chunk(&mut self) -> anyhow::Result<bool> {
if self.eof {
return Ok(false);
}
if let Some(ready_chunks) = &self.ready_chunks {
if !self.buffer.is_empty() {
let buffer = std::mem::take(&mut self.buffer);
if let Some(recycled_chunks) = &self.recycled_chunks {
let _ = recycled_chunks.send(buffer);
}
}
return match ready_chunks
.recv()
.context("SBE prefetch thread stopped")??
{
Some(buffer) => {
self.buffer = buffer;
self.buffer_pos = 0;
Ok(true)
}
None => {
self.eof = true;
Ok(false)
}
};
}
let end = self
.file_offset
.checked_add(self.chunk_size as u64)
.context("offset overflow")?;
if end > self.file_len {
return Ok(false);
}
self.file
.as_ref()
.context("file moved to prefetch task")?
.read_exact_at(&mut self.buffer, self.file_offset)?;
self.file_offset = end;
self.buffer_pos = 0;
Ok(true)
}
/// Decode the next message from our chunk buffer.
///
/// Returns `Ok(None)` once we reach the end of the file, and `Err(...)` if
/// there is an erroring reading the file, decoding the next message, etc.
pub fn read_msg(&mut self) -> anyhow::Result<Option<T::Message<'_>>> {
loop {
let header_end = self
.buffer_pos
.checked_add(HEADER_LENGTH)
.context("pos overflow")?;
if header_end <= self.buffer.len()
&& self.buffer[self.buffer_pos..header_end] != [0; HEADER_LENGTH]
{
break;
}
if !self.load_chunk()? {
return Ok(None);
}
}
// All generated schemas share this header layout:
// blockLength, templateId, schemaId, version -- all little-endian u16.
let header_end = self.buffer_pos + HEADER_LENGTH;
let header = &self.buffer[self.buffer_pos..header_end];
let block_length = usize::from(u16::from_le_bytes([header[0], header[1]]));
let schema_id = u16::from_le_bytes([header[4], header[5]]);
let schema_version = u16::from_le_bytes([header[6], header[7]]);
anyhow::ensure!(schema_id == T::SCHEMA_ID);
anyhow::ensure!(schema_version == T::SCHEMA_VERSION);
let decoded = T::try_decode(&self.buffer, self.buffer_pos)?;
anyhow::ensure!(block_length == T::expected_block_length(&decoded)?);
let message_len = T::encoded_length(&decoded)?;
let end = self
.buffer_pos
.checked_add(message_len)
.context("overflow")?;
self.buffer_pos = end;
Ok(Some(decoded))
}
}
+76
View File
@@ -0,0 +1,76 @@
//! [`AlignedSbeWriter`] writes SBE messages in fixed-size, zero-padded file chunks.
use std::{fs::File, io::Write, path::Path};
use super::dtos::SbeEncode;
// Every schema in `dtos` uses the standard eight-byte SBE message header.
const HEADER_LENGTH: usize = 8;
/// Writes SBE messages to a flat file in fixed size, zero padded chunks.
///
/// See [`AlignedSbeReader`](crate::util::AlignedSbeReader) for more details on why
/// this struct exists.
///
/// Every chunk reserves enough trailing zeroes for an empty SBE header, so a
/// reader can stop decoding at the sentinel without carrying a frame across a
/// chunk boundary.
///
/// Call [`Self::flush`] to write the final buffered chunk before dropping the
/// writer.
pub struct AlignedSbeWriter {
file: File,
buf: Vec<u8>,
chunk: usize,
}
impl AlignedSbeWriter {
/// Create an [`AlignedSbeWriter`], writing to `path` with `chunk` chunk
/// size. Clears the file at `path`, if it exists.
pub fn create(path: impl AsRef<Path>, chunk: usize) -> anyhow::Result<Self> {
anyhow::ensure!(
chunk > HEADER_LENGTH,
"chunk must leave room for SBE data and an SBE header"
);
Ok(Self {
file: File::create(path)?,
buf: Vec::with_capacity(chunk),
chunk,
})
}
/// Encodes a complete message directly into the chunk buffer. Rotates to a
/// new chunk if it does not fit in the existing chunk.
pub fn write_msg<T: SbeEncode + ?Sized>(&mut self, message: &T) -> anyhow::Result<()> {
let len = message.encoded_length_with_header()?;
let payload_limit = self.chunk - HEADER_LENGTH;
anyhow::ensure!(len <= payload_limit, "SBE message does not fit in a chunk");
if len > payload_limit - self.buf.len() {
self.flush()?;
}
let start = self.buf.len();
self.buf.resize(start + len, 0);
let result = message.encode(&mut self.buf[start..]).and_then(|written| {
anyhow::ensure!(
written == len,
"SBE encoded length mismatch: expected {len} bytes, wrote {written}"
);
Ok(())
});
if result.is_err() {
self.buf.truncate(start);
}
result
}
/// Flush the current buffer to disk, and clear it.
pub fn flush(&mut self) -> anyhow::Result<()> {
if !self.buf.is_empty() {
self.buf.resize(self.chunk, 0);
self.file.write_all(&self.buf)?;
self.buf.clear();
}
Ok(())
}
}
+32
View File
@@ -0,0 +1,32 @@
//! Build helpers to auto-generate the rust code for the SBE schemas.
pub mod sbe_yaml;
use std::fs;
use ergo_sbe::miette::{IntoDiagnostic, Result, miette};
use ergo_sbe::{ConversionSelector, DomainVarData, GenerationConfig, generate_str_to_out_dir};
const SCHEMA: &str = "sbe_schemas/engineering_challenge/challenge_orderbook.yaml";
const COMMON_TYPES: &str = "sbe_schemas/includes/common-types.xml";
fn main() -> Result<()> {
println!("cargo::rerun-if-changed={SCHEMA}");
println!("cargo::rerun-if-changed={COMMON_TYPES}");
let yaml = fs::read_to_string(SCHEMA).into_diagnostic()?;
let schema = sbe_yaml::parse(&yaml).map_err(|error| miette!("{error}"))?;
let common_types = fs::read_to_string(COMMON_TYPES).into_diagnostic()?;
let xml = schema
.to_xml("challenge_orderbook_messages", &common_types)
.map_err(|error| miette!("{error}"))?;
let config = GenerationConfig::new("challenge_orderbook")
.with_domain_objects(DomainVarData::Strings)
.with_domain_type(
ConversionSelector::named_type("UTCTimestampMicrosEncoding"),
"chrono::DateTime<chrono::Utc>",
);
generate_str_to_out_dir(&xml, config)?;
Ok(())
}
+82
View File
@@ -0,0 +1,82 @@
//! SBE helper types, such as [`SbeEncode`].
use chrono::{DateTime, Utc};
macro_rules! schemas {
($($schema:ident),* $(,)?) => {$(
pub mod $schema {
#![allow(
dead_code,
unused_variables,
unused_mut,
unused_assignments,
unused_comparisons,
unused_unsafe,
clippy::all
)]
include!(concat!(env!("OUT_DIR"), "/", stringify!($schema), ".rs"));
}
impl $schema::TryFromSbe<$schema::UTCTimestampMicrosEncoding> for DateTime<Utc> {
type Error = &'static str;
fn try_from_sbe(wire: $schema::UTCTimestampMicrosEncoding) -> Result<Self, Self::Error> {
i64::try_from(wire.time())
.ok()
.and_then(DateTime::from_timestamp_micros)
.ok_or("UTC microseconds outside chrono's range")
}
}
impl $schema::TryToSbe<$schema::UTCTimestampMicrosEncoding> for DateTime<Utc> {
type Error = &'static str;
fn try_to_sbe(&self) -> Result<$schema::UTCTimestampMicrosEncoding, Self::Error> {
u64::try_from(self.timestamp_micros())
.map($schema::UTCTimestampMicrosEncoding::new)
.map_err(|_| "UTC timestamp predates the Unix epoch")
}
}
)*};
}
/// This trait allows us to encode multiple SBE schemas using the same function.
pub trait SbeEncode {
/// Returns the exact encoded length, including the SBE message header.
fn encoded_length_with_header(&self) -> anyhow::Result<usize>;
/// Encodes the message, including its header, into `buf` and returns the bytes written.
/// The buffer must be at least `encoded_length_with_header()` bytes long.
/// An encoding error may leave the buffer partially written.
fn encode(&self, buf: &mut [u8]) -> anyhow::Result<usize>;
fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
let mut buf = vec![0u8; self.encoded_length_with_header()?];
let len = self.encode(&mut buf)?;
debug_assert_eq!(len, buf.len());
Ok(buf)
}
}
macro_rules! sbe_encode {
($($message:ty),* $(,)?) => {$(
impl SbeEncode for $message {
#[inline]
fn encoded_length_with_header(&self) -> anyhow::Result<usize> {
Ok(<$message>::encoded_length_with_header(self)?)
}
#[inline]
fn encode(&self, buf: &mut [u8]) -> anyhow::Result<usize> {
Ok(<$message>::encode(self, buf)?)
}
}
)*};
}
schemas!(challenge_orderbook);
sbe_encode!(
challenge_orderbook::SnapshotDomain,
challenge_orderbook::PriceLevelUpdateDomain,
);
+8
View File
@@ -0,0 +1,8 @@
mod aligned_sbe_reader;
mod aligned_sbe_writer;
pub mod dtos;
mod orderbook_schema;
pub use aligned_sbe_reader::{AlignedSbeReader, AlignedSbeSchema, PrefetchTask};
pub use aligned_sbe_writer::AlignedSbeWriter;
pub use orderbook_schema::OrderbookSchema;
+29
View File
@@ -0,0 +1,29 @@
use super::AlignedSbeSchema;
use super::dtos::challenge_orderbook::{
AnyMessage, PriceLevelUpdateSchema, SCHEMA_ID, SCHEMA_VERSION, SnapshotSchema,
};
pub struct OrderbookSchema;
impl AlignedSbeSchema for OrderbookSchema {
const SCHEMA_ID: u16 = SCHEMA_ID;
const SCHEMA_VERSION: u16 = SCHEMA_VERSION;
type Message<'buf> = AnyMessage<'buf>;
fn try_decode<'buf>(buffer: &'buf [u8], offset: usize) -> anyhow::Result<Self::Message<'buf>> {
Ok(AnyMessage::try_decode(buffer, offset)?)
}
fn encoded_length(message: &Self::Message<'_>) -> anyhow::Result<usize> {
Ok(message.encoded_length_with_header()?)
}
fn expected_block_length(message: &Self::Message<'_>) -> anyhow::Result<usize> {
Ok(match message {
AnyMessage::Snapshot(_) => SnapshotSchema::BLOCK_LENGTH,
AnyMessage::PriceLevelUpdate(_) => PriceLevelUpdateSchema::BLOCK_LENGTH,
_ => anyhow::bail!("unknown message"),
})
}
}
+362
View File
@@ -0,0 +1,362 @@
//! Custom transpiler from yaml to FIX SBE XML.
use std::fmt::{self, Write as _};
use std::marker::PhantomData;
use std::ops::Deref;
use indexmap::IndexMap;
use serde::Deserialize;
use serde::de::{Deserializer, Error as _, MapAccess, Visitor};
pub type Error = Box<dyn std::error::Error + Send + Sync>;
#[derive(Debug)]
pub struct Unique<V>(IndexMap<String, V>);
impl<V> Default for Unique<V> {
fn default() -> Self {
Self(IndexMap::new())
}
}
impl<V> Deref for Unique<V> {
type Target = IndexMap<String, V>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<'de, V: Deserialize<'de>> Deserialize<'de> for Unique<V> {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct UniqueVisitor<V>(PhantomData<V>);
impl<'de, V: Deserialize<'de>> Visitor<'de> for UniqueVisitor<V> {
type Value = Unique<V>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("a mapping with unique keys")
}
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Self::Value, A::Error> {
let mut entries = IndexMap::new();
while let Some((key, value)) = map.next_entry::<String, V>()? {
if entries.insert(key.clone(), value).is_some() {
return Err(A::Error::custom(format!("duplicate key `{key}`")));
}
}
Ok(Unique(entries))
}
}
deserializer.deserialize_map(UniqueVisitor(PhantomData))
}
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Schema {
pub id: u16,
pub version: u16,
pub description: Option<String>,
#[serde(default)]
pub imports: Vec<String>,
#[serde(default)]
pub enums: Unique<Unique<u8>>,
#[serde(default)]
pub composites: Unique<Composite>,
pub messages: Unique<Message>,
}
/// A shared YAML file contains type definitions, without a schema ID or messages.
#[derive(Debug, Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SharedTypes {
#[serde(default)]
pub enums: Unique<Unique<u8>>,
#[serde(default)]
pub composites: Unique<Composite>,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Composite {
pub description: Option<String>,
pub fields: Unique<CompositeMember>,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CompositeMember {
/// Only used when a message expands this composite with `fields_from`.
pub id: Option<u16>,
pub r#type: String,
#[serde(default)]
pub optional: bool,
pub since: Option<u16>,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Message {
pub id: u16,
pub description: Option<String>,
pub fields: Option<Unique<Member>>,
/// Expand a shared composite as ordinary fields, preserving flat message APIs.
pub fields_from: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Member {
pub id: u16,
/// Documentation only; emitted as an XML attribute without changing the wire layout.
pub description: Option<String>,
pub r#type: Option<String>,
pub fields: Option<Unique<Member>>,
#[serde(default)]
pub optional: bool,
pub since: Option<u16>,
}
pub fn parse(yaml: &str) -> Result<Schema, Error> {
Ok(serde_yaml_ng::from_str(yaml)?)
}
pub fn parse_shared_types(yaml: &str) -> Result<SharedTypes, Error> {
Ok(serde_yaml_ng::from_str(yaml)?)
}
impl Schema {
pub fn import_types(&mut self, types: SharedTypes) -> Result<(), Error> {
for name in types.enums.keys().chain(types.composites.keys()) {
if self.enums.contains_key(name) || self.composites.contains_key(name) {
return Err(format!("duplicate type `{name}`").into());
}
}
self.enums.0.extend(types.enums.0);
self.composites.0.extend(types.composites.0);
Ok(())
}
pub fn to_xml(&self, package: &str, common_types: &str) -> Result<String, Error> {
let mut xml = String::new();
writeln!(xml, r#"<?xml version="1.0" encoding="UTF-8"?>"#)?;
writeln!(
xml,
r#"<sbe:messageSchema xmlns:sbe="http://fixprotocol.io/2016/sbe""#
)?;
writeln!(xml, r#" package="{}""#, escape(package))?;
writeln!(xml, r#" id="{}""#, self.id)?;
writeln!(xml, r#" version="{}""#, self.version)?;
write!(xml, r#" byteOrder="littleEndian""#)?;
if let Some(description) = &self.description {
write!(
xml,
"\n description=\"{}\"",
escape(description)
)?;
}
writeln!(xml, ">")?;
writeln!(xml, "{}", types_element(common_types)?)?;
if !self.enums.is_empty() || !self.composites.is_empty() {
writeln!(xml, " <types>")?;
for (name, values) in self.enums.iter() {
writeln!(xml, r#" <enum name="{name}" encodingType="uint8">"#)?;
for (value, number) in values.iter() {
writeln!(
xml,
r#" <validValue name="{value}">{number}</validValue>"#
)?;
}
writeln!(xml, " </enum>")?;
}
for (name, composite) in self.composites.iter() {
if self.enums.contains_key(name) {
return Err(format!("duplicate type `{name}`").into());
}
if composite.fields.is_empty() {
return Err(format!("composite `{name}` must have fields").into());
}
write!(xml, r#" <composite name="{name}""#)?;
if let Some(description) = &composite.description {
write!(xml, r#" description="{}""#, escape(description))?;
}
writeln!(xml, ">")?;
for (field, member) in composite.fields.iter() {
let path = format!("{name}.{field}");
let (kind, presence) = composite_member_type(&path, member)?;
let since = since_attribute(member.since);
if PRIMITIVES.contains(&kind) {
writeln!(
xml,
r#" <type name="{field}" primitiveType="{kind}"{presence}{since}/>"#
)?;
} else {
writeln!(xml, r#" <ref name="{field}" type="{kind}"{since}/>"#)?;
}
}
writeln!(xml, " </composite>")?;
}
writeln!(xml, " </types>")?;
}
for (name, message) in self.messages.iter() {
write!(xml, r#" <sbe:message name="{name}" id="{}""#, message.id)?;
if let Some(description) = &message.description {
write!(xml, r#" description="{}""#, escape(description))?;
}
writeln!(xml, ">")?;
match (&message.fields, &message.fields_from) {
(Some(fields), None) => write_members(&mut xml, name, fields, 2)?,
(None, Some(composite_name)) => {
let composite = self.composites.get(composite_name).ok_or_else(|| {
format!("{name}: unknown composite `{composite_name}` in `fields_from`")
})?;
for (field, member) in composite.fields.iter() {
let path = format!("{name}.{field}");
let id = member.id.ok_or_else(|| {
format!(
"{path}: `fields_from` requires an `id` for each composite member"
)
})?;
let (kind, presence) = composite_member_type(&path, member)?;
let since = since_attribute(member.since);
writeln!(
xml,
r#" <field name="{field}" id="{id}" type="{kind}"{presence}{since}/>"#
)?;
}
}
_ => {
return Err(
format!("{name}: needs exactly one of `fields` or `fields_from`").into(),
);
}
}
writeln!(xml, " </sbe:message>")?;
}
writeln!(xml, "</sbe:messageSchema>")?;
Ok(xml)
}
}
const PRIMITIVES: &[&str] = &[
"char", "uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64", "float",
"double",
];
fn since_attribute(version: Option<u16>) -> String {
version
.map(|v| format!(r#" sinceVersion="{v}""#))
.unwrap_or_default()
}
fn composite_member_type<'a>(
path: &str,
member: &'a CompositeMember,
) -> Result<(&'a str, &'static str), Error> {
if matches!(member.r#type.as_str(), "string" | "bytes") {
return Err(format!("{path}: composites require fixed-size fields").into());
}
field_type(path, &member.r#type, member.optional)
}
fn field_type<'a>(
path: &str,
kind: &'a str,
optional: bool,
) -> Result<(&'a str, &'static str), Error> {
Ok(match (kind, optional) {
("bool", false) => ("BooleanType", ""),
("timestamp", false) => ("UTCTimestampMicrosEncoding", ""),
("timestamp", true) => ("UTCTimestampMicrosNullableEncoding", ""),
(kind, true) if NULLABLE_PRIMITIVES.contains(&kind) => (kind, r#" presence="optional""#),
(kind, true) => {
return Err(format!(
"{path}: `{kind}` has no null to spare, so `optional` would be \
silently dropped. Enums and `bool` are already nullable — match \
on `NullVal`; for anything else use a 0-or-1 group."
)
.into());
}
(kind, false) => (kind, ""),
})
}
/// SBE reserves the max value of these as the null, so `optional` works as-is.
const NULLABLE_PRIMITIVES: &[&str] = &[
"uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64", "float", "double",
];
fn write_members(
xml: &mut String,
path: &str,
members: &Unique<Member>,
depth: usize,
) -> Result<(), Error> {
let indent = " ".repeat(depth);
for (name, member) in members.iter() {
let path = format!("{path}.{name}");
let id = member.id;
let since = since_attribute(member.since);
let description = member
.description
.as_ref()
.map(|text| format!(r#" description="{}""#, escape(text)))
.unwrap_or_default();
match (member.r#type.as_deref(), &member.fields) {
(Some(_), Some(_)) | (None, None) => {
return Err(format!("{path}: needs exactly one of `type` or `fields`").into());
}
(None, Some(fields)) => {
if member.optional {
return Err(format!("{path}: a group cannot be `optional`").into());
}
writeln!(
xml,
r#"{indent}<group name="{name}" id="{id}" dimensionType="groupSizeEncoding"{since}{description}>"#
)?;
write_members(xml, &path, fields, depth + 1)?;
writeln!(xml, "{indent}</group>")?;
}
(Some("string"), None) => {
if member.optional {
return Err(format!("{path}: a string cannot be `optional`").into());
}
writeln!(
xml,
r#"{indent}<data name="{name}" id="{id}" type="varStringEncoding"{since}{description}/>"#
)?;
}
(Some(kind), None) => {
// `optional` is only ever a reserved value in the encoding, so it
// is spelled differently per type: primitives get SBE's own null
// (the type's max), a timestamp swaps to the nullable composite,
// and anything else has nowhere to put one.
let (kind, presence) = field_type(&path, kind, member.optional)?;
writeln!(
xml,
r#"{indent}<field name="{name}" id="{id}" type="{kind}"{presence}{since}{description}/>"#
)?;
}
}
}
Ok(())
}
fn types_element(xml: &str) -> Result<&str, Error> {
let start = xml
.find("<types>")
.ok_or("common types must be a <types> element")?;
let end = xml
.rfind("</types>")
.ok_or("common types must be a <types> element")?;
Ok(&xml[start..end + "</types>".len()])
}
fn escape(text: &str) -> String {
text.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
}