Archived
init commit.
This commit is contained in:
commit
c24decd482
17 files changed
+2211
No files matched your search
@@ -0,0 +1 @@
|
|||||||
|
/target/
|
||||||
Vendored
+12
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"recommendations": [
|
||||||
|
"rust-lang.rust-analyzer",
|
||||||
|
"tamasfe.even-better-toml",
|
||||||
|
"redhat.vscode-xml",
|
||||||
|
"hangxingliu.vscode-systemd-support",
|
||||||
|
"hashicorp.terraform",
|
||||||
|
"redhat.ansible",
|
||||||
|
"redhat.vscode-yaml",
|
||||||
|
"mkhl.shfmt"
|
||||||
|
]
|
||||||
|
}
|
||||||
Vendored
+49
@@ -0,0 +1,49 @@
|
|||||||
|
{
|
||||||
|
"editor.formatOnSave": true,
|
||||||
|
"[rust]": {
|
||||||
|
"editor.defaultFormatter": "rust-lang.rust-analyzer"
|
||||||
|
},
|
||||||
|
"[toml]": {
|
||||||
|
"editor.defaultFormatter": "tamasfe.even-better-toml",
|
||||||
|
"editor.formatOnSave": true
|
||||||
|
},
|
||||||
|
"[xml]": {
|
||||||
|
"editor.defaultFormatter": "redhat.vscode-xml",
|
||||||
|
"editor.tabSize": 2,
|
||||||
|
"editor.insertSpaces": true,
|
||||||
|
"editor.detectIndentation": false
|
||||||
|
},
|
||||||
|
"xml.format.preserveAttributeLineBreaks": true,
|
||||||
|
"[terraform]": {
|
||||||
|
"editor.defaultFormatter": "hashicorp.terraform",
|
||||||
|
"editor.formatOnSave": true
|
||||||
|
},
|
||||||
|
"[terraform-vars]": {
|
||||||
|
"editor.defaultFormatter": "hashicorp.terraform",
|
||||||
|
"editor.formatOnSave": true
|
||||||
|
},
|
||||||
|
"[ansible]": {
|
||||||
|
"editor.defaultFormatter": "redhat.vscode-yaml",
|
||||||
|
"editor.formatOnSave": true,
|
||||||
|
"editor.insertSpaces": true,
|
||||||
|
"editor.tabSize": 2,
|
||||||
|
"yaml.format.enable": true
|
||||||
|
},
|
||||||
|
"[shellscript]": {
|
||||||
|
"editor.defaultFormatter": "mkhl.shfmt",
|
||||||
|
"editor.formatOnSave": true,
|
||||||
|
"editor.insertSpaces": true,
|
||||||
|
"editor.tabSize": 2
|
||||||
|
},
|
||||||
|
"files.associations": {
|
||||||
|
// *.tfvars.example doesn't match the extension's *.tfvars glob
|
||||||
|
"*.tfvars.example": "terraform-vars",
|
||||||
|
"**/.bashrc": "shellscript",
|
||||||
|
"infra/ansible/**/*.yml": "ansible",
|
||||||
|
"infra/ansible/**/*.yaml": "ansible"
|
||||||
|
},
|
||||||
|
"files.watcherExclude": {
|
||||||
|
"**/target/**": true,
|
||||||
|
"**/.terraform/**": true
|
||||||
|
}
|
||||||
|
}
|
||||||
Generated
+1099
File diff suppressed because it is too large.
Load diff
+17
@@ -0,0 +1,17 @@
|
|||||||
|
[package]
|
||||||
|
name = "engineering-challenge"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
build = "src/util/build.rs"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
anyhow = "1"
|
||||||
|
chrono = "0.4"
|
||||||
|
crossbeam = "0.8.4"
|
||||||
|
tempfile = "3"
|
||||||
|
|
||||||
|
[build-dependencies]
|
||||||
|
ergo-sbe = "=0.1.16"
|
||||||
|
indexmap = { version = "2", features = ["serde"] }
|
||||||
|
serde = { version = "1", features = ["derive"] }
|
||||||
|
serde_yaml_ng = "0.9"
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# Read before getting started!!
|
||||||
|
|
||||||
|
First, read `src/main.rs`. Then, some background on this repo:
|
||||||
|
|
||||||
|
This repo uses the [FIX SBE standard](https://fixtrading.org/packages/simple-binary-encoding-technical-specification-v2-rc2/)
|
||||||
|
for writing binary messages. The rust code abstracts this away for you, and you can simply use the helpers, as demonstrated in `main.rs`.
|
||||||
|
|
||||||
|
### More background, in case you are curious!
|
||||||
|
|
||||||
|
FIX SBE is defined in XML, which I find extremely cumbersome to edit, especially
|
||||||
|
when updating schemas frequently. To solve this, I have also introduced some
|
||||||
|
helper code that takes our own custom YAML format, and turns it into FIX SBE
|
||||||
|
XML, which in turn is used to generate rust code to encode and decode the
|
||||||
|
messages. **You don't need to edit any of these files**. This explanation is
|
||||||
|
simply to help you understand the origins of the various files in this repo.
|
||||||
|
|
||||||
|
The yaml for the schema you will be using is located in
|
||||||
|
`sbe_schemas/engineering_challenge/challenge_orderbook.yaml`.
|
||||||
|
We also use a custom timestamp field, so that the Rust struct can use
|
||||||
|
`chrono::DateTime` in the generated struct, and in SBE schema, it is a uint64
|
||||||
|
representing the timestamp as the number of microseconds since unix epoch.
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
id: 1
|
||||||
|
version: 1
|
||||||
|
description: Voluntas engineering challenge orderbook update schema.
|
||||||
|
|
||||||
|
messages:
|
||||||
|
Snapshot:
|
||||||
|
id: 1
|
||||||
|
description: A full snapshot of the orderbook
|
||||||
|
fields:
|
||||||
|
received_at: { id: 1, type: timestamp }
|
||||||
|
exchange_timestamp: { id: 2, type: timestamp }
|
||||||
|
bids:
|
||||||
|
id: 4
|
||||||
|
description: Bid price levels
|
||||||
|
fields:
|
||||||
|
price_level:
|
||||||
|
id: 5
|
||||||
|
type: int64
|
||||||
|
description: The price level, in fixed point; 1_000_000 = 1 USDC
|
||||||
|
quantity:
|
||||||
|
id: 6
|
||||||
|
type: int64
|
||||||
|
description: The absolute available quantity, in fixed point; 1_000_000 = 1 share/contract
|
||||||
|
asks:
|
||||||
|
id: 7
|
||||||
|
description: Ask price levels
|
||||||
|
fields:
|
||||||
|
price_level:
|
||||||
|
id: 8
|
||||||
|
type: int64
|
||||||
|
description: The price level, in fixed point; 1_000_000 = 1 USDC
|
||||||
|
quantity:
|
||||||
|
id: 9
|
||||||
|
type: int64
|
||||||
|
description: The absolute available quantity, in fixed point; 1_000_000 = 1 share/contract
|
||||||
|
symbol: { id: 3, type: string }
|
||||||
|
|
||||||
|
PriceLevelUpdate:
|
||||||
|
id: 2
|
||||||
|
description: An update to a single price level, on a single orderbook
|
||||||
|
fields:
|
||||||
|
backtest_timestamp: { id: 1, type: timestamp }
|
||||||
|
exchange_timestamp: { id: 2, type: timestamp }
|
||||||
|
price_level:
|
||||||
|
id: 4
|
||||||
|
type: int64
|
||||||
|
description: The price level being updated, in fixed point; 1_000_000 = 1 USDC
|
||||||
|
quantity:
|
||||||
|
id: 5
|
||||||
|
type: int64
|
||||||
|
description: The absolute available quantity, in fixed point; 1_000_000 = 1 share/contract
|
||||||
|
symbol: { id: 3, type: string }
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Shared SBE wire encodings. These definitions are included by every schema so
|
||||||
|
message files contain only schema-specific types and message declarations.
|
||||||
|
|
||||||
|
Changing an existing encoding changes every schema that uses it. Add a new
|
||||||
|
named encoding instead when a different wire representation is required.
|
||||||
|
-->
|
||||||
|
<types>
|
||||||
|
<!-- Standard 8-byte SBE message header. These names are required by the tooling. -->
|
||||||
|
<composite name="messageHeader" description="Message identifiers and length of message root">
|
||||||
|
<type name="blockLength" primitiveType="uint16"/>
|
||||||
|
<type name="templateId" primitiveType="uint16"/>
|
||||||
|
<type name="schemaId" primitiveType="uint16"/>
|
||||||
|
<type name="version" primitiveType="uint16"/>
|
||||||
|
</composite>
|
||||||
|
|
||||||
|
<!-- Repeating-group dimensions written before each group instance. -->
|
||||||
|
<composite name="groupSizeEncoding" description="Repeating group dimensions">
|
||||||
|
<type name="blockLength" primitiveType="uint16"/>
|
||||||
|
<type name="numInGroup" primitiveType="uint16"/>
|
||||||
|
</composite>
|
||||||
|
|
||||||
|
<!-- Variable-length UTF-8 string with a uint32 byte-length prefix. -->
|
||||||
|
<composite name="varStringEncoding" description="Variable-length UTF-8 string">
|
||||||
|
<type name="length" primitiveType="uint32" maxValue="1073741824"/>
|
||||||
|
<type name="varData" primitiveType="uint8" length="0" characterEncoding="UTF-8"/>
|
||||||
|
</composite>
|
||||||
|
|
||||||
|
<!-- UTC microseconds since the UNIX epoch; the constant unit occupies no wire bytes. -->
|
||||||
|
<composite name="UTCTimestampMicrosEncoding" semanticType="UTCTimestamp"
|
||||||
|
description="UTC microseconds since the UNIX epoch">
|
||||||
|
<type name="time" primitiveType="uint64"/>
|
||||||
|
<type name="unit" primitiveType="uint8" presence="constant">6</type>
|
||||||
|
</composite>
|
||||||
|
|
||||||
|
<!-- Same 8 bytes, but u64::MAX is reserved to mean "unknown". SBE has no
|
||||||
|
presence="optional" for a composite as a whole, so the nullability rides
|
||||||
|
on `time` and you test that member — the pattern real-logic points at in
|
||||||
|
issue #509. Picked by `type: timestamp, optional: true` in the yaml. -->
|
||||||
|
<composite name="UTCTimestampMicrosNullableEncoding" semanticType="UTCTimestamp"
|
||||||
|
description="UTC microseconds since the UNIX epoch, or null">
|
||||||
|
<type name="time" primitiveType="uint64" presence="optional"/>
|
||||||
|
<type name="unit" primitiveType="uint8" presence="constant">6</type>
|
||||||
|
</composite>
|
||||||
|
|
||||||
|
<type name="bytes32" primitiveType="uint8" length="32"/>
|
||||||
|
<type name="address" primitiveType="uint8" length="20"/>
|
||||||
|
|
||||||
|
<!-- FIX-standard boolean: one byte on the wire. -->
|
||||||
|
<enum name="BooleanType" encodingType="uint8">
|
||||||
|
<validValue name="False">0</validValue>
|
||||||
|
<validValue name="True">1</validValue>
|
||||||
|
</enum>
|
||||||
|
</types>
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
pub mod util;
|
||||||
+64
@@ -0,0 +1,64 @@
|
|||||||
|
use chrono::DateTime;
|
||||||
|
use engineering_challenge::util::dtos::challenge_orderbook::{
|
||||||
|
AnyMessage, PriceLevelUpdateDomain, SnapshotAsksEntryDomain, SnapshotBidsEntryDomain,
|
||||||
|
SnapshotDomain,
|
||||||
|
};
|
||||||
|
use engineering_challenge::util::{AlignedSbeReader, AlignedSbeWriter, OrderbookSchema};
|
||||||
|
|
||||||
|
const CHUNK_SIZE_64_MEBIBYTES: usize = 64 << 20;
|
||||||
|
|
||||||
|
/// This is a dummy example where we define a snapshot and price level update,
|
||||||
|
/// write them to a file using [`AlignedSbeWriter`], and then read them back
|
||||||
|
/// from the same file using [`AlignedSbeReader`], asserting that what we read
|
||||||
|
/// back is the exact same as what we wrote.
|
||||||
|
///
|
||||||
|
/// You should replace this main function with your actual solution, and split
|
||||||
|
/// up your code into modules in `src/` in a way that makes sense to you.
|
||||||
|
fn main() -> anyhow::Result<()> {
|
||||||
|
let timestamp = DateTime::from_timestamp_micros(1_700_000_000_000_000).unwrap();
|
||||||
|
let symbol = "EXAMPLE".to_owned();
|
||||||
|
let snapshot = SnapshotDomain {
|
||||||
|
received_at: timestamp,
|
||||||
|
exchange_timestamp: timestamp,
|
||||||
|
bids: vec![SnapshotBidsEntryDomain {
|
||||||
|
price_level: 400_000,
|
||||||
|
quantity: 2_000_000,
|
||||||
|
}],
|
||||||
|
asks: vec![SnapshotAsksEntryDomain {
|
||||||
|
price_level: 600_000,
|
||||||
|
quantity: 3_500_000,
|
||||||
|
}],
|
||||||
|
symbol: symbol.clone(),
|
||||||
|
};
|
||||||
|
let update = PriceLevelUpdateDomain {
|
||||||
|
backtest_timestamp: timestamp,
|
||||||
|
exchange_timestamp: timestamp,
|
||||||
|
price_level: 400_000,
|
||||||
|
quantity: 1_250_000,
|
||||||
|
symbol: symbol.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let file = tempfile::NamedTempFile::new()?;
|
||||||
|
let mut writer = AlignedSbeWriter::create(file.path(), CHUNK_SIZE_64_MEBIBYTES)?;
|
||||||
|
writer.write_msg(&snapshot)?;
|
||||||
|
writer.write_msg(&update)?;
|
||||||
|
writer.flush()?;
|
||||||
|
drop(writer);
|
||||||
|
|
||||||
|
let mut reader = AlignedSbeReader::<OrderbookSchema>::open(file.path())?;
|
||||||
|
|
||||||
|
let Some(AnyMessage::Snapshot(decoded)) = reader.read_msg()? else {
|
||||||
|
anyhow::bail!("expected snapshot");
|
||||||
|
};
|
||||||
|
assert_eq!(SnapshotDomain::try_from_decoder(decoded)?, snapshot);
|
||||||
|
|
||||||
|
let Some(AnyMessage::PriceLevelUpdate(decoded)) = reader.read_msg()? else {
|
||||||
|
anyhow::bail!("expected price level update");
|
||||||
|
};
|
||||||
|
assert_eq!(PriceLevelUpdateDomain::try_from_decoder(decoded)?, update);
|
||||||
|
assert!(reader.read_msg()?.is_none());
|
||||||
|
|
||||||
|
println!("Verification success!");
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -0,0 +1,251 @@
|
|||||||
|
//! Defines the [`AlignedSbeReader`], which allows us to quickly read SBE
|
||||||
|
//! messages written by the
|
||||||
|
//! [`AlignedSbeWriter`](crate::util::aligned_sbe_writer::AlignedSbeWriter)
|
||||||
|
|
||||||
|
use std::{fs::File, marker::PhantomData, os::unix::fs::FileExt, path::Path};
|
||||||
|
|
||||||
|
use anyhow::Context;
|
||||||
|
use crossbeam::channel::{Receiver, Sender, bounded};
|
||||||
|
|
||||||
|
/// Any SBEs we want to read with the AlignedSbeReader must implement this
|
||||||
|
/// trait.
|
||||||
|
pub trait AlignedSbeSchema: 'static {
|
||||||
|
const SCHEMA_ID: u16;
|
||||||
|
const SCHEMA_VERSION: u16;
|
||||||
|
|
||||||
|
type Message<'buf>
|
||||||
|
where
|
||||||
|
Self: 'buf;
|
||||||
|
|
||||||
|
fn try_decode<'buf>(buffer: &'buf [u8], offset: usize) -> anyhow::Result<Self::Message<'buf>>;
|
||||||
|
|
||||||
|
fn encoded_length(message: &Self::Message<'_>) -> anyhow::Result<usize>;
|
||||||
|
|
||||||
|
fn expected_block_length(message: &Self::Message<'_>) -> anyhow::Result<usize>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_CHUNK_SIZE: usize = 64 << 20; // 64MiB
|
||||||
|
|
||||||
|
// Every schema in `dtos` uses the standard eight-byte SBE message header.
|
||||||
|
const HEADER_LENGTH: usize = 8;
|
||||||
|
|
||||||
|
/// See [`AlignedSbeReader::create_prefetch_task`] for more details.
|
||||||
|
pub type PrefetchTask = Box<dyn FnOnce() + Send + 'static>;
|
||||||
|
|
||||||
|
/// The [`AlignedSbeReader`] is designed to solve the issue of reading partial
|
||||||
|
/// messages from SBE flat files. Some background:
|
||||||
|
///
|
||||||
|
/// We want to read aeron-spec SBEs from disk at extremely fast speeds -- 100M
|
||||||
|
/// messages/sec on a single reader thread (with another prefetch thread/task in
|
||||||
|
/// the background).
|
||||||
|
///
|
||||||
|
/// Additionally, we wanted a very simple API to do so. To achieve this, we
|
||||||
|
/// prefetch our reads, and read in big chunks ([`DEFAULT_CHUNK_SIZE`]). However, when
|
||||||
|
/// you read in chunks, messages might tear across the chunk boundary. This is
|
||||||
|
/// undesireable, because now we need to coordinate our prefetch with our
|
||||||
|
/// reader, in case we need to read the next chunk to finish the currenet
|
||||||
|
/// message that is truncated on the current chunk.
|
||||||
|
///
|
||||||
|
/// This is annoying AF, so to solve the problem, we moved upstream: **Write the
|
||||||
|
/// encoded SBE messages all-or-nothing, aligned to the chunk size.** If we are
|
||||||
|
/// writing messages and we find that the next message will tear across two
|
||||||
|
/// chunks, we simply pad the rest of the current chunk with zeros, and write
|
||||||
|
/// the message, in full, starting at the next chunk.
|
||||||
|
///
|
||||||
|
/// This way the reader can then see the zeros, know the chunk is done, and skip
|
||||||
|
/// to the next one.
|
||||||
|
pub struct AlignedSbeReader<T: AlignedSbeSchema> {
|
||||||
|
file: Option<File>,
|
||||||
|
file_len: u64,
|
||||||
|
chunk_size: usize,
|
||||||
|
buffer: Vec<u8>,
|
||||||
|
|
||||||
|
file_offset: u64,
|
||||||
|
buffer_pos: usize,
|
||||||
|
eof: bool,
|
||||||
|
ready_chunks: Option<Receiver<std::io::Result<Option<Vec<u8>>>>>,
|
||||||
|
recycled_chunks: Option<Sender<Vec<u8>>>,
|
||||||
|
_schema: PhantomData<T>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: AlignedSbeSchema> AlignedSbeReader<T> {
|
||||||
|
/// Create a new [`AlignedSbeReader`] from the file at `path`. Chunk size is
|
||||||
|
/// set as a constant in this file, [`DEFAULT_CHUNK_SIZE`].
|
||||||
|
pub fn open(path: impl AsRef<Path>) -> anyhow::Result<Self> {
|
||||||
|
Self::open_with_chunk_size(path, DEFAULT_CHUNK_SIZE)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a new [`AlignedSbeReader`] from the file at `path`, with
|
||||||
|
/// `chunk_size` for chunk size.
|
||||||
|
fn open_with_chunk_size(path: impl AsRef<Path>, chunk_size: usize) -> anyhow::Result<Self> {
|
||||||
|
let file = File::open(path)?;
|
||||||
|
let file_len = file.metadata()?.len();
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
file: Some(file),
|
||||||
|
file_len,
|
||||||
|
chunk_size,
|
||||||
|
buffer: vec![0; chunk_size],
|
||||||
|
file_offset: 0,
|
||||||
|
buffer_pos: 0,
|
||||||
|
eof: false,
|
||||||
|
ready_chunks: None,
|
||||||
|
recycled_chunks: None,
|
||||||
|
_schema: PhantomData,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a task to be passed up to the owner of this reader, so it can run
|
||||||
|
/// chunk prefetching on a separate thread (optionally). When this method is
|
||||||
|
/// run, this consumes certain fields of `self` so they may only be used by
|
||||||
|
/// the prefetch task.
|
||||||
|
pub fn create_prefetch_task(&mut self) -> anyhow::Result<PrefetchTask> {
|
||||||
|
anyhow::ensure!(
|
||||||
|
self.file_offset == 0 && self.buffer_pos == 0,
|
||||||
|
"prefetch must start before reading"
|
||||||
|
);
|
||||||
|
|
||||||
|
let file = self.file.take().context("prefetch already started")?;
|
||||||
|
let file_len = self.file_len;
|
||||||
|
let chunk_size = self.chunk_size;
|
||||||
|
let spare_buffers = [std::mem::take(&mut self.buffer), vec![0; self.chunk_size]];
|
||||||
|
let (ready_chunks_tx, ready_chunks) = bounded(1);
|
||||||
|
let (recycled_chunks, recycled_chunks_rx) = bounded(2);
|
||||||
|
self.ready_chunks = Some(ready_chunks);
|
||||||
|
self.recycled_chunks = Some(recycled_chunks);
|
||||||
|
|
||||||
|
Ok(Box::new(move || {
|
||||||
|
let mut spare_buffers = spare_buffers.into_iter();
|
||||||
|
let mut file_offset = 0u64;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let Some(end) = file_offset.checked_add(chunk_size as u64) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if end > file_len {
|
||||||
|
let _ = ready_chunks_tx.send(Ok(None));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut buffer = match spare_buffers.next() {
|
||||||
|
Some(buffer) => buffer,
|
||||||
|
None => match recycled_chunks_rx.recv() {
|
||||||
|
Ok(buffer) => buffer,
|
||||||
|
Err(_) => return,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
if let Err(error) = file.read_exact_at(&mut buffer, file_offset) {
|
||||||
|
let _ = ready_chunks_tx.send(Err(error));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ready_chunks_tx.send(Ok(Some(buffer))).is_err() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
file_offset = end;
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Signal to the prefetch task that it should stop running.
|
||||||
|
pub fn signal_stop_prefetch(&mut self) {
|
||||||
|
self.ready_chunks = None;
|
||||||
|
self.recycled_chunks = None;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns whether the next chunk could be loaded or not.
|
||||||
|
fn load_chunk(&mut self) -> anyhow::Result<bool> {
|
||||||
|
if self.eof {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(ready_chunks) = &self.ready_chunks {
|
||||||
|
if !self.buffer.is_empty() {
|
||||||
|
let buffer = std::mem::take(&mut self.buffer);
|
||||||
|
if let Some(recycled_chunks) = &self.recycled_chunks {
|
||||||
|
let _ = recycled_chunks.send(buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return match ready_chunks
|
||||||
|
.recv()
|
||||||
|
.context("SBE prefetch thread stopped")??
|
||||||
|
{
|
||||||
|
Some(buffer) => {
|
||||||
|
self.buffer = buffer;
|
||||||
|
self.buffer_pos = 0;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
self.eof = true;
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let end = self
|
||||||
|
.file_offset
|
||||||
|
.checked_add(self.chunk_size as u64)
|
||||||
|
.context("offset overflow")?;
|
||||||
|
if end > self.file_len {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.file
|
||||||
|
.as_ref()
|
||||||
|
.context("file moved to prefetch task")?
|
||||||
|
.read_exact_at(&mut self.buffer, self.file_offset)?;
|
||||||
|
|
||||||
|
self.file_offset = end;
|
||||||
|
self.buffer_pos = 0;
|
||||||
|
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decode the next message from our chunk buffer.
|
||||||
|
///
|
||||||
|
/// Returns `Ok(None)` once we reach the end of the file, and `Err(...)` if
|
||||||
|
/// there is an erroring reading the file, decoding the next message, etc.
|
||||||
|
pub fn read_msg(&mut self) -> anyhow::Result<Option<T::Message<'_>>> {
|
||||||
|
loop {
|
||||||
|
let header_end = self
|
||||||
|
.buffer_pos
|
||||||
|
.checked_add(HEADER_LENGTH)
|
||||||
|
.context("pos overflow")?;
|
||||||
|
|
||||||
|
if header_end <= self.buffer.len()
|
||||||
|
&& self.buffer[self.buffer_pos..header_end] != [0; HEADER_LENGTH]
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if !self.load_chunk()? {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// All generated schemas share this header layout:
|
||||||
|
// blockLength, templateId, schemaId, version -- all little-endian u16.
|
||||||
|
let header_end = self.buffer_pos + HEADER_LENGTH;
|
||||||
|
let header = &self.buffer[self.buffer_pos..header_end];
|
||||||
|
let block_length = usize::from(u16::from_le_bytes([header[0], header[1]]));
|
||||||
|
let schema_id = u16::from_le_bytes([header[4], header[5]]);
|
||||||
|
let schema_version = u16::from_le_bytes([header[6], header[7]]);
|
||||||
|
|
||||||
|
anyhow::ensure!(schema_id == T::SCHEMA_ID);
|
||||||
|
anyhow::ensure!(schema_version == T::SCHEMA_VERSION);
|
||||||
|
|
||||||
|
let decoded = T::try_decode(&self.buffer, self.buffer_pos)?;
|
||||||
|
|
||||||
|
anyhow::ensure!(block_length == T::expected_block_length(&decoded)?);
|
||||||
|
|
||||||
|
let message_len = T::encoded_length(&decoded)?;
|
||||||
|
let end = self
|
||||||
|
.buffer_pos
|
||||||
|
.checked_add(message_len)
|
||||||
|
.context("overflow")?;
|
||||||
|
|
||||||
|
self.buffer_pos = end;
|
||||||
|
|
||||||
|
Ok(Some(decoded))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
//! [`AlignedSbeWriter`] writes SBE messages in fixed-size, zero-padded file chunks.
|
||||||
|
|
||||||
|
use std::{fs::File, io::Write, path::Path};
|
||||||
|
|
||||||
|
use super::dtos::SbeEncode;
|
||||||
|
|
||||||
|
// Every schema in `dtos` uses the standard eight-byte SBE message header.
|
||||||
|
const HEADER_LENGTH: usize = 8;
|
||||||
|
|
||||||
|
/// Writes SBE messages to a flat file in fixed size, zero padded chunks.
|
||||||
|
///
|
||||||
|
/// See [`AlignedSbeReader`](crate::util::AlignedSbeReader) for more details on why
|
||||||
|
/// this struct exists.
|
||||||
|
///
|
||||||
|
/// Every chunk reserves enough trailing zeroes for an empty SBE header, so a
|
||||||
|
/// reader can stop decoding at the sentinel without carrying a frame across a
|
||||||
|
/// chunk boundary.
|
||||||
|
///
|
||||||
|
/// Call [`Self::flush`] to write the final buffered chunk before dropping the
|
||||||
|
/// writer.
|
||||||
|
pub struct AlignedSbeWriter {
|
||||||
|
file: File,
|
||||||
|
buf: Vec<u8>,
|
||||||
|
chunk: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AlignedSbeWriter {
|
||||||
|
/// Create an [`AlignedSbeWriter`], writing to `path` with `chunk` chunk
|
||||||
|
/// size. Clears the file at `path`, if it exists.
|
||||||
|
pub fn create(path: impl AsRef<Path>, chunk: usize) -> anyhow::Result<Self> {
|
||||||
|
anyhow::ensure!(
|
||||||
|
chunk > HEADER_LENGTH,
|
||||||
|
"chunk must leave room for SBE data and an SBE header"
|
||||||
|
);
|
||||||
|
Ok(Self {
|
||||||
|
file: File::create(path)?,
|
||||||
|
buf: Vec::with_capacity(chunk),
|
||||||
|
chunk,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Encodes a complete message directly into the chunk buffer. Rotates to a
|
||||||
|
/// new chunk if it does not fit in the existing chunk.
|
||||||
|
pub fn write_msg<T: SbeEncode + ?Sized>(&mut self, message: &T) -> anyhow::Result<()> {
|
||||||
|
let len = message.encoded_length_with_header()?;
|
||||||
|
let payload_limit = self.chunk - HEADER_LENGTH;
|
||||||
|
anyhow::ensure!(len <= payload_limit, "SBE message does not fit in a chunk");
|
||||||
|
if len > payload_limit - self.buf.len() {
|
||||||
|
self.flush()?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let start = self.buf.len();
|
||||||
|
self.buf.resize(start + len, 0);
|
||||||
|
let result = message.encode(&mut self.buf[start..]).and_then(|written| {
|
||||||
|
anyhow::ensure!(
|
||||||
|
written == len,
|
||||||
|
"SBE encoded length mismatch: expected {len} bytes, wrote {written}"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
});
|
||||||
|
if result.is_err() {
|
||||||
|
self.buf.truncate(start);
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Flush the current buffer to disk, and clear it.
|
||||||
|
pub fn flush(&mut self) -> anyhow::Result<()> {
|
||||||
|
if !self.buf.is_empty() {
|
||||||
|
self.buf.resize(self.chunk, 0);
|
||||||
|
self.file.write_all(&self.buf)?;
|
||||||
|
self.buf.clear();
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
//! Build helpers to auto-generate the rust code for the SBE schemas.
|
||||||
|
|
||||||
|
pub mod sbe_yaml;
|
||||||
|
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
use ergo_sbe::miette::{IntoDiagnostic, Result, miette};
|
||||||
|
use ergo_sbe::{ConversionSelector, DomainVarData, GenerationConfig, generate_str_to_out_dir};
|
||||||
|
|
||||||
|
const SCHEMA: &str = "sbe_schemas/engineering_challenge/challenge_orderbook.yaml";
|
||||||
|
const COMMON_TYPES: &str = "sbe_schemas/includes/common-types.xml";
|
||||||
|
|
||||||
|
fn main() -> Result<()> {
|
||||||
|
println!("cargo::rerun-if-changed={SCHEMA}");
|
||||||
|
println!("cargo::rerun-if-changed={COMMON_TYPES}");
|
||||||
|
|
||||||
|
let yaml = fs::read_to_string(SCHEMA).into_diagnostic()?;
|
||||||
|
let schema = sbe_yaml::parse(&yaml).map_err(|error| miette!("{error}"))?;
|
||||||
|
let common_types = fs::read_to_string(COMMON_TYPES).into_diagnostic()?;
|
||||||
|
let xml = schema
|
||||||
|
.to_xml("challenge_orderbook_messages", &common_types)
|
||||||
|
.map_err(|error| miette!("{error}"))?;
|
||||||
|
|
||||||
|
let config = GenerationConfig::new("challenge_orderbook")
|
||||||
|
.with_domain_objects(DomainVarData::Strings)
|
||||||
|
.with_domain_type(
|
||||||
|
ConversionSelector::named_type("UTCTimestampMicrosEncoding"),
|
||||||
|
"chrono::DateTime<chrono::Utc>",
|
||||||
|
);
|
||||||
|
generate_str_to_out_dir(&xml, config)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
//! SBE helper types, such as [`SbeEncode`].
|
||||||
|
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
|
||||||
|
macro_rules! schemas {
|
||||||
|
($($schema:ident),* $(,)?) => {$(
|
||||||
|
pub mod $schema {
|
||||||
|
#![allow(
|
||||||
|
dead_code,
|
||||||
|
unused_variables,
|
||||||
|
unused_mut,
|
||||||
|
unused_assignments,
|
||||||
|
unused_comparisons,
|
||||||
|
unused_unsafe,
|
||||||
|
clippy::all
|
||||||
|
)]
|
||||||
|
include!(concat!(env!("OUT_DIR"), "/", stringify!($schema), ".rs"));
|
||||||
|
}
|
||||||
|
|
||||||
|
impl $schema::TryFromSbe<$schema::UTCTimestampMicrosEncoding> for DateTime<Utc> {
|
||||||
|
type Error = &'static str;
|
||||||
|
|
||||||
|
fn try_from_sbe(wire: $schema::UTCTimestampMicrosEncoding) -> Result<Self, Self::Error> {
|
||||||
|
i64::try_from(wire.time())
|
||||||
|
.ok()
|
||||||
|
.and_then(DateTime::from_timestamp_micros)
|
||||||
|
.ok_or("UTC microseconds outside chrono's range")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl $schema::TryToSbe<$schema::UTCTimestampMicrosEncoding> for DateTime<Utc> {
|
||||||
|
type Error = &'static str;
|
||||||
|
|
||||||
|
fn try_to_sbe(&self) -> Result<$schema::UTCTimestampMicrosEncoding, Self::Error> {
|
||||||
|
u64::try_from(self.timestamp_micros())
|
||||||
|
.map($schema::UTCTimestampMicrosEncoding::new)
|
||||||
|
.map_err(|_| "UTC timestamp predates the Unix epoch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)*};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// This trait allows us to encode multiple SBE schemas using the same function.
|
||||||
|
pub trait SbeEncode {
|
||||||
|
/// Returns the exact encoded length, including the SBE message header.
|
||||||
|
fn encoded_length_with_header(&self) -> anyhow::Result<usize>;
|
||||||
|
|
||||||
|
/// Encodes the message, including its header, into `buf` and returns the bytes written.
|
||||||
|
/// The buffer must be at least `encoded_length_with_header()` bytes long.
|
||||||
|
/// An encoding error may leave the buffer partially written.
|
||||||
|
fn encode(&self, buf: &mut [u8]) -> anyhow::Result<usize>;
|
||||||
|
|
||||||
|
fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
|
||||||
|
let mut buf = vec![0u8; self.encoded_length_with_header()?];
|
||||||
|
let len = self.encode(&mut buf)?;
|
||||||
|
debug_assert_eq!(len, buf.len());
|
||||||
|
Ok(buf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! sbe_encode {
|
||||||
|
($($message:ty),* $(,)?) => {$(
|
||||||
|
impl SbeEncode for $message {
|
||||||
|
#[inline]
|
||||||
|
fn encoded_length_with_header(&self) -> anyhow::Result<usize> {
|
||||||
|
Ok(<$message>::encoded_length_with_header(self)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn encode(&self, buf: &mut [u8]) -> anyhow::Result<usize> {
|
||||||
|
Ok(<$message>::encode(self, buf)?)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)*};
|
||||||
|
}
|
||||||
|
|
||||||
|
schemas!(challenge_orderbook);
|
||||||
|
|
||||||
|
sbe_encode!(
|
||||||
|
challenge_orderbook::SnapshotDomain,
|
||||||
|
challenge_orderbook::PriceLevelUpdateDomain,
|
||||||
|
);
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
mod aligned_sbe_reader;
|
||||||
|
mod aligned_sbe_writer;
|
||||||
|
pub mod dtos;
|
||||||
|
mod orderbook_schema;
|
||||||
|
|
||||||
|
pub use aligned_sbe_reader::{AlignedSbeReader, AlignedSbeSchema, PrefetchTask};
|
||||||
|
pub use aligned_sbe_writer::AlignedSbeWriter;
|
||||||
|
pub use orderbook_schema::OrderbookSchema;
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
use super::AlignedSbeSchema;
|
||||||
|
use super::dtos::challenge_orderbook::{
|
||||||
|
AnyMessage, PriceLevelUpdateSchema, SCHEMA_ID, SCHEMA_VERSION, SnapshotSchema,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub struct OrderbookSchema;
|
||||||
|
|
||||||
|
impl AlignedSbeSchema for OrderbookSchema {
|
||||||
|
const SCHEMA_ID: u16 = SCHEMA_ID;
|
||||||
|
const SCHEMA_VERSION: u16 = SCHEMA_VERSION;
|
||||||
|
|
||||||
|
type Message<'buf> = AnyMessage<'buf>;
|
||||||
|
|
||||||
|
fn try_decode<'buf>(buffer: &'buf [u8], offset: usize) -> anyhow::Result<Self::Message<'buf>> {
|
||||||
|
Ok(AnyMessage::try_decode(buffer, offset)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encoded_length(message: &Self::Message<'_>) -> anyhow::Result<usize> {
|
||||||
|
Ok(message.encoded_length_with_header()?)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn expected_block_length(message: &Self::Message<'_>) -> anyhow::Result<usize> {
|
||||||
|
Ok(match message {
|
||||||
|
AnyMessage::Snapshot(_) => SnapshotSchema::BLOCK_LENGTH,
|
||||||
|
AnyMessage::PriceLevelUpdate(_) => PriceLevelUpdateSchema::BLOCK_LENGTH,
|
||||||
|
_ => anyhow::bail!("unknown message"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,362 @@
|
|||||||
|
//! Custom transpiler from yaml to FIX SBE XML.
|
||||||
|
|
||||||
|
use std::fmt::{self, Write as _};
|
||||||
|
use std::marker::PhantomData;
|
||||||
|
use std::ops::Deref;
|
||||||
|
|
||||||
|
use indexmap::IndexMap;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use serde::de::{Deserializer, Error as _, MapAccess, Visitor};
|
||||||
|
|
||||||
|
pub type Error = Box<dyn std::error::Error + Send + Sync>;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct Unique<V>(IndexMap<String, V>);
|
||||||
|
|
||||||
|
impl<V> Default for Unique<V> {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self(IndexMap::new())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<V> Deref for Unique<V> {
|
||||||
|
type Target = IndexMap<String, V>;
|
||||||
|
|
||||||
|
fn deref(&self) -> &Self::Target {
|
||||||
|
&self.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'de, V: Deserialize<'de>> Deserialize<'de> for Unique<V> {
|
||||||
|
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||||
|
struct UniqueVisitor<V>(PhantomData<V>);
|
||||||
|
|
||||||
|
impl<'de, V: Deserialize<'de>> Visitor<'de> for UniqueVisitor<V> {
|
||||||
|
type Value = Unique<V>;
|
||||||
|
|
||||||
|
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||||
|
f.write_str("a mapping with unique keys")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Self::Value, A::Error> {
|
||||||
|
let mut entries = IndexMap::new();
|
||||||
|
while let Some((key, value)) = map.next_entry::<String, V>()? {
|
||||||
|
if entries.insert(key.clone(), value).is_some() {
|
||||||
|
return Err(A::Error::custom(format!("duplicate key `{key}`")));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Unique(entries))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
deserializer.deserialize_map(UniqueVisitor(PhantomData))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct Schema {
|
||||||
|
pub id: u16,
|
||||||
|
pub version: u16,
|
||||||
|
pub description: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub imports: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub enums: Unique<Unique<u8>>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub composites: Unique<Composite>,
|
||||||
|
pub messages: Unique<Message>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A shared YAML file contains type definitions, without a schema ID or messages.
|
||||||
|
#[derive(Debug, Default, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct SharedTypes {
|
||||||
|
#[serde(default)]
|
||||||
|
pub enums: Unique<Unique<u8>>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub composites: Unique<Composite>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct Composite {
|
||||||
|
pub description: Option<String>,
|
||||||
|
pub fields: Unique<CompositeMember>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct CompositeMember {
|
||||||
|
/// Only used when a message expands this composite with `fields_from`.
|
||||||
|
pub id: Option<u16>,
|
||||||
|
pub r#type: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub optional: bool,
|
||||||
|
pub since: Option<u16>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct Message {
|
||||||
|
pub id: u16,
|
||||||
|
pub description: Option<String>,
|
||||||
|
pub fields: Option<Unique<Member>>,
|
||||||
|
/// Expand a shared composite as ordinary fields, preserving flat message APIs.
|
||||||
|
pub fields_from: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct Member {
|
||||||
|
pub id: u16,
|
||||||
|
/// Documentation only; emitted as an XML attribute without changing the wire layout.
|
||||||
|
pub description: Option<String>,
|
||||||
|
pub r#type: Option<String>,
|
||||||
|
pub fields: Option<Unique<Member>>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub optional: bool,
|
||||||
|
pub since: Option<u16>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parse(yaml: &str) -> Result<Schema, Error> {
|
||||||
|
Ok(serde_yaml_ng::from_str(yaml)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parse_shared_types(yaml: &str) -> Result<SharedTypes, Error> {
|
||||||
|
Ok(serde_yaml_ng::from_str(yaml)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Schema {
|
||||||
|
pub fn import_types(&mut self, types: SharedTypes) -> Result<(), Error> {
|
||||||
|
for name in types.enums.keys().chain(types.composites.keys()) {
|
||||||
|
if self.enums.contains_key(name) || self.composites.contains_key(name) {
|
||||||
|
return Err(format!("duplicate type `{name}`").into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.enums.0.extend(types.enums.0);
|
||||||
|
self.composites.0.extend(types.composites.0);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn to_xml(&self, package: &str, common_types: &str) -> Result<String, Error> {
|
||||||
|
let mut xml = String::new();
|
||||||
|
writeln!(xml, r#"<?xml version="1.0" encoding="UTF-8"?>"#)?;
|
||||||
|
writeln!(
|
||||||
|
xml,
|
||||||
|
r#"<sbe:messageSchema xmlns:sbe="http://fixprotocol.io/2016/sbe""#
|
||||||
|
)?;
|
||||||
|
writeln!(xml, r#" package="{}""#, escape(package))?;
|
||||||
|
writeln!(xml, r#" id="{}""#, self.id)?;
|
||||||
|
writeln!(xml, r#" version="{}""#, self.version)?;
|
||||||
|
write!(xml, r#" byteOrder="littleEndian""#)?;
|
||||||
|
if let Some(description) = &self.description {
|
||||||
|
write!(
|
||||||
|
xml,
|
||||||
|
"\n description=\"{}\"",
|
||||||
|
escape(description)
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
writeln!(xml, ">")?;
|
||||||
|
writeln!(xml, "{}", types_element(common_types)?)?;
|
||||||
|
if !self.enums.is_empty() || !self.composites.is_empty() {
|
||||||
|
writeln!(xml, " <types>")?;
|
||||||
|
for (name, values) in self.enums.iter() {
|
||||||
|
writeln!(xml, r#" <enum name="{name}" encodingType="uint8">"#)?;
|
||||||
|
for (value, number) in values.iter() {
|
||||||
|
writeln!(
|
||||||
|
xml,
|
||||||
|
r#" <validValue name="{value}">{number}</validValue>"#
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
writeln!(xml, " </enum>")?;
|
||||||
|
}
|
||||||
|
for (name, composite) in self.composites.iter() {
|
||||||
|
if self.enums.contains_key(name) {
|
||||||
|
return Err(format!("duplicate type `{name}`").into());
|
||||||
|
}
|
||||||
|
if composite.fields.is_empty() {
|
||||||
|
return Err(format!("composite `{name}` must have fields").into());
|
||||||
|
}
|
||||||
|
write!(xml, r#" <composite name="{name}""#)?;
|
||||||
|
if let Some(description) = &composite.description {
|
||||||
|
write!(xml, r#" description="{}""#, escape(description))?;
|
||||||
|
}
|
||||||
|
writeln!(xml, ">")?;
|
||||||
|
for (field, member) in composite.fields.iter() {
|
||||||
|
let path = format!("{name}.{field}");
|
||||||
|
let (kind, presence) = composite_member_type(&path, member)?;
|
||||||
|
let since = since_attribute(member.since);
|
||||||
|
if PRIMITIVES.contains(&kind) {
|
||||||
|
writeln!(
|
||||||
|
xml,
|
||||||
|
r#" <type name="{field}" primitiveType="{kind}"{presence}{since}/>"#
|
||||||
|
)?;
|
||||||
|
} else {
|
||||||
|
writeln!(xml, r#" <ref name="{field}" type="{kind}"{since}/>"#)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeln!(xml, " </composite>")?;
|
||||||
|
}
|
||||||
|
writeln!(xml, " </types>")?;
|
||||||
|
}
|
||||||
|
for (name, message) in self.messages.iter() {
|
||||||
|
write!(xml, r#" <sbe:message name="{name}" id="{}""#, message.id)?;
|
||||||
|
if let Some(description) = &message.description {
|
||||||
|
write!(xml, r#" description="{}""#, escape(description))?;
|
||||||
|
}
|
||||||
|
writeln!(xml, ">")?;
|
||||||
|
match (&message.fields, &message.fields_from) {
|
||||||
|
(Some(fields), None) => write_members(&mut xml, name, fields, 2)?,
|
||||||
|
(None, Some(composite_name)) => {
|
||||||
|
let composite = self.composites.get(composite_name).ok_or_else(|| {
|
||||||
|
format!("{name}: unknown composite `{composite_name}` in `fields_from`")
|
||||||
|
})?;
|
||||||
|
for (field, member) in composite.fields.iter() {
|
||||||
|
let path = format!("{name}.{field}");
|
||||||
|
let id = member.id.ok_or_else(|| {
|
||||||
|
format!(
|
||||||
|
"{path}: `fields_from` requires an `id` for each composite member"
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let (kind, presence) = composite_member_type(&path, member)?;
|
||||||
|
let since = since_attribute(member.since);
|
||||||
|
writeln!(
|
||||||
|
xml,
|
||||||
|
r#" <field name="{field}" id="{id}" type="{kind}"{presence}{since}/>"#
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
return Err(
|
||||||
|
format!("{name}: needs exactly one of `fields` or `fields_from`").into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeln!(xml, " </sbe:message>")?;
|
||||||
|
}
|
||||||
|
writeln!(xml, "</sbe:messageSchema>")?;
|
||||||
|
Ok(xml)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const PRIMITIVES: &[&str] = &[
|
||||||
|
"char", "uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64", "float",
|
||||||
|
"double",
|
||||||
|
];
|
||||||
|
|
||||||
|
fn since_attribute(version: Option<u16>) -> String {
|
||||||
|
version
|
||||||
|
.map(|v| format!(r#" sinceVersion="{v}""#))
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn composite_member_type<'a>(
|
||||||
|
path: &str,
|
||||||
|
member: &'a CompositeMember,
|
||||||
|
) -> Result<(&'a str, &'static str), Error> {
|
||||||
|
if matches!(member.r#type.as_str(), "string" | "bytes") {
|
||||||
|
return Err(format!("{path}: composites require fixed-size fields").into());
|
||||||
|
}
|
||||||
|
field_type(path, &member.r#type, member.optional)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn field_type<'a>(
|
||||||
|
path: &str,
|
||||||
|
kind: &'a str,
|
||||||
|
optional: bool,
|
||||||
|
) -> Result<(&'a str, &'static str), Error> {
|
||||||
|
Ok(match (kind, optional) {
|
||||||
|
("bool", false) => ("BooleanType", ""),
|
||||||
|
("timestamp", false) => ("UTCTimestampMicrosEncoding", ""),
|
||||||
|
("timestamp", true) => ("UTCTimestampMicrosNullableEncoding", ""),
|
||||||
|
(kind, true) if NULLABLE_PRIMITIVES.contains(&kind) => (kind, r#" presence="optional""#),
|
||||||
|
(kind, true) => {
|
||||||
|
return Err(format!(
|
||||||
|
"{path}: `{kind}` has no null to spare, so `optional` would be \
|
||||||
|
silently dropped. Enums and `bool` are already nullable — match \
|
||||||
|
on `NullVal`; for anything else use a 0-or-1 group."
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
(kind, false) => (kind, ""),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// SBE reserves the max value of these as the null, so `optional` works as-is.
|
||||||
|
const NULLABLE_PRIMITIVES: &[&str] = &[
|
||||||
|
"uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64", "float", "double",
|
||||||
|
];
|
||||||
|
|
||||||
|
fn write_members(
|
||||||
|
xml: &mut String,
|
||||||
|
path: &str,
|
||||||
|
members: &Unique<Member>,
|
||||||
|
depth: usize,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
let indent = " ".repeat(depth);
|
||||||
|
for (name, member) in members.iter() {
|
||||||
|
let path = format!("{path}.{name}");
|
||||||
|
let id = member.id;
|
||||||
|
let since = since_attribute(member.since);
|
||||||
|
let description = member
|
||||||
|
.description
|
||||||
|
.as_ref()
|
||||||
|
.map(|text| format!(r#" description="{}""#, escape(text)))
|
||||||
|
.unwrap_or_default();
|
||||||
|
match (member.r#type.as_deref(), &member.fields) {
|
||||||
|
(Some(_), Some(_)) | (None, None) => {
|
||||||
|
return Err(format!("{path}: needs exactly one of `type` or `fields`").into());
|
||||||
|
}
|
||||||
|
(None, Some(fields)) => {
|
||||||
|
if member.optional {
|
||||||
|
return Err(format!("{path}: a group cannot be `optional`").into());
|
||||||
|
}
|
||||||
|
writeln!(
|
||||||
|
xml,
|
||||||
|
r#"{indent}<group name="{name}" id="{id}" dimensionType="groupSizeEncoding"{since}{description}>"#
|
||||||
|
)?;
|
||||||
|
write_members(xml, &path, fields, depth + 1)?;
|
||||||
|
writeln!(xml, "{indent}</group>")?;
|
||||||
|
}
|
||||||
|
(Some("string"), None) => {
|
||||||
|
if member.optional {
|
||||||
|
return Err(format!("{path}: a string cannot be `optional`").into());
|
||||||
|
}
|
||||||
|
writeln!(
|
||||||
|
xml,
|
||||||
|
r#"{indent}<data name="{name}" id="{id}" type="varStringEncoding"{since}{description}/>"#
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
(Some(kind), None) => {
|
||||||
|
// `optional` is only ever a reserved value in the encoding, so it
|
||||||
|
// is spelled differently per type: primitives get SBE's own null
|
||||||
|
// (the type's max), a timestamp swaps to the nullable composite,
|
||||||
|
// and anything else has nowhere to put one.
|
||||||
|
let (kind, presence) = field_type(&path, kind, member.optional)?;
|
||||||
|
writeln!(
|
||||||
|
xml,
|
||||||
|
r#"{indent}<field name="{name}" id="{id}" type="{kind}"{presence}{since}{description}/>"#
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn types_element(xml: &str) -> Result<&str, Error> {
|
||||||
|
let start = xml
|
||||||
|
.find("<types>")
|
||||||
|
.ok_or("common types must be a <types> element")?;
|
||||||
|
let end = xml
|
||||||
|
.rfind("</types>")
|
||||||
|
.ok_or("common types must be a <types> element")?;
|
||||||
|
Ok(&xml[start..end + "</types>".len()])
|
||||||
|
}
|
||||||
|
|
||||||
|
fn escape(text: &str) -> String {
|
||||||
|
text.replace('&', "&")
|
||||||
|
.replace('<', "<")
|
||||||
|
.replace('>', ">")
|
||||||
|
.replace('"', """)
|
||||||
|
}
|
||||||
Reference in new issue
Block a user