From c24decd4822a4f2d0a6a4da5478e685444bfeaaa Mon Sep 17 00:00:00 2001 From: Calder White Date: Thu, 22 Oct 2026 00:07:38 -0400 Subject: [PATCH] init commit. --- .gitignore | 1 + .vscode/extensions.json | 12 + .vscode/settings.json | 49 + Cargo.lock | 1099 +++++++++++++++++ Cargo.toml | 17 + README.md | 21 + .../challenge_orderbook.yaml | 52 + sbe_schemas/includes/common-types.xml | 55 + src/lib.rs | 1 + src/main.rs | 64 + src/util/aligned_sbe_reader.rs | 251 ++++ src/util/aligned_sbe_writer.rs | 76 ++ src/util/build.rs | 32 + src/util/dtos.rs | 82 ++ src/util/mod.rs | 8 + src/util/orderbook_schema.rs | 29 + src/util/sbe_yaml.rs | 362 ++++++ 17 files changed, 2211 insertions(+) create mode 100644 .gitignore create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 sbe_schemas/engineering_challenge/challenge_orderbook.yaml create mode 100644 sbe_schemas/includes/common-types.xml create mode 100644 src/lib.rs create mode 100644 src/main.rs create mode 100644 src/util/aligned_sbe_reader.rs create mode 100644 src/util/aligned_sbe_writer.rs create mode 100644 src/util/build.rs create mode 100644 src/util/dtos.rs create mode 100644 src/util/mod.rs create mode 100644 src/util/orderbook_schema.rs create mode 100644 src/util/sbe_yaml.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b83d222 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..434c835 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,12 @@ +{ + "recommendations": [ + "rust-lang.rust-analyzer", + "tamasfe.even-better-toml", + "redhat.vscode-xml", + "hangxingliu.vscode-systemd-support", + "hashicorp.terraform", + "redhat.ansible", + "redhat.vscode-yaml", + "mkhl.shfmt" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..6824019 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,49 @@ +{ + "editor.formatOnSave": true, + "[rust]": { + "editor.defaultFormatter": "rust-lang.rust-analyzer" + }, + "[toml]": { + "editor.defaultFormatter": "tamasfe.even-better-toml", + "editor.formatOnSave": true + }, + "[xml]": { + "editor.defaultFormatter": "redhat.vscode-xml", + "editor.tabSize": 2, + "editor.insertSpaces": true, + "editor.detectIndentation": false + }, + "xml.format.preserveAttributeLineBreaks": true, + "[terraform]": { + "editor.defaultFormatter": "hashicorp.terraform", + "editor.formatOnSave": true + }, + "[terraform-vars]": { + "editor.defaultFormatter": "hashicorp.terraform", + "editor.formatOnSave": true + }, + "[ansible]": { + "editor.defaultFormatter": "redhat.vscode-yaml", + "editor.formatOnSave": true, + "editor.insertSpaces": true, + "editor.tabSize": 2, + "yaml.format.enable": true + }, + "[shellscript]": { + "editor.defaultFormatter": "mkhl.shfmt", + "editor.formatOnSave": true, + "editor.insertSpaces": true, + "editor.tabSize": 2 + }, + "files.associations": { + // *.tfvars.example doesn't match the extension's *.tfvars glob + "*.tfvars.example": "terraform-vars", + "**/.bashrc": "shellscript", + "infra/ansible/**/*.yml": "ansible", + "infra/ansible/**/*.yaml": "ansible" + }, + "files.watcherExclude": { + "**/target/**": true, + "**/.terraform/**": true + } +} diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..77d6a75 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1099 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "backtrace-ext" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" +dependencies = [ + "backtrace", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e71406cd8807725f7ac2f999a4cdd32e98f829fdf65f528343cebf945e41df1e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "engineering-challenge" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "crossbeam", + "ergo-sbe", + "indexmap", + "serde", + "serde_yaml_ng", + "tempfile", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "ergo-sbe" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2eb5418a01016b5cd295b76fbd322ccd1b20204db0193782a477ced69411e9" +dependencies = [ + "miette", + "prettyplease", + "proc-macro2", + "quote", + "roxmltree", + "sha2", + "syn 3.0.9", + "thiserror", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "serde", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "serde", + "stable_deref_trait", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_segmenter" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db" +dependencies = [ + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", + "serde", + "serde_core", +] + +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "backtrace", + "backtrace-ext", + "cfg-if", + "miette-derive", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "owo-colors" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "serde_core", + "writeable", + "zerovec", +] + +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.9", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_yaml_ng" +version = "0.9.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd24347956e682cf958c95e82deb9914cad4010d3efc035d579f81f4c426038c" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "supports-color" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" +dependencies = [ + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e396b6523b11ccb83120b115a0b7366de372751aa6edf19844dfb13a6af97e91" + +[[package]] +name = "supports-unicode" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys", +] + +[[package]] +name = "textwrap" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ecfad6c3abc80a577f2b91c1e412ee57e7a060d430b553c1b0c940974ebcd49" +dependencies = [ + "icu_segmenter", + "unicode-width 0.2.2", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "serde_core", + "zerovec", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "serde", + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..05b273d --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "engineering-challenge" +version = "0.1.0" +edition = "2024" +build = "src/util/build.rs" + +[dependencies] +anyhow = "1" +chrono = "0.4" +crossbeam = "0.8.4" +tempfile = "3" + +[build-dependencies] +ergo-sbe = "=0.1.16" +indexmap = { version = "2", features = ["serde"] } +serde = { version = "1", features = ["derive"] } +serde_yaml_ng = "0.9" diff --git a/README.md b/README.md new file mode 100644 index 0000000..53b9e0d --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ +# Read before getting started!! + +First, read `src/main.rs`. Then, some background on this repo: + +This repo uses the [FIX SBE standard](https://fixtrading.org/packages/simple-binary-encoding-technical-specification-v2-rc2/) +for writing binary messages. The rust code abstracts this away for you, and you can simply use the helpers, as demonstrated in `main.rs`. + +### More background, in case you are curious! + +FIX SBE is defined in XML, which I find extremely cumbersome to edit, especially +when updating schemas frequently. To solve this, I have also introduced some +helper code that takes our own custom YAML format, and turns it into FIX SBE +XML, which in turn is used to generate rust code to encode and decode the +messages. **You don't need to edit any of these files**. This explanation is +simply to help you understand the origins of the various files in this repo. + +The yaml for the schema you will be using is located in +`sbe_schemas/engineering_challenge/challenge_orderbook.yaml`. +We also use a custom timestamp field, so that the Rust struct can use +`chrono::DateTime` in the generated struct, and in SBE schema, it is a uint64 +representing the timestamp as the number of microseconds since unix epoch. diff --git a/sbe_schemas/engineering_challenge/challenge_orderbook.yaml b/sbe_schemas/engineering_challenge/challenge_orderbook.yaml new file mode 100644 index 0000000..54d54ea --- /dev/null +++ b/sbe_schemas/engineering_challenge/challenge_orderbook.yaml @@ -0,0 +1,52 @@ +id: 1 +version: 1 +description: Voluntas engineering challenge orderbook update schema. + +messages: + Snapshot: + id: 1 + description: A full snapshot of the orderbook + fields: + received_at: { id: 1, type: timestamp } + exchange_timestamp: { id: 2, type: timestamp } + bids: + id: 4 + description: Bid price levels + fields: + price_level: + id: 5 + type: int64 + description: The price level, in fixed point; 1_000_000 = 1 USDC + quantity: + id: 6 + type: int64 + description: The absolute available quantity, in fixed point; 1_000_000 = 1 share/contract + asks: + id: 7 + description: Ask price levels + fields: + price_level: + id: 8 + type: int64 + description: The price level, in fixed point; 1_000_000 = 1 USDC + quantity: + id: 9 + type: int64 + description: The absolute available quantity, in fixed point; 1_000_000 = 1 share/contract + symbol: { id: 3, type: string } + + PriceLevelUpdate: + id: 2 + description: An update to a single price level, on a single orderbook + fields: + backtest_timestamp: { id: 1, type: timestamp } + exchange_timestamp: { id: 2, type: timestamp } + price_level: + id: 4 + type: int64 + description: The price level being updated, in fixed point; 1_000_000 = 1 USDC + quantity: + id: 5 + type: int64 + description: The absolute available quantity, in fixed point; 1_000_000 = 1 share/contract + symbol: { id: 3, type: string } diff --git a/sbe_schemas/includes/common-types.xml b/sbe_schemas/includes/common-types.xml new file mode 100644 index 0000000..46cb4f9 --- /dev/null +++ b/sbe_schemas/includes/common-types.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + 6 + + + + + + 6 + + + + + + + + 0 + 1 + + diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..812d1ed --- /dev/null +++ b/src/lib.rs @@ -0,0 +1 @@ +pub mod util; diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..dffd52a --- /dev/null +++ b/src/main.rs @@ -0,0 +1,64 @@ +use chrono::DateTime; +use engineering_challenge::util::dtos::challenge_orderbook::{ + AnyMessage, PriceLevelUpdateDomain, SnapshotAsksEntryDomain, SnapshotBidsEntryDomain, + SnapshotDomain, +}; +use engineering_challenge::util::{AlignedSbeReader, AlignedSbeWriter, OrderbookSchema}; + +const CHUNK_SIZE_64_MEBIBYTES: usize = 64 << 20; + +/// This is a dummy example where we define a snapshot and price level update, +/// write them to a file using [`AlignedSbeWriter`], and then read them back +/// from the same file using [`AlignedSbeReader`], asserting that what we read +/// back is the exact same as what we wrote. +/// +/// You should replace this main function with your actual solution, and split +/// up your code into modules in `src/` in a way that makes sense to you. +fn main() -> anyhow::Result<()> { + let timestamp = DateTime::from_timestamp_micros(1_700_000_000_000_000).unwrap(); + let symbol = "EXAMPLE".to_owned(); + let snapshot = SnapshotDomain { + received_at: timestamp, + exchange_timestamp: timestamp, + bids: vec![SnapshotBidsEntryDomain { + price_level: 400_000, + quantity: 2_000_000, + }], + asks: vec![SnapshotAsksEntryDomain { + price_level: 600_000, + quantity: 3_500_000, + }], + symbol: symbol.clone(), + }; + let update = PriceLevelUpdateDomain { + backtest_timestamp: timestamp, + exchange_timestamp: timestamp, + price_level: 400_000, + quantity: 1_250_000, + symbol: symbol.clone(), + }; + + let file = tempfile::NamedTempFile::new()?; + let mut writer = AlignedSbeWriter::create(file.path(), CHUNK_SIZE_64_MEBIBYTES)?; + writer.write_msg(&snapshot)?; + writer.write_msg(&update)?; + writer.flush()?; + drop(writer); + + let mut reader = AlignedSbeReader::::open(file.path())?; + + let Some(AnyMessage::Snapshot(decoded)) = reader.read_msg()? else { + anyhow::bail!("expected snapshot"); + }; + assert_eq!(SnapshotDomain::try_from_decoder(decoded)?, snapshot); + + let Some(AnyMessage::PriceLevelUpdate(decoded)) = reader.read_msg()? else { + anyhow::bail!("expected price level update"); + }; + assert_eq!(PriceLevelUpdateDomain::try_from_decoder(decoded)?, update); + assert!(reader.read_msg()?.is_none()); + + println!("Verification success!"); + + Ok(()) +} diff --git a/src/util/aligned_sbe_reader.rs b/src/util/aligned_sbe_reader.rs new file mode 100644 index 0000000..9484d61 --- /dev/null +++ b/src/util/aligned_sbe_reader.rs @@ -0,0 +1,251 @@ +//! Defines the [`AlignedSbeReader`], which allows us to quickly read SBE +//! messages written by the +//! [`AlignedSbeWriter`](crate::util::aligned_sbe_writer::AlignedSbeWriter) + +use std::{fs::File, marker::PhantomData, os::unix::fs::FileExt, path::Path}; + +use anyhow::Context; +use crossbeam::channel::{Receiver, Sender, bounded}; + +/// Any SBEs we want to read with the AlignedSbeReader must implement this +/// trait. +pub trait AlignedSbeSchema: 'static { + const SCHEMA_ID: u16; + const SCHEMA_VERSION: u16; + + type Message<'buf> + where + Self: 'buf; + + fn try_decode<'buf>(buffer: &'buf [u8], offset: usize) -> anyhow::Result>; + + fn encoded_length(message: &Self::Message<'_>) -> anyhow::Result; + + fn expected_block_length(message: &Self::Message<'_>) -> anyhow::Result; +} + +const DEFAULT_CHUNK_SIZE: usize = 64 << 20; // 64MiB + +// Every schema in `dtos` uses the standard eight-byte SBE message header. +const HEADER_LENGTH: usize = 8; + +/// See [`AlignedSbeReader::create_prefetch_task`] for more details. +pub type PrefetchTask = Box; + +/// The [`AlignedSbeReader`] is designed to solve the issue of reading partial +/// messages from SBE flat files. Some background: +/// +/// We want to read aeron-spec SBEs from disk at extremely fast speeds -- 100M +/// messages/sec on a single reader thread (with another prefetch thread/task in +/// the background). +/// +/// Additionally, we wanted a very simple API to do so. To achieve this, we +/// prefetch our reads, and read in big chunks ([`DEFAULT_CHUNK_SIZE`]). However, when +/// you read in chunks, messages might tear across the chunk boundary. This is +/// undesireable, because now we need to coordinate our prefetch with our +/// reader, in case we need to read the next chunk to finish the currenet +/// message that is truncated on the current chunk. +/// +/// This is annoying AF, so to solve the problem, we moved upstream: **Write the +/// encoded SBE messages all-or-nothing, aligned to the chunk size.** If we are +/// writing messages and we find that the next message will tear across two +/// chunks, we simply pad the rest of the current chunk with zeros, and write +/// the message, in full, starting at the next chunk. +/// +/// This way the reader can then see the zeros, know the chunk is done, and skip +/// to the next one. +pub struct AlignedSbeReader { + file: Option, + file_len: u64, + chunk_size: usize, + buffer: Vec, + + file_offset: u64, + buffer_pos: usize, + eof: bool, + ready_chunks: Option>>>>, + recycled_chunks: Option>>, + _schema: PhantomData, +} + +impl AlignedSbeReader { + /// Create a new [`AlignedSbeReader`] from the file at `path`. Chunk size is + /// set as a constant in this file, [`DEFAULT_CHUNK_SIZE`]. + pub fn open(path: impl AsRef) -> anyhow::Result { + Self::open_with_chunk_size(path, DEFAULT_CHUNK_SIZE) + } + + /// Create a new [`AlignedSbeReader`] from the file at `path`, with + /// `chunk_size` for chunk size. + fn open_with_chunk_size(path: impl AsRef, chunk_size: usize) -> anyhow::Result { + let file = File::open(path)?; + let file_len = file.metadata()?.len(); + + Ok(Self { + file: Some(file), + file_len, + chunk_size, + buffer: vec![0; chunk_size], + file_offset: 0, + buffer_pos: 0, + eof: false, + ready_chunks: None, + recycled_chunks: None, + _schema: PhantomData, + }) + } + + /// Create a task to be passed up to the owner of this reader, so it can run + /// chunk prefetching on a separate thread (optionally). When this method is + /// run, this consumes certain fields of `self` so they may only be used by + /// the prefetch task. + pub fn create_prefetch_task(&mut self) -> anyhow::Result { + anyhow::ensure!( + self.file_offset == 0 && self.buffer_pos == 0, + "prefetch must start before reading" + ); + + let file = self.file.take().context("prefetch already started")?; + let file_len = self.file_len; + let chunk_size = self.chunk_size; + let spare_buffers = [std::mem::take(&mut self.buffer), vec![0; self.chunk_size]]; + let (ready_chunks_tx, ready_chunks) = bounded(1); + let (recycled_chunks, recycled_chunks_rx) = bounded(2); + self.ready_chunks = Some(ready_chunks); + self.recycled_chunks = Some(recycled_chunks); + + Ok(Box::new(move || { + let mut spare_buffers = spare_buffers.into_iter(); + let mut file_offset = 0u64; + + loop { + let Some(end) = file_offset.checked_add(chunk_size as u64) else { + return; + }; + if end > file_len { + let _ = ready_chunks_tx.send(Ok(None)); + return; + } + + let mut buffer = match spare_buffers.next() { + Some(buffer) => buffer, + None => match recycled_chunks_rx.recv() { + Ok(buffer) => buffer, + Err(_) => return, + }, + }; + if let Err(error) = file.read_exact_at(&mut buffer, file_offset) { + let _ = ready_chunks_tx.send(Err(error)); + return; + } + if ready_chunks_tx.send(Ok(Some(buffer))).is_err() { + return; + } + file_offset = end; + } + })) + } + + /// Signal to the prefetch task that it should stop running. + pub fn signal_stop_prefetch(&mut self) { + self.ready_chunks = None; + self.recycled_chunks = None; + } + + /// Returns whether the next chunk could be loaded or not. + fn load_chunk(&mut self) -> anyhow::Result { + if self.eof { + return Ok(false); + } + + if let Some(ready_chunks) = &self.ready_chunks { + if !self.buffer.is_empty() { + let buffer = std::mem::take(&mut self.buffer); + if let Some(recycled_chunks) = &self.recycled_chunks { + let _ = recycled_chunks.send(buffer); + } + } + + return match ready_chunks + .recv() + .context("SBE prefetch thread stopped")?? + { + Some(buffer) => { + self.buffer = buffer; + self.buffer_pos = 0; + Ok(true) + } + None => { + self.eof = true; + Ok(false) + } + }; + } + + let end = self + .file_offset + .checked_add(self.chunk_size as u64) + .context("offset overflow")?; + if end > self.file_len { + return Ok(false); + } + + self.file + .as_ref() + .context("file moved to prefetch task")? + .read_exact_at(&mut self.buffer, self.file_offset)?; + + self.file_offset = end; + self.buffer_pos = 0; + + Ok(true) + } + + /// Decode the next message from our chunk buffer. + /// + /// Returns `Ok(None)` once we reach the end of the file, and `Err(...)` if + /// there is an erroring reading the file, decoding the next message, etc. + pub fn read_msg(&mut self) -> anyhow::Result>> { + loop { + let header_end = self + .buffer_pos + .checked_add(HEADER_LENGTH) + .context("pos overflow")?; + + if header_end <= self.buffer.len() + && self.buffer[self.buffer_pos..header_end] != [0; HEADER_LENGTH] + { + break; + } + + if !self.load_chunk()? { + return Ok(None); + } + } + + // All generated schemas share this header layout: + // blockLength, templateId, schemaId, version -- all little-endian u16. + let header_end = self.buffer_pos + HEADER_LENGTH; + let header = &self.buffer[self.buffer_pos..header_end]; + let block_length = usize::from(u16::from_le_bytes([header[0], header[1]])); + let schema_id = u16::from_le_bytes([header[4], header[5]]); + let schema_version = u16::from_le_bytes([header[6], header[7]]); + + anyhow::ensure!(schema_id == T::SCHEMA_ID); + anyhow::ensure!(schema_version == T::SCHEMA_VERSION); + + let decoded = T::try_decode(&self.buffer, self.buffer_pos)?; + + anyhow::ensure!(block_length == T::expected_block_length(&decoded)?); + + let message_len = T::encoded_length(&decoded)?; + let end = self + .buffer_pos + .checked_add(message_len) + .context("overflow")?; + + self.buffer_pos = end; + + Ok(Some(decoded)) + } +} diff --git a/src/util/aligned_sbe_writer.rs b/src/util/aligned_sbe_writer.rs new file mode 100644 index 0000000..52a215f --- /dev/null +++ b/src/util/aligned_sbe_writer.rs @@ -0,0 +1,76 @@ +//! [`AlignedSbeWriter`] writes SBE messages in fixed-size, zero-padded file chunks. + +use std::{fs::File, io::Write, path::Path}; + +use super::dtos::SbeEncode; + +// Every schema in `dtos` uses the standard eight-byte SBE message header. +const HEADER_LENGTH: usize = 8; + +/// Writes SBE messages to a flat file in fixed size, zero padded chunks. +/// +/// See [`AlignedSbeReader`](crate::util::AlignedSbeReader) for more details on why +/// this struct exists. +/// +/// Every chunk reserves enough trailing zeroes for an empty SBE header, so a +/// reader can stop decoding at the sentinel without carrying a frame across a +/// chunk boundary. +/// +/// Call [`Self::flush`] to write the final buffered chunk before dropping the +/// writer. +pub struct AlignedSbeWriter { + file: File, + buf: Vec, + chunk: usize, +} + +impl AlignedSbeWriter { + /// Create an [`AlignedSbeWriter`], writing to `path` with `chunk` chunk + /// size. Clears the file at `path`, if it exists. + pub fn create(path: impl AsRef, chunk: usize) -> anyhow::Result { + anyhow::ensure!( + chunk > HEADER_LENGTH, + "chunk must leave room for SBE data and an SBE header" + ); + Ok(Self { + file: File::create(path)?, + buf: Vec::with_capacity(chunk), + chunk, + }) + } + + /// Encodes a complete message directly into the chunk buffer. Rotates to a + /// new chunk if it does not fit in the existing chunk. + pub fn write_msg(&mut self, message: &T) -> anyhow::Result<()> { + let len = message.encoded_length_with_header()?; + let payload_limit = self.chunk - HEADER_LENGTH; + anyhow::ensure!(len <= payload_limit, "SBE message does not fit in a chunk"); + if len > payload_limit - self.buf.len() { + self.flush()?; + } + + let start = self.buf.len(); + self.buf.resize(start + len, 0); + let result = message.encode(&mut self.buf[start..]).and_then(|written| { + anyhow::ensure!( + written == len, + "SBE encoded length mismatch: expected {len} bytes, wrote {written}" + ); + Ok(()) + }); + if result.is_err() { + self.buf.truncate(start); + } + result + } + + /// Flush the current buffer to disk, and clear it. + pub fn flush(&mut self) -> anyhow::Result<()> { + if !self.buf.is_empty() { + self.buf.resize(self.chunk, 0); + self.file.write_all(&self.buf)?; + self.buf.clear(); + } + Ok(()) + } +} diff --git a/src/util/build.rs b/src/util/build.rs new file mode 100644 index 0000000..491d7a9 --- /dev/null +++ b/src/util/build.rs @@ -0,0 +1,32 @@ +//! Build helpers to auto-generate the rust code for the SBE schemas. + +pub mod sbe_yaml; + +use std::fs; + +use ergo_sbe::miette::{IntoDiagnostic, Result, miette}; +use ergo_sbe::{ConversionSelector, DomainVarData, GenerationConfig, generate_str_to_out_dir}; + +const SCHEMA: &str = "sbe_schemas/engineering_challenge/challenge_orderbook.yaml"; +const COMMON_TYPES: &str = "sbe_schemas/includes/common-types.xml"; + +fn main() -> Result<()> { + println!("cargo::rerun-if-changed={SCHEMA}"); + println!("cargo::rerun-if-changed={COMMON_TYPES}"); + + let yaml = fs::read_to_string(SCHEMA).into_diagnostic()?; + let schema = sbe_yaml::parse(&yaml).map_err(|error| miette!("{error}"))?; + let common_types = fs::read_to_string(COMMON_TYPES).into_diagnostic()?; + let xml = schema + .to_xml("challenge_orderbook_messages", &common_types) + .map_err(|error| miette!("{error}"))?; + + let config = GenerationConfig::new("challenge_orderbook") + .with_domain_objects(DomainVarData::Strings) + .with_domain_type( + ConversionSelector::named_type("UTCTimestampMicrosEncoding"), + "chrono::DateTime", + ); + generate_str_to_out_dir(&xml, config)?; + Ok(()) +} diff --git a/src/util/dtos.rs b/src/util/dtos.rs new file mode 100644 index 0000000..5ffdc51 --- /dev/null +++ b/src/util/dtos.rs @@ -0,0 +1,82 @@ +//! SBE helper types, such as [`SbeEncode`]. + +use chrono::{DateTime, Utc}; + +macro_rules! schemas { + ($($schema:ident),* $(,)?) => {$( + pub mod $schema { + #![allow( + dead_code, + unused_variables, + unused_mut, + unused_assignments, + unused_comparisons, + unused_unsafe, + clippy::all + )] + include!(concat!(env!("OUT_DIR"), "/", stringify!($schema), ".rs")); + } + + impl $schema::TryFromSbe<$schema::UTCTimestampMicrosEncoding> for DateTime { + type Error = &'static str; + + fn try_from_sbe(wire: $schema::UTCTimestampMicrosEncoding) -> Result { + i64::try_from(wire.time()) + .ok() + .and_then(DateTime::from_timestamp_micros) + .ok_or("UTC microseconds outside chrono's range") + } + } + + impl $schema::TryToSbe<$schema::UTCTimestampMicrosEncoding> for DateTime { + type Error = &'static str; + + fn try_to_sbe(&self) -> Result<$schema::UTCTimestampMicrosEncoding, Self::Error> { + u64::try_from(self.timestamp_micros()) + .map($schema::UTCTimestampMicrosEncoding::new) + .map_err(|_| "UTC timestamp predates the Unix epoch") + } + } + )*}; +} + +/// This trait allows us to encode multiple SBE schemas using the same function. +pub trait SbeEncode { + /// Returns the exact encoded length, including the SBE message header. + fn encoded_length_with_header(&self) -> anyhow::Result; + + /// Encodes the message, including its header, into `buf` and returns the bytes written. + /// The buffer must be at least `encoded_length_with_header()` bytes long. + /// An encoding error may leave the buffer partially written. + fn encode(&self, buf: &mut [u8]) -> anyhow::Result; + + fn to_bytes(&self) -> anyhow::Result> { + let mut buf = vec![0u8; self.encoded_length_with_header()?]; + let len = self.encode(&mut buf)?; + debug_assert_eq!(len, buf.len()); + Ok(buf) + } +} + +macro_rules! sbe_encode { + ($($message:ty),* $(,)?) => {$( + impl SbeEncode for $message { + #[inline] + fn encoded_length_with_header(&self) -> anyhow::Result { + Ok(<$message>::encoded_length_with_header(self)?) + } + + #[inline] + fn encode(&self, buf: &mut [u8]) -> anyhow::Result { + Ok(<$message>::encode(self, buf)?) + } + } + )*}; +} + +schemas!(challenge_orderbook); + +sbe_encode!( + challenge_orderbook::SnapshotDomain, + challenge_orderbook::PriceLevelUpdateDomain, +); diff --git a/src/util/mod.rs b/src/util/mod.rs new file mode 100644 index 0000000..22d35d8 --- /dev/null +++ b/src/util/mod.rs @@ -0,0 +1,8 @@ +mod aligned_sbe_reader; +mod aligned_sbe_writer; +pub mod dtos; +mod orderbook_schema; + +pub use aligned_sbe_reader::{AlignedSbeReader, AlignedSbeSchema, PrefetchTask}; +pub use aligned_sbe_writer::AlignedSbeWriter; +pub use orderbook_schema::OrderbookSchema; diff --git a/src/util/orderbook_schema.rs b/src/util/orderbook_schema.rs new file mode 100644 index 0000000..b122a9e --- /dev/null +++ b/src/util/orderbook_schema.rs @@ -0,0 +1,29 @@ +use super::AlignedSbeSchema; +use super::dtos::challenge_orderbook::{ + AnyMessage, PriceLevelUpdateSchema, SCHEMA_ID, SCHEMA_VERSION, SnapshotSchema, +}; + +pub struct OrderbookSchema; + +impl AlignedSbeSchema for OrderbookSchema { + const SCHEMA_ID: u16 = SCHEMA_ID; + const SCHEMA_VERSION: u16 = SCHEMA_VERSION; + + type Message<'buf> = AnyMessage<'buf>; + + fn try_decode<'buf>(buffer: &'buf [u8], offset: usize) -> anyhow::Result> { + Ok(AnyMessage::try_decode(buffer, offset)?) + } + + fn encoded_length(message: &Self::Message<'_>) -> anyhow::Result { + Ok(message.encoded_length_with_header()?) + } + + fn expected_block_length(message: &Self::Message<'_>) -> anyhow::Result { + Ok(match message { + AnyMessage::Snapshot(_) => SnapshotSchema::BLOCK_LENGTH, + AnyMessage::PriceLevelUpdate(_) => PriceLevelUpdateSchema::BLOCK_LENGTH, + _ => anyhow::bail!("unknown message"), + }) + } +} diff --git a/src/util/sbe_yaml.rs b/src/util/sbe_yaml.rs new file mode 100644 index 0000000..66f0ee7 --- /dev/null +++ b/src/util/sbe_yaml.rs @@ -0,0 +1,362 @@ +//! Custom transpiler from yaml to FIX SBE XML. + +use std::fmt::{self, Write as _}; +use std::marker::PhantomData; +use std::ops::Deref; + +use indexmap::IndexMap; +use serde::Deserialize; +use serde::de::{Deserializer, Error as _, MapAccess, Visitor}; + +pub type Error = Box; + +#[derive(Debug)] +pub struct Unique(IndexMap); + +impl Default for Unique { + fn default() -> Self { + Self(IndexMap::new()) + } +} + +impl Deref for Unique { + type Target = IndexMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'de, V: Deserialize<'de>> Deserialize<'de> for Unique { + fn deserialize>(deserializer: D) -> Result { + struct UniqueVisitor(PhantomData); + + impl<'de, V: Deserialize<'de>> Visitor<'de> for UniqueVisitor { + type Value = Unique; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a mapping with unique keys") + } + + fn visit_map>(self, mut map: A) -> Result { + let mut entries = IndexMap::new(); + while let Some((key, value)) = map.next_entry::()? { + if entries.insert(key.clone(), value).is_some() { + return Err(A::Error::custom(format!("duplicate key `{key}`"))); + } + } + Ok(Unique(entries)) + } + } + + deserializer.deserialize_map(UniqueVisitor(PhantomData)) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Schema { + pub id: u16, + pub version: u16, + pub description: Option, + #[serde(default)] + pub imports: Vec, + #[serde(default)] + pub enums: Unique>, + #[serde(default)] + pub composites: Unique, + pub messages: Unique, +} + +/// A shared YAML file contains type definitions, without a schema ID or messages. +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SharedTypes { + #[serde(default)] + pub enums: Unique>, + #[serde(default)] + pub composites: Unique, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Composite { + pub description: Option, + pub fields: Unique, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CompositeMember { + /// Only used when a message expands this composite with `fields_from`. + pub id: Option, + pub r#type: String, + #[serde(default)] + pub optional: bool, + pub since: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Message { + pub id: u16, + pub description: Option, + pub fields: Option>, + /// Expand a shared composite as ordinary fields, preserving flat message APIs. + pub fields_from: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Member { + pub id: u16, + /// Documentation only; emitted as an XML attribute without changing the wire layout. + pub description: Option, + pub r#type: Option, + pub fields: Option>, + #[serde(default)] + pub optional: bool, + pub since: Option, +} + +pub fn parse(yaml: &str) -> Result { + Ok(serde_yaml_ng::from_str(yaml)?) +} + +pub fn parse_shared_types(yaml: &str) -> Result { + Ok(serde_yaml_ng::from_str(yaml)?) +} + +impl Schema { + pub fn import_types(&mut self, types: SharedTypes) -> Result<(), Error> { + for name in types.enums.keys().chain(types.composites.keys()) { + if self.enums.contains_key(name) || self.composites.contains_key(name) { + return Err(format!("duplicate type `{name}`").into()); + } + } + self.enums.0.extend(types.enums.0); + self.composites.0.extend(types.composites.0); + Ok(()) + } + + pub fn to_xml(&self, package: &str, common_types: &str) -> Result { + let mut xml = String::new(); + writeln!(xml, r#""#)?; + writeln!( + xml, + r#"")?; + writeln!(xml, "{}", types_element(common_types)?)?; + if !self.enums.is_empty() || !self.composites.is_empty() { + writeln!(xml, " ")?; + for (name, values) in self.enums.iter() { + writeln!(xml, r#" "#)?; + for (value, number) in values.iter() { + writeln!( + xml, + r#" {number}"# + )?; + } + writeln!(xml, " ")?; + } + for (name, composite) in self.composites.iter() { + if self.enums.contains_key(name) { + return Err(format!("duplicate type `{name}`").into()); + } + if composite.fields.is_empty() { + return Err(format!("composite `{name}` must have fields").into()); + } + write!(xml, r#" ")?; + for (field, member) in composite.fields.iter() { + let path = format!("{name}.{field}"); + let (kind, presence) = composite_member_type(&path, member)?; + let since = since_attribute(member.since); + if PRIMITIVES.contains(&kind) { + writeln!( + xml, + r#" "# + )?; + } else { + writeln!(xml, r#" "#)?; + } + } + writeln!(xml, " ")?; + } + writeln!(xml, " ")?; + } + for (name, message) in self.messages.iter() { + write!(xml, r#" ")?; + match (&message.fields, &message.fields_from) { + (Some(fields), None) => write_members(&mut xml, name, fields, 2)?, + (None, Some(composite_name)) => { + let composite = self.composites.get(composite_name).ok_or_else(|| { + format!("{name}: unknown composite `{composite_name}` in `fields_from`") + })?; + for (field, member) in composite.fields.iter() { + let path = format!("{name}.{field}"); + let id = member.id.ok_or_else(|| { + format!( + "{path}: `fields_from` requires an `id` for each composite member" + ) + })?; + let (kind, presence) = composite_member_type(&path, member)?; + let since = since_attribute(member.since); + writeln!( + xml, + r#" "# + )?; + } + } + _ => { + return Err( + format!("{name}: needs exactly one of `fields` or `fields_from`").into(), + ); + } + } + writeln!(xml, " ")?; + } + writeln!(xml, "")?; + Ok(xml) + } +} + +const PRIMITIVES: &[&str] = &[ + "char", "uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64", "float", + "double", +]; + +fn since_attribute(version: Option) -> String { + version + .map(|v| format!(r#" sinceVersion="{v}""#)) + .unwrap_or_default() +} + +fn composite_member_type<'a>( + path: &str, + member: &'a CompositeMember, +) -> Result<(&'a str, &'static str), Error> { + if matches!(member.r#type.as_str(), "string" | "bytes") { + return Err(format!("{path}: composites require fixed-size fields").into()); + } + field_type(path, &member.r#type, member.optional) +} + +fn field_type<'a>( + path: &str, + kind: &'a str, + optional: bool, +) -> Result<(&'a str, &'static str), Error> { + Ok(match (kind, optional) { + ("bool", false) => ("BooleanType", ""), + ("timestamp", false) => ("UTCTimestampMicrosEncoding", ""), + ("timestamp", true) => ("UTCTimestampMicrosNullableEncoding", ""), + (kind, true) if NULLABLE_PRIMITIVES.contains(&kind) => (kind, r#" presence="optional""#), + (kind, true) => { + return Err(format!( + "{path}: `{kind}` has no null to spare, so `optional` would be \ + silently dropped. Enums and `bool` are already nullable — match \ + on `NullVal`; for anything else use a 0-or-1 group." + ) + .into()); + } + (kind, false) => (kind, ""), + }) +} + +/// SBE reserves the max value of these as the null, so `optional` works as-is. +const NULLABLE_PRIMITIVES: &[&str] = &[ + "uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64", "float", "double", +]; + +fn write_members( + xml: &mut String, + path: &str, + members: &Unique, + depth: usize, +) -> Result<(), Error> { + let indent = " ".repeat(depth); + for (name, member) in members.iter() { + let path = format!("{path}.{name}"); + let id = member.id; + let since = since_attribute(member.since); + let description = member + .description + .as_ref() + .map(|text| format!(r#" description="{}""#, escape(text))) + .unwrap_or_default(); + match (member.r#type.as_deref(), &member.fields) { + (Some(_), Some(_)) | (None, None) => { + return Err(format!("{path}: needs exactly one of `type` or `fields`").into()); + } + (None, Some(fields)) => { + if member.optional { + return Err(format!("{path}: a group cannot be `optional`").into()); + } + writeln!( + xml, + r#"{indent}"# + )?; + write_members(xml, &path, fields, depth + 1)?; + writeln!(xml, "{indent}")?; + } + (Some("string"), None) => { + if member.optional { + return Err(format!("{path}: a string cannot be `optional`").into()); + } + writeln!( + xml, + r#"{indent}"# + )?; + } + (Some(kind), None) => { + // `optional` is only ever a reserved value in the encoding, so it + // is spelled differently per type: primitives get SBE's own null + // (the type's max), a timestamp swaps to the nullable composite, + // and anything else has nowhere to put one. + let (kind, presence) = field_type(&path, kind, member.optional)?; + writeln!( + xml, + r#"{indent}"# + )?; + } + } + } + Ok(()) +} + +fn types_element(xml: &str) -> Result<&str, Error> { + let start = xml + .find("") + .ok_or("common types must be a element")?; + let end = xml + .rfind("") + .ok_or("common types must be a element")?; + Ok(&xml[start..end + "".len()]) +} + +fn escape(text: &str) -> String { + text.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +}