Rewrite interpreter to simplify and add minimal runtime debug harness

This commit is contained in:
NGnius 2022-03-25 22:52:17 -04:00
parent 04efebb7ca
commit b0f2250368
10 changed files with 100436 additions and 9 deletions

169
Cargo.lock generated
View file

@ -200,6 +200,18 @@ dependencies = [
"byte-tools",
]
[[package]]
name = "bstr"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
dependencies = [
"lazy_static 1.4.0",
"memchr 2.4.1",
"regex-automata",
"serde",
]
[[package]]
name = "bumpalo"
version = "3.9.1"
@ -251,6 +263,15 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "cast"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a"
dependencies = [
"rustc_version",
]
[[package]]
name = "cc"
version = "1.0.73"
@ -437,6 +458,42 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "criterion"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10"
dependencies = [
"atty",
"cast",
"clap 2.34.0",
"criterion-plot",
"csv",
"itertools",
"lazy_static 1.4.0",
"num-traits",
"oorandom",
"plotters",
"rayon",
"regex 1.5.5",
"serde",
"serde_cbor",
"serde_derive",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "crossbeam"
version = "0.8.1"
@ -506,6 +563,28 @@ dependencies = [
"lazy_static 1.4.0",
]
[[package]]
name = "csv"
version = "1.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1"
dependencies = [
"bstr",
"csv-core",
"itoa 0.4.8",
"ryu",
"serde",
]
[[package]]
name = "csv-core"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
dependencies = [
"memchr 2.4.1",
]
[[package]]
name = "darling"
version = "0.13.1"
@ -757,6 +836,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]]
name = "half"
version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
[[package]]
name = "hashbrown"
version = "0.11.2"
@ -845,6 +930,12 @@ dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "itoa"
version = "1.0.1"
@ -1113,6 +1204,7 @@ name = "mps-interpreter"
version = "0.7.0"
dependencies = [
"bliss-audio",
"criterion",
"dirs",
"rand",
"regex 1.5.5",
@ -1431,6 +1523,12 @@ version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
[[package]]
name = "oorandom"
version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "opaque-debug"
version = "0.2.3"
@ -1532,6 +1630,34 @@ version = "0.3.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
[[package]]
name = "plotters"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a"
dependencies = [
"num-traits",
"plotters-backend",
"plotters-svg",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "plotters-backend"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c"
[[package]]
name = "plotters-svg"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9"
dependencies = [
"plotters-backend",
]
[[package]]
name = "ppv-lite86"
version = "0.2.16"
@ -1716,6 +1842,12 @@ dependencies = [
"regex-syntax 0.6.25",
]
[[package]]
name = "regex-automata"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
[[package]]
name = "regex-syntax"
version = "0.3.9"
@ -1774,6 +1906,15 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc_version"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
"semver",
]
[[package]]
name = "rustfft"
version = "5.1.1"
@ -1809,6 +1950,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "semver"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d"
[[package]]
name = "serde"
version = "1.0.136"
@ -1818,6 +1965,16 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde_cbor"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
dependencies = [
"half",
"serde",
]
[[package]]
name = "serde_derive"
version = "1.0.136"
@ -1835,7 +1992,7 @@ version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
dependencies = [
"itoa",
"itoa 1.0.1",
"ryu",
"serde",
]
@ -2338,6 +2495,16 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "tinyvec"
version = "1.5.1"

View file

@ -38,6 +38,9 @@ strip = true
lto = true
codegen-units = 4
[profile.bench]
lto = false
[profile.dev.package.bliss-audio]
debug-assertions = false
overflow-checks = false

View file

@ -16,6 +16,13 @@ rand = { version = "0.8" }
shellexpand = { version = "2.1", optional = true }
bliss-audio = { version = "0.4", optional = true, path = "../bliss-rs" }
[dev-dependencies]
criterion = "0.3"
[[bench]]
name = "file_parse"
harness = false
[features]
default = [ "music_library", "ergonomics", "advanced" ]
music_library = [ "symphonia" ] # song metadata parsing and database auto-population

View file

@ -0,0 +1,54 @@
use mps_interpreter::{MpsFaye, MpsRunner};
use std::fs::File;
use std::io::{BufReader, Read, Seek};
use criterion::{criterion_group, criterion_main, Criterion};
fn interpretor_benchmark(c: &mut Criterion) {
let f = File::open("benches/lots_of_empty.mps").unwrap();
let mut reader = BufReader::with_capacity(1024 * 1024 /* 1 MiB */, f);
// read everything into buffer before starting
let mut buf = Vec::with_capacity(1024 * 1024);
reader.read_to_end(&mut buf).unwrap();
drop(buf);
c.bench_function("mps lots_of_empty.mps", |b| {
b.iter(|| {
//let f = File::open("benches/lots_of_empty.mps").unwrap();
//let mut reader = BufReader::new(f);
reader.rewind().unwrap();
let mps = MpsRunner::with_stream(&mut reader);
for item in mps {
match item {
Err(e) => panic!("{}", e),
Ok(_) => {}
}
}
})
});
}
fn faye_benchmark(c: &mut Criterion) {
let f = File::open("benches/lots_of_empty.mps").unwrap();
let mut reader = BufReader::with_capacity(1024 * 1024 /* 1 MiB */, f);
// read everything into buffer before starting
let mut buf = Vec::with_capacity(1024 * 1024);
reader.read_to_end(&mut buf).unwrap();
drop(buf);
c.bench_function("mps-faye lots_of_empty.mps", |b| {
b.iter(|| {
//let f = File::open("benches/lots_of_empty.mps").unwrap();
//let mut reader = BufReader::new(f);
reader.rewind().unwrap();
let mps = MpsFaye::with_stream(&mut reader);
for item in mps {
match item {
Err(e) => panic!("{}", e),
Ok(_) => {}
}
}
})
});
}
criterion_group!(parse_benches, interpretor_benchmark, faye_benchmark);
criterion_main!(parse_benches);

File diff suppressed because it is too large Load diff

192
mps-interpreter/src/faye.rs Normal file
View file

@ -0,0 +1,192 @@
use std::collections::VecDeque;
use std::io::Read;
use std::iter::Iterator;
use super::lang::{MpsLanguageDictionary, MpsLanguageError, MpsOp};
use super::tokens::{MpsToken, MpsTokenizer};
use super::MpsContext;
use super::MpsError;
use super::MpsItem;
const DEFAULT_TOKEN_BUFFER_SIZE: usize = 16;
pub enum MpsDebuggableEvent {
FileEnd,
StatementComplete,
NewStatementReady,
}
/// The script interpreter.
pub struct MpsFaye<'a, T>
where
T: crate::tokens::MpsTokenReader,
{
tokenizer: T,
buffer: VecDeque<MpsToken>,
current_stmt: Box<dyn MpsOp>,
vocabulary: MpsLanguageDictionary,
callback: &'a dyn Fn(&mut MpsFaye<'a, T>, MpsDebuggableEvent) -> Result<(), MpsError>,
}
#[inline]
fn empty_callback<'a, T: crate::tokens::MpsTokenReader>(
_s: &mut MpsFaye<'a, T>,
_d: MpsDebuggableEvent,
) -> Result<(), MpsError> {
Ok(())
}
/*impl <T> MpsFaye<'static, T>
where
T: crate::tokens::MpsTokenReader,
{
/// Create a new interpreter for the provided token reader, using the standard MPS language.
#[inline]
pub fn with_standard_vocab(token_reader: T) -> Self {
let mut vocab = MpsLanguageDictionary::default();
super::interpretor::standard_vocab(&mut vocab);
Self::with_vocab(vocab, token_reader)
}
/// Create a new interpreter with the provided vocabulary and token reader.
#[inline]
pub fn with_vocab(vocab: MpsLanguageDictionary, token_reader: T) -> Self {
Self::with(vocab, token_reader, &empty_callback)
}
}*/
impl<'a, R: Read> MpsFaye<'a, MpsTokenizer<R>> {
pub fn with_stream(stream: R) -> Self {
let tokenizer = MpsTokenizer::new(stream);
Self::with_standard_vocab(tokenizer)
}
}
impl<'a, T> MpsFaye<'a, T>
where
T: crate::tokens::MpsTokenReader,
{
#[inline]
pub fn with_standard_vocab(token_reader: T) -> Self {
let mut vocab = MpsLanguageDictionary::default();
super::interpretor::standard_vocab(&mut vocab);
Self::with_vocab(vocab, token_reader)
}
/// Create a new interpreter with the provided vocabulary and token reader.
#[inline]
pub fn with_vocab(vocab: MpsLanguageDictionary, token_reader: T) -> Self {
Self::with(vocab, token_reader, &empty_callback)
}
/// Create a custom interpreter instance.
#[inline]
pub fn with(
vocab: MpsLanguageDictionary,
token_reader: T,
debugger: &'a dyn Fn(&mut MpsFaye<'a, T>, MpsDebuggableEvent) -> Result<(), MpsError>,
) -> Self {
Self {
tokenizer: token_reader,
buffer: VecDeque::with_capacity(DEFAULT_TOKEN_BUFFER_SIZE),
current_stmt: Box::new(crate::lang::vocabulary::empty::EmptyStatement {
context: Some(MpsContext::default()),
}),
vocabulary: vocab,
callback: debugger,
}
}
#[inline]
fn new_statement(
tokenizer: &mut T,
buffer: &mut VecDeque<MpsToken>,
vocab: &MpsLanguageDictionary,
) -> Option<Result<Box<dyn MpsOp>, MpsError>> {
while !tokenizer.end_of_file() && buffer.is_empty() {
let result = tokenizer.next_statement(buffer);
match result {
Ok(_) => {}
Err(e) => return Some(Err(error_with_ctx(e, tokenizer.current_line()))),
}
}
if buffer.is_empty() {
return None;
}
let result = vocab.try_build_statement(buffer);
let stmt = match result {
Ok(stmt) => stmt,
Err(e) => return Some(Err(error_with_ctx(e, tokenizer.current_line()))),
};
#[cfg(debug_assertions)]
if !buffer.is_empty() {
panic!("Token buffer was not emptied! (rem: {:?})", buffer)
}
Some(Ok(stmt))
}
}
impl<'a, T> Iterator for MpsFaye<'a, T>
where
T: crate::tokens::MpsTokenReader,
{
type Item = Result<MpsItem, MpsError>;
fn next(&mut self) -> Option<Self::Item> {
loop {
match self.current_stmt.next() {
Some(item) => {
return Some(item.map_err(|e| error_with_ctx(e, self.tokenizer.current_line())))
}
None => {
// current_stmt has terminated
if self.tokenizer.end_of_file() {
// notify reached end of file
let callback_result = (self.callback)(self, MpsDebuggableEvent::FileEnd);
match callback_result {
Ok(_) => {}
Err(e) => return Some(Err(e)),
}
// nothing left to return
return None;
} else {
// notify old statement is complete
let callback_result =
(self.callback)(self, MpsDebuggableEvent::StatementComplete);
match callback_result {
Ok(_) => {}
Err(e) => return Some(Err(e)),
}
// build next statement
let result = Self::new_statement(
&mut self.tokenizer,
&mut self.buffer,
&self.vocabulary,
);
let mut stmt = match result {
Some(Ok(stmt)) => stmt,
Some(Err(e)) => return Some(Err(e)),
None => return None,
};
let ctx = self.current_stmt.escape();
stmt.enter(ctx);
self.current_stmt = stmt;
// notify new statement is ready
let callback_result =
(self.callback)(self, MpsDebuggableEvent::NewStatementReady);
match callback_result {
Ok(_) => {}
Err(e) => return Some(Err(e)),
}
}
}
}
}
}
}
fn error_with_ctx<T: std::convert::Into<MpsError>>(error: T, line: usize) -> MpsError {
let mut err = error.into();
err.set_line(line);
err
}

View file

@ -11,7 +11,7 @@ use crate::lang::{RuntimeError, SyntaxError};
#[derive(Debug)]
pub struct EmptyStatement {
context: Option<MpsContext>,
pub(crate) context: Option<MpsContext>,
}
impl Display for EmptyStatement {

View file

@ -197,11 +197,13 @@ fn regex_flags(tokens: &mut VecDeque<MpsToken>) -> Result<u8, SyntaxError> {
'U' => result |= 1 << 3,
'u' => result |= 1 << 4,
'x' => result |= 1 << 5,
c => return Err(SyntaxError{
c => {
return Err(SyntaxError {
line: 0,
token: MpsToken::Literal("[one or more of imsUux]".to_string()),
got: Some(MpsToken::Literal(format!("{}", c))),
}),
})
}
}
}
Ok(result)

View file

@ -1,6 +1,6 @@
mod comment;
mod empties;
mod empty;
pub(crate) mod empty;
mod files;
mod intersection;
mod repeat;

View file

@ -241,6 +241,7 @@
mod context;
mod errors;
mod faye;
mod interpretor;
mod item;
pub mod lang;
@ -253,6 +254,7 @@ pub mod tokens;
pub use context::MpsContext;
pub use errors::MpsError;
pub use faye::MpsFaye;
pub use interpretor::{interpretor, MpsInterpretor};
pub use item::MpsItem;
//pub(crate) use item::MpsItemRuntimeUtil;