Rewrite interpreter to simplify and add minimal runtime debug harness

Cargo.lock

@@ -200,6 +200,18 @@ dependencies = [
  "byte-tools",
 ]
 
+[[package]]
+name = "bstr"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
+dependencies = [
+ "lazy_static 1.4.0",
+ "memchr 2.4.1",
+ "regex-automata",
+ "serde",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.9.1"
@@ -251,6 +263,15 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "cast"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a"
+dependencies = [
+ "rustc_version",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.73"
@@ -437,6 +458,42 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10"
+dependencies = [
+ "atty",
+ "cast",
+ "clap 2.34.0",
+ "criterion-plot",
+ "csv",
+ "itertools",
+ "lazy_static 1.4.0",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex 1.5.5",
+ "serde",
+ "serde_cbor",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
 [[package]]
 name = "crossbeam"
 version = "0.8.1"
@@ -506,6 +563,28 @@ dependencies = [
  "lazy_static 1.4.0",
 ]
 
+[[package]]
+name = "csv"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1"
+dependencies = [
+ "bstr",
+ "csv-core",
+ "itoa 0.4.8",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
+dependencies = [
+ "memchr 2.4.1",
+]
+
 [[package]]
 name = "darling"
 version = "0.13.1"
@@ -757,6 +836,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
 
+[[package]]
+name = "half"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
+
 [[package]]
 name = "hashbrown"
 version = "0.11.2"
@@ -845,6 +930,12 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itoa"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
+
 [[package]]
 name = "itoa"
 version = "1.0.1"
@@ -1113,6 +1204,7 @@ name = "mps-interpreter"
 version = "0.7.0"
 dependencies = [
  "bliss-audio",
+ "criterion",
  "dirs",
  "rand",
  "regex 1.5.5",
@@ -1431,6 +1523,12 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
 
+[[package]]
+name = "oorandom"
+version = "11.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+
 [[package]]
 name = "opaque-debug"
 version = "0.2.3"
@@ -1532,6 +1630,34 @@ version = "0.3.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
 
+[[package]]
+name = "plotters"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.16"
@@ -1716,6 +1842,12 @@ dependencies = [
  "regex-syntax 0.6.25",
 ]
 
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+
 [[package]]
 name = "regex-syntax"
 version = "0.3.9"
@@ -1774,6 +1906,15 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustfft"
 version = "5.1.1"
@@ -1809,6 +1950,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "semver"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d"
+
 [[package]]
 name = "serde"
 version = "1.0.136"
@@ -1818,6 +1965,16 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "serde_cbor"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
+dependencies = [
+ "half",
+ "serde",
+]
+
 [[package]]
 name = "serde_derive"
 version = "1.0.136"
@@ -1835,7 +1992,7 @@ version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
 dependencies = [
- "itoa",
+ "itoa 1.0.1",
  "ryu",
  "serde",
 ]
@@ -2338,6 +2495,16 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.5.1"

Cargo.toml

@@ -38,6 +38,9 @@ strip = true
 lto = true
 codegen-units = 4
 
+[profile.bench]
+lto = false
+
 [profile.dev.package.bliss-audio]
 debug-assertions = false
 overflow-checks = false

mps-interpreter/Cargo.toml

@@ -16,6 +16,13 @@ rand = { version = "0.8" }
 shellexpand = { version = "2.1", optional = true }
 bliss-audio = { version = "0.4", optional = true, path = "../bliss-rs" }
 
+[dev-dependencies]
+criterion = "0.3"
+
+[[bench]]
+name = "file_parse"
+harness = false
+
 [features]
 default = [ "music_library", "ergonomics", "advanced" ]
 music_library = [ "symphonia" ] # song metadata parsing and database auto-population

mps-interpreter/benches/file_parse.rs

@@ -0,0 +1,54 @@
+use mps_interpreter::{MpsFaye, MpsRunner};
+use std::fs::File;
+use std::io::{BufReader, Read, Seek};
+
+use criterion::{criterion_group, criterion_main, Criterion};
+
+fn interpretor_benchmark(c: &mut Criterion) {
+    let f = File::open("benches/lots_of_empty.mps").unwrap();
+    let mut reader = BufReader::with_capacity(1024 * 1024 /* 1 MiB */, f);
+    // read everything into buffer before starting
+    let mut buf = Vec::with_capacity(1024 * 1024);
+    reader.read_to_end(&mut buf).unwrap();
+    drop(buf);
+    c.bench_function("mps lots_of_empty.mps", |b| {
+        b.iter(|| {
+            //let f = File::open("benches/lots_of_empty.mps").unwrap();
+            //let mut reader = BufReader::new(f);
+            reader.rewind().unwrap();
+            let mps = MpsRunner::with_stream(&mut reader);
+            for item in mps {
+                match item {
+                    Err(e) => panic!("{}", e),
+                    Ok(_) => {}
+                }
+            }
+        })
+    });
+}
+
+fn faye_benchmark(c: &mut Criterion) {
+    let f = File::open("benches/lots_of_empty.mps").unwrap();
+    let mut reader = BufReader::with_capacity(1024 * 1024 /* 1 MiB */, f);
+    // read everything into buffer before starting
+    let mut buf = Vec::with_capacity(1024 * 1024);
+    reader.read_to_end(&mut buf).unwrap();
+    drop(buf);
+    c.bench_function("mps-faye lots_of_empty.mps", |b| {
+        b.iter(|| {
+            //let f = File::open("benches/lots_of_empty.mps").unwrap();
+            //let mut reader = BufReader::new(f);
+            reader.rewind().unwrap();
+            let mps = MpsFaye::with_stream(&mut reader);
+            for item in mps {
+                match item {
+                    Err(e) => panic!("{}", e),
+                    Ok(_) => {}
+                }
+            }
+        })
+    });
+}
+
+criterion_group!(parse_benches, interpretor_benchmark, faye_benchmark);
+criterion_main!(parse_benches);

mps-interpreter/src/faye.rs

@@ -0,0 +1,192 @@
+use std::collections::VecDeque;
+use std::io::Read;
+use std::iter::Iterator;
+
+use super::lang::{MpsLanguageDictionary, MpsLanguageError, MpsOp};
+use super::tokens::{MpsToken, MpsTokenizer};
+use super::MpsContext;
+use super::MpsError;
+use super::MpsItem;
+
+const DEFAULT_TOKEN_BUFFER_SIZE: usize = 16;
+
+pub enum MpsDebuggableEvent {
+    FileEnd,
+    StatementComplete,
+    NewStatementReady,
+}
+
+/// The script interpreter.
+pub struct MpsFaye<'a, T>
+where
+    T: crate::tokens::MpsTokenReader,
+{
+    tokenizer: T,
+    buffer: VecDeque<MpsToken>,
+    current_stmt: Box<dyn MpsOp>,
+    vocabulary: MpsLanguageDictionary,
+    callback: &'a dyn Fn(&mut MpsFaye<'a, T>, MpsDebuggableEvent) -> Result<(), MpsError>,
+}
+
+#[inline]
+fn empty_callback<'a, T: crate::tokens::MpsTokenReader>(
+    _s: &mut MpsFaye<'a, T>,
+    _d: MpsDebuggableEvent,
+) -> Result<(), MpsError> {
+    Ok(())
+}
+
+/*impl <T> MpsFaye<'static, T>
+where
+    T: crate::tokens::MpsTokenReader,
+{
+    /// Create a new interpreter for the provided token reader, using the standard MPS language.
+    #[inline]
+    pub fn with_standard_vocab(token_reader: T) -> Self {
+        let mut vocab = MpsLanguageDictionary::default();
+        super::interpretor::standard_vocab(&mut vocab);
+        Self::with_vocab(vocab, token_reader)
+    }
+
+    /// Create a new interpreter with the provided vocabulary and token reader.
+    #[inline]
+    pub fn with_vocab(vocab: MpsLanguageDictionary, token_reader: T) -> Self {
+        Self::with(vocab, token_reader, &empty_callback)
+    }
+}*/
+
+impl<'a, R: Read> MpsFaye<'a, MpsTokenizer<R>> {
+    pub fn with_stream(stream: R) -> Self {
+        let tokenizer = MpsTokenizer::new(stream);
+        Self::with_standard_vocab(tokenizer)
+    }
+}
+
+impl<'a, T> MpsFaye<'a, T>
+where
+    T: crate::tokens::MpsTokenReader,
+{
+    #[inline]
+    pub fn with_standard_vocab(token_reader: T) -> Self {
+        let mut vocab = MpsLanguageDictionary::default();
+        super::interpretor::standard_vocab(&mut vocab);
+        Self::with_vocab(vocab, token_reader)
+    }
+
+    /// Create a new interpreter with the provided vocabulary and token reader.
+    #[inline]
+    pub fn with_vocab(vocab: MpsLanguageDictionary, token_reader: T) -> Self {
+        Self::with(vocab, token_reader, &empty_callback)
+    }
+
+    /// Create a custom interpreter instance.
+    #[inline]
+    pub fn with(
+        vocab: MpsLanguageDictionary,
+        token_reader: T,
+        debugger: &'a dyn Fn(&mut MpsFaye<'a, T>, MpsDebuggableEvent) -> Result<(), MpsError>,
+    ) -> Self {
+        Self {
+            tokenizer: token_reader,
+            buffer: VecDeque::with_capacity(DEFAULT_TOKEN_BUFFER_SIZE),
+            current_stmt: Box::new(crate::lang::vocabulary::empty::EmptyStatement {
+                context: Some(MpsContext::default()),
+            }),
+            vocabulary: vocab,
+            callback: debugger,
+        }
+    }
+
+    #[inline]
+    fn new_statement(
+        tokenizer: &mut T,
+        buffer: &mut VecDeque<MpsToken>,
+        vocab: &MpsLanguageDictionary,
+    ) -> Option<Result<Box<dyn MpsOp>, MpsError>> {
+        while !tokenizer.end_of_file() && buffer.is_empty() {
+            let result = tokenizer.next_statement(buffer);
+            match result {
+                Ok(_) => {}
+                Err(e) => return Some(Err(error_with_ctx(e, tokenizer.current_line()))),
+            }
+        }
+        if buffer.is_empty() {
+            return None;
+        }
+        let result = vocab.try_build_statement(buffer);
+        let stmt = match result {
+            Ok(stmt) => stmt,
+            Err(e) => return Some(Err(error_with_ctx(e, tokenizer.current_line()))),
+        };
+        #[cfg(debug_assertions)]
+        if !buffer.is_empty() {
+            panic!("Token buffer was not emptied! (rem: {:?})", buffer)
+        }
+        Some(Ok(stmt))
+    }
+}
+
+impl<'a, T> Iterator for MpsFaye<'a, T>
+where
+    T: crate::tokens::MpsTokenReader,
+{
+    type Item = Result<MpsItem, MpsError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            match self.current_stmt.next() {
+                Some(item) => {
+                    return Some(item.map_err(|e| error_with_ctx(e, self.tokenizer.current_line())))
+                }
+                None => {
+                    // current_stmt has terminated
+                    if self.tokenizer.end_of_file() {
+                        // notify reached end of file
+                        let callback_result = (self.callback)(self, MpsDebuggableEvent::FileEnd);
+                        match callback_result {
+                            Ok(_) => {}
+                            Err(e) => return Some(Err(e)),
+                        }
+                        // nothing left to return
+                        return None;
+                    } else {
+                        // notify old statement is complete
+                        let callback_result =
+                            (self.callback)(self, MpsDebuggableEvent::StatementComplete);
+                        match callback_result {
+                            Ok(_) => {}
+                            Err(e) => return Some(Err(e)),
+                        }
+                        // build next statement
+                        let result = Self::new_statement(
+                            &mut self.tokenizer,
+                            &mut self.buffer,
+                            &self.vocabulary,
+                        );
+                        let mut stmt = match result {
+                            Some(Ok(stmt)) => stmt,
+                            Some(Err(e)) => return Some(Err(e)),
+                            None => return None,
+                        };
+                        let ctx = self.current_stmt.escape();
+                        stmt.enter(ctx);
+                        self.current_stmt = stmt;
+                        // notify new statement is ready
+                        let callback_result =
+                            (self.callback)(self, MpsDebuggableEvent::NewStatementReady);
+                        match callback_result {
+                            Ok(_) => {}
+                            Err(e) => return Some(Err(e)),
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+fn error_with_ctx<T: std::convert::Into<MpsError>>(error: T, line: usize) -> MpsError {
+    let mut err = error.into();
+    err.set_line(line);
+    err
+}

mps-interpreter/src/lang/vocabulary/empty.rs

@@ -11,7 +11,7 @@ use crate::lang::{RuntimeError, SyntaxError};
 
 #[derive(Debug)]
 pub struct EmptyStatement {
-    context: Option<MpsContext>,
+    pub(crate) context: Option<MpsContext>,
 }
 
 impl Display for EmptyStatement {

mps-interpreter/src/lang/vocabulary/filters/field_match_filter.rs

@@ -173,7 +173,7 @@ impl MpsFilterFactory<FieldRegexFilter> for FieldRegexFilterFactory {
 }
 
 #[inline]
-fn regex_flags(tokens: &mut VecDeque<MpsToken>) ->  Result<u8, SyntaxError> {
+fn regex_flags(tokens: &mut VecDeque<MpsToken>) -> Result<u8, SyntaxError> {
     // syntax: , "flags"
     let mut result = 0_u8;
     if tokens.is_empty() {
@@ -197,11 +197,13 @@ fn regex_flags(tokens: &mut VecDeque<MpsToken>) ->  Result<u8, SyntaxError> {
                 'U' => result |= 1 << 3,
                 'u' => result |= 1 << 4,
                 'x' => result |= 1 << 5,
-                c => return Err(SyntaxError{
+                c => {
-                    line: 0,
+                    return Err(SyntaxError {
-                    token: MpsToken::Literal("[one or more of imsUux]".to_string()),
+                        line: 0,
-                    got: Some(MpsToken::Literal(format!("{}", c))),
+                        token: MpsToken::Literal("[one or more of imsUux]".to_string()),
-                }),
+                        got: Some(MpsToken::Literal(format!("{}", c))),
+                    })
+                }
             }
         }
         Ok(result)

mps-interpreter/src/lang/vocabulary/mod.rs

@@ -1,6 +1,6 @@
 mod comment;
 mod empties;
-mod empty;
+pub(crate) mod empty;
 mod files;
 mod intersection;
 mod repeat;

mps-interpreter/src/lib.rs

@@ -241,6 +241,7 @@
 
 mod context;
 mod errors;
+mod faye;
 mod interpretor;
 mod item;
 pub mod lang;
@@ -253,6 +254,7 @@ pub mod tokens;
 
 pub use context::MpsContext;
 pub use errors::MpsError;
+pub use faye::MpsFaye;
 pub use interpretor::{interpretor, MpsInterpretor};
 pub use item::MpsItem;
 //pub(crate) use item::MpsItemRuntimeUtil;