Fix playlist filepath generation to ignore already-absolute paths

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: NGnius
+liberapay: NGnius

.gitignore

@@ -1,5 +1,5 @@
 /target
 **/target
-/*/metadata.mps.sqlite
-metadata.mps.sqlite
+/*/metadata.muss.sqlite
+metadata.muss.sqlite
 **.m3u8

.gitmodules

@@ -4,3 +4,5 @@
 [submodule "bliss-rs"]
 	path = bliss-rs
 	url = https://github.com/NGnius/bliss-rs
+[submodule "bliss-rs/"]
+	url = https://git.ngni.us/NGnius/bliss-rs

Cargo.toml

@@ -1,32 +1,57 @@
 [package]
-name = "mps"
-version = "0.5.0"
+name = "muss"
+version = "0.9.0"
 edition = "2021"
 authors = ["NGnius (Graham) <ngniusness@gmail.com>"]
-description = "Music Playlist Script language (MPS)"
+description = "Music Set Script language (MuSS)"
 license = "LGPL-2.1-only OR GPL-3.0-only"
-repository = "https://github.com/NGnius/mps"
+repository = "https://git.ngni.us/NGnius/muss"
 keywords = ["audio", "playlist", "scripting", "language"]
 readme = "README.md"
 exclude = ["extras/"]
 
 [workspace]
 members = [
-    "mps-interpreter",
-    "mps-player",
-    "mps-m3u8"
+    "interpreter",
+    "player",
+    "m3u8"
 ]
 
 [dependencies]
 # local
-mps-interpreter = { version = "0.5.0", path = "./mps-interpreter" }
+muss-interpreter = { version = "0.9.0", path = "./interpreter" }
 # external
 clap = { version = "3.0", features = ["derive"] }
-# termios = { version = "^0.3"}
+console = { version = "0.15" }
+lazy_static = { version = "1.4" }
+
+# cli add to playlist functionality
+m3u8-rs = { version = "^3.0.0" }
 
 [target.'cfg(not(target_os = "linux"))'.dependencies]
-mps-player = { version = "0.5.0", path = "./mps-player", default-features = false }
+muss-player = { version = "0.9.0", path = "./player", default-features = false, features = ["mpd"] }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 # TODO fix need to specify OS-specific dependency of mps-player
-mps-player = { version = "0.5.0", path = "./mps-player", features = ["mpris-player"] }
+muss-player = { version = "0.9.0", path = "./player", features = ["mpris-player", "mpd"] }
+
+[profile.release]
+debug = false
+strip = true
+lto = true
+codegen-units = 4
+
+[profile.bench]
+lto = false
+
+[profile.dev.package.bliss-audio-symphonia]
+debug-assertions = false
+overflow-checks = false
+debug = true
+opt-level = 3
+
+[profile.dev.package."*"]
+debug-assertions = false
+overflow-checks = false
+debug = true
+opt-level = 3

README.md

@@ -1,42 +1,60 @@
-# mps
+# muss
 
-![repl_demo](https://raw.githubusercontent.com/NGnius/mps/master/extras/demo.png)
+![repl_demo](https://raw.githubusercontent.com/NGnius/muss/master/extras/demo.png)
 
-A language all about iteration to play your music files.
-This project implements the interpreter (mps-interpreter), music player (mps-player), and CLI interface for MPS (root).
+Sort, filter and analyse your music to create great playlists.
+This project implements the interpreter (`./interpreter`), music player (`./player`), and CLI interface for Muss (`./`).
 The CLI interface includes a REPL for running scripts.
-The REPL interactive mode also provides more details about using MPS through the `?help` command.
+The REPL interactive mode also provides more details about using Muss through the `?help` command.
 
 ## Usage
-To access the REPL, simply run `cargo run`. You will need the [Rust toolchain installed](https://rustup.rs/).
+To access the REPL, simply run `cargo run`. You will need the [Rust toolchain installed](https://rustup.rs/). For a bit of extra performance, run `cargo run --release` instead.
 
 ## Examples
-For now, check out `./src/tests`, `./mps-player/tests`, and `./mps-interpreter/tests` for examples.
+
+### One-liners
+
+All songs by artist `<artist>` (in your library), sorted by similarity to a random first song by the artist.
+```muss
+files().(.artist? like "<artist>")~(~radio);
+```
+
+All songs with a `.flac` file extension (anywhere in their path -- not necessarily at the end).
+```muss
+files().(.filename? like ".flac");
+```
+
+All songs by artist `<artist1>` or `<artist2>`, sorted by similarity to a random first song by either artist.
+```muss
+files().(.artist? like "<artist1>" || .artist? like "<artist2>")~(~radio);
+```
+
+### Bigger examples
+
+For now, check out `./src/tests`, `./player/tests`, and `./interpreter/tests` for examples.
 One day I'll add pretty REPL example pictures and some script files...
 // TODO
 
 ## FAQ
-### Is MPS Turing-Complete?
-**No**. It can't perform arbitrary calculations (yet), which easily disqualifies MPS from being Turing-complete.
 
-### Can I use MPS right now?
-**Sure!** It's not complete, but MPS is completely useable for basic music queries right now. Hopefully most of the bugs have been ironed out as well...
+### Can I use Muss right now?
+**Sure!** It's never complete, but Muss is completely useable right now. Hopefully most of the bugs have been ironed out as well :)
 
 ### Why write a new language?
-**I thought it would be fun**. I also wanted to be able to play my music without having to be at the whim of someone else's algorithm (and music), and playing just by album or artist was getting boring. I also thought designing a language specifically for iteration would be a novel approach to a language (though every approach is a novel approach for me).
+**I thought it would be fun**. I also wanted to be able to play my music without having to be at the whim of someone else's algorithm (and music), and playing just by album or artist was getting boring. Designing a language specifically for iteration seemed like a cool & novel way of doing it, too (though every approach is a novel approach for me).
 
-### What is MPS?
-**Music Playlist Script (MPS) is technically a query language for music files.** It uses an (auto-generated) SQLite3 database for SQL queries and can also directly query the filesystem. Queries can be modified by using filters, functions, and sorters built-in to MPS (see mps-interpreter's README.md).
+### What is Muss?
+**Music Set Script (MuSS) is a language for describing a playlist of music.** It can execute SQLite select clauses or directly query the filesystem. Queries can be modified by using filters, functions, and sorters built-in to Muss (see interpreter's README.md).
 
-### Is MPS a scripting language?
-**No**. Technically, it was designed to be one, but it doesn't meet the requirements of a scripting language (yet). One day, I would like it be Turing-complete and then it could be considered a scripting language. At the moment it is barely a query language.
+### Is Muss a scripting language?
+**Yes**. It evolved from a simple query language into something that can do arbitrary calculations. Whether it's Turing-complete is still unproven, but it's powerful enough for what I want it to do.
 
 
 ## License
 
 LGPL-2.1-only OR GPL-3.0-only
 
-**NOTE**: When advanced features are enabled, GPL-3.0 must be used.
+**NOTE**: Advanced features make use of [a fork of bliss-rs](https://github.com/NGnius/bliss-rs) a GPL-3.0 licensed music analysis library.
 
 ## Contribution
 

README.tpl

@@ -1,6 +1,6 @@
 # {{crate}}
 
-![repl_demo](https://raw.githubusercontent.com/NGnius/mps/master/extras/demo.png)
+![repl_demo](https://raw.githubusercontent.com/NGnius/muss/master/extras/demo.png)
 
 {{readme}}
 
@@ -8,7 +8,7 @@
 
 {{license}}
 
-**NOTE**: When advanced features are enabled, GPL-3.0 must be used.
+**NOTE**: Advanced features make use of [a fork of bliss-rs](https://github.com/NGnius/bliss-rs) a GPL-3.0 licensed music analysis library.
 
 ## Contribution
 

bliss-rs

@@ -1 +1 @@
-Subproject commit 1f101b5975f923d3a13a9d122c0f19a91e6d0b82
+Subproject commit 2dcdd5643bc0cd055887128637abbb9bed041f77

interpreter/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "muss-interpreter"
+version = "0.9.0"
+edition = "2021"
+license = "LGPL-2.1-only OR GPL-3.0-only"
+readme = "README.md"
+repository = "https://git.ngni.us/NGnius/muss"
+rust-version = "1.59"
+
+[dependencies]
+rusqlite = { version = "0.27", features = ["bundled"], optional = true }
+sqlparser = { version = "0.23", optional = true }
+symphonia = { version = "0.5", optional = true, features = ["all"] }
+dirs = { version = "4" }
+regex = { version = "1" }
+rand = { version = "0.8" }
+shellexpand = { version = "2", optional = true }
+bliss-audio-symphonia = { version = "0.6", optional = true, path = "../bliss-rs" }
+mpd = { version = "0.1", optional = true }
+unidecode = { version = "0.3.0", optional = true }
+base64 = { version = "0.13", optional = true }
+m3u8-rs = { version = "3.0.0", optional = true }
+
+[dev-dependencies]
+criterion = "0.3"
+
+[[bench]]
+name = "file_parse"
+harness = false
+
+[features]
+default = [ "music_library", "ergonomics", "advanced", "advanced-bliss", "fakesql", "collections" ]
+music_library = [ "symphonia", "mpd", "base64" ] # song metadata parsing and database auto-population
+collections = [ "m3u8-rs" ] # read from m3u8 playlists (and other song collections, eventually)
+ergonomics = ["shellexpand", "unidecode"] # niceties like ~ in paths and unicode string sanitisation
+advanced = [] # advanced language features like music analysis
+advanced-bliss = ["bliss-audio-symphonia"] # bliss audio analysis
+sql = [ "rusqlite" ] # sqlite database for music
+fakesql = [ "sqlparser" ] # transpiled sqlite interpreter

interpreter/README.md

@@ -1,29 +1,26 @@
-# mps-interpreter
+# muss-interpreter
 
-All necessary components to interpret and run a MPS script.
+All necessary components to interpret and run a Muss script.
 
-MpsInterpretor uses a non-standard Iterator implementation,
-so it is recommended to use MpsRunner to execute a script.
-Since MPS is centered around iterators, script execution is also done by iterating.
+Interpreter is the Muss script interpreter.
+Debugger can be used to run scripts within a custom debug harness.
+Since Muss is centered around iterators, script execution is also done by iterating.
 
-MpsInterpretor is misspelt to emphasise that it behaves strangely:
-after every MPS statement, a None item is returned even when the script is not complete.
-MpsRunner wraps MpsInterpretor so that this behaviour is hidden when iterating.
 
 ```rust
 use std::io::Cursor;
-use mps_interpreter::*;
+use muss_interpreter::*;
 
 let cursor = Cursor::new(
 "files(folder=`~/Music/`, recursive=true)" // retrieve all files from Music folder
 );
 
-let interpreter = MpsRunner::with_stream(cursor);
+let interpreter = Interpreter::with_stream(cursor);
 
 // warning: my library has ~3800 songs, so this outputs too much information to be useful.
 for result in interpreter {
     match result {
-        Ok(item) => println!("Got song `{}` (file: `{}`)", item.title, item.filename),
+        Ok(item) => println!("Got song `{}` (file: `{}`)", item.field("title").unwrap(), item.field("filename").unwrap()),
         Err(e) => panic!("Got error while executing: {}", e),
     }
 }
@@ -31,38 +28,40 @@ for result in interpreter {
 
 ## Standard Vocabulary
 By default, the standard vocabulary is used to parse the stream when iterating the interpreter.
-The standard vocabulary defines the valid statement syntax for MPS and parses syntax into special Rust Iterators which can be used to execute the statement.
-To declare your own vocabulary, use MpsRunner::with_settings to provide a MpsInterpretor with a custom vocabulary (I'm not sure why you would, but I'm not going to stop you).
+The standard vocabulary defines the valid statement syntax for Muss and parses syntax into special Rust Iterators which can be used to execute the statement.
+To declare your own vocabulary, use Interpretor::with or Interpreter::with_vocab with a custom vocabulary (I'm not sure why you would, but I'm not going to stop you).
 
 ### Oddities
-The MPS standard syntax does a few things that most other languages don't, because I wanted it to.
+The Muss standard syntax does a few things that most other languages don't, because I wanted it to.
 
 \` can be used in place of " -- To make it easier to write SQL, string literals may be surrounded by backticks instead of quotation marks.
 
-; -- The REPL will automatically place semicolons when Enter is pressed and it's not inside of brackets or a literal. MPS requires semicolons at the end of every statement, though, so MPS files must use semicolons.
+; -- The REPL will automatically place semicolons when Enter is pressed and it's not inside of brackets or a literal. Muss requires semicolons at the end of every statement, though, so Muss files must use semicolons.
 
 ### Filters
 Operations to reduce the items in an iterable: `iterable.(filter);`.
 Filters are statements of the format `something.(predicate)`, where "something" is a variable name or another valid statement, and "predicate" is a valid filter predicate (see below).
 E.g. `files(folder="~/Music/", recursive=true).(title == "Romantic Traffic");` is valid filter syntax to filter all songs in the Music folder for songs named "Romantic Traffic" (probably just one song).
 
-#### field == something
+#### .field == something
 
-#### field like something
+#### .field like something
 
-#### field matches some_regex
+#### .field unlike something
 
-#### field != something
+#### .field matches some_regex
 
-#### field >= something
+#### .field != something
 
-#### field > something
+#### .field >= something
 
-#### field <= something
+#### .field > something
 
-#### field < something -- e.g. `iterable.(title == "Romantic Traffic");`
+#### .field <= something
 
-Compare all items, keeping only those that match the condition. Valid field names are those of the MpsMusicItem (title, artist, album, genre, track, etc.), though this will change when proper object support is added. Optionally, a ? or ! can be added to the end of the field name to skip items whose field is missing/incomparable, or keep all items whose field is missing/incomparable (respectively).
+#### .field < something -- e.g. `iterable.(.title == "Romantic Traffic");`
+
+Compare all items, keeping only those that match the condition. Valid field names change depending on what information is available when the Item is populated, but usually title, artist, album, genre, track, filename are valid fields. Optionally, a ? or ! can be added to the end of the field name to skip items whose field is missing/incomparable, or keep all items whose field is missing/incomparable (respectively).
 
 
 #### start..end -- e.g. `iterable.(0..42);`
@@ -89,9 +88,18 @@ Matches all items
 
 Replace items matching the filter with operation1 and replace items not matching the filter with operation2. The `else operation2` part may be omitted to preserve items not matching the filter. To perform operations with the current item, use the special variable `item`. The replacement filter may not contain || -- instead, use multiple filters chained together.
 
+#### unique
+#### unique field -- e.g. `iterable.(unique .title);`
+
+Keep only items which are do not duplicate another item, or keep only items whoes specified field does not duplicate another item's same field. The first non-duplicated instance of an item is always the one that is kept.
+
+#### ?? -- e.g. `iterable.(??);`
+
+Keep only the items that contain at least one field (not including the filename field).
+
 ### Functions
 Similar to most other languages: `function_name(param1, param2, etc.);`.
-These always return an iterable which can be manipulated.
+These always return an iterable which can be manipulated with other syntax (filters, sorters, etc.).
 Functions are statements of the format `function_name(params)`, where "function_name" is one of the function names (below) and params is a valid parameter input for the function.
 Each function is responsible for parsing it's own parameters when the statement is parsed, so this is very flexible.
 E.g. `files(folder="~/Music/", recursive=true);` is valid function syntax to execute the files function with parameters `folder="~/Music/", recursive=true`.
@@ -99,11 +107,11 @@ E.g. `files(folder="~/Music/", recursive=true);` is valid function syntax to exe
 
 #### sql_init(generate = true|false, folder = "path/to/music");
 
-Initialize the SQLite database connection using the provided parameters. This must be performed before any other database operation (otherwise the database will already be connected with default settings).
+Initialize the SQLite database connection using the provided parameters. This must be performed before any other database operation (otherwise the database will already be connected with default settings). This returns an empty iterable (contains zero items).
 
 #### sql("SQL query here");
 
-Perform a raw SQLite query on the database which MPS auto-generates. An iterator of the results is returned.
+Perform a raw SQLite query on the database which Muss auto-generates. An iterator of the results is returned.
 
 #### song("something");
 
@@ -129,65 +137,77 @@ Repeat the iterable count times, or infinite times if count is omitted.
 
 Retrieve all files from a folder, matching a regex pattern.
 
+#### mpd(address, term = value, term2 = value2, ...);
+
+Retrieve songs from a music player daemon at `address`. If compiled without the `music_library` feature, this is equivalent to `empty()`.
+
 #### reset(iterable);
 
 Explicitly reset an iterable. This useful for reusing an iterable variable.
 
 #### interlace(iterable1, iterable2, ...);
 
-Combine multiple iterables in an interleaved pattern. This is a variant of union(...) where the first item in iterable1, then iterable2, ... is returned, then the second item, etc. until all iterables are depleted. There is no limit on the amount of iterables which can be provided as parameters.
+Combine multiple iterables in an interleaved pattern. This is a variant of union(...) where the first item in iterable1, then iterable2, ... is returned, then the second item, etc. until all iterables are depleted. There is no limit to the amount of iterables which can be provided as parameters.
 
 #### union(iterable1, iterable2, ...);
 
-Combine multiple iterables in a sequential pattern. All items in iterable1 are returned, then all items in iterable2, ... until all provided iterables are depleted. There is no limit on the amount of iterables which can be provided as parameters.
+Combine multiple iterables in a sequential pattern. All items in iterable1 are returned, then all items in iterable2, ... until all provided iterables are depleted. There is no limit to the amount of iterables which can be provided as parameters.
 
 #### intersection(iterable1, iterable2, ...);
 
-Combine multiple iterables such that only items that exist in iterable1 and iterable2 and ... are returned. The order of items from iterable1 is maintained. There is no limit on the amount of iterables which can be provided as parameters.
+Combine multiple iterables such that only items that exist in iterable1 and iterable2 and ... are returned. The order of items from iterable1 is maintained. There is no limit to the amount of iterables which can be provided as parameters.
 
 #### empty();
 
-Empty iterator. Useful for deleting items using replacement filters.
+Empty iterator containing zero items. Useful for deleting items using replacement filters.
+
+#### empties(count);
+
+Iterate over count empty items. The items in this iterator have no fields (i.e. are empty).
 
 ### Sorters
-Operations to sort the items in an iterable: iterable~(sorter) OR iterable.sort(sorter)
+Operations to sort the items in an iterable: `iterable~(sorter)`.
 
-#### field -- e.g. iterable~(filename)
+#### .field -- e.g. `iterable~(.filename);`
 
-Sort by a MpsItem field. Valid field names change depending on what information is available when the MpsItem is populated, but usually title, artist, album, genre, track, filename are valid fields. Items with a missing/incomparable fields will be sorted to the end.
+Sort by an Item field. Valid field names change depending on what information is available when the Item is populated, but usually title, artist, album, genre, track, filename are valid fields. Items with a missing/incomparable fields will be sorted to the end.
 
-#### shuffle
-#### random shuffle -- e.g. iterable~(shuffle)
+#### ~shuffle
+#### random shuffle -- e.g. `iterable~(~shuffle);`
 
 Shuffle the songs in the iterator. This is random for up to 2^16 items, and then the randomness degrades (but at that point you won't notice). The more verbose syntax is allowed in preparation for future randomisation strategies.
 
-#### advanced bliss_first -- e.g. iterable~(advanced bliss_first)
+#### ~radio
+#### ~radio qualifier -- e.g. `iterable~(~radio)`
+Sort by musical similarity, starting with a random first song from the iterator. The optional qualifier may be `chroma`, `loudness`, `spectrum`, or `tempo`. When the qualifier is omitted, they are all considered for comparing audio similarity.
+
+#### advanced bliss_first -- e.g. `iterable~(advanced bliss_first);`
 
 Sort by the distance (similarity) from the first song in the iterator. Songs which are more similar (lower distance) to the first song in the iterator will be placed closer to the first song, while less similar songs will be sorted to the end. This uses the [bliss music analyser](https://github.com/polochon-street/bliss-rs), which is a very slow operation and can cause music playback interruptions for large iterators. This requires the `advanced` feature to be enabled (without the feature enabled this is still valid syntax but doesn't change the order).
 
-#### advanced bliss_next -- e.g. iterable~(advanced bliss_next)
+#### advanced bliss_next -- e.g. `iterable~(advanced bliss_next);`
 
 Sort by the distance (similarity) between the last played song in the iterator. Similar to bliss_first. The song which is the most similar (lower distance) to the previous song in the iterator will be placed next to it, then the process is repeated. This uses the [bliss music analyser](https://github.com/polochon-street/bliss-rs), which is a very slow operation and can cause music playback interruptions for large iterators. This requires the `advanced` feature to be enabled (without the feature enabled this is still valid syntax but doesn't change the order).
 
 ### Procedures
-Operations to apply to each item in an iterable: iterable.{step1, step2, ...}
+Operations to apply to each item in an iterable: `iterable.{step1, step2, ...}`;
 
 Comma-separated procedure steps will be executed sequentially (like a for loop in regular programming languages). The variable item contains the current item of the iterable.
 
-#### let variable = something -- e.g. let my_var = 42
+#### let variable = something -- e.g. `let my_var = 42,`
 
 Declare the variable and (optionally) set the initial value to something. The assignment will only be performed when the variable has not yet been declared. When the initial value (and equals sign) is omitted, the variable is initialized as empty().
 
-#### variable = something -- e.g. my_var = 42
+#### variable = something -- e.g. `my_var = 42,`
 
 Assign something to the variable. The variable must have already been declared.
 
-#### empty() -- e.g. empty()
+#### empty() -- e.g. `empty(),`
 
 The empty or null constant.
 
 #### if condition { something } else { something_else } -- e.g.
-```mps
+```muss
 if item.title == `Romantic Traffic` {
 
    } else {
@@ -202,11 +222,11 @@ if item.title == `Romantic Traffic` {
 #### something1 >= something2
 #### something1 > something2
 #### something1 <= something2
-#### something1 < something2 -- e.g. item.filename != item.title
+#### something1 < something2 -- e.g. `item.filename != item.title,`
 
 Compare something1 to something2. The result is a boolean which is useful for branch conditions.
 
-#### op iterable_operation -- e.g. op files().(0..=42)~(shuffle)
+#### op iterable_operation -- e.g. `op files().(0..=42)~(shuffle),`
 
 An iterable operation inside of the procedure. When assigned to item, this can be used to replace item with multiple others. Note that iterable operations are never executed inside the procedure; when item is iterable, it will be executed immediately after the end of the procedure for the current item.
 
@@ -215,16 +235,20 @@ An iterable operation inside of the procedure. When assigned to item, this can b
 #### something1 - something2
 #### something1 + something2
 #### something1 || something2
-#### something1 && something2 -- e.g. 42 + (128 - 64)
+#### something1 && something2 -- e.g. `42 + (128 - 64),`
 
 Various algebraic operations: brackets (order of operations), negation, subtraction, addition, logical OR, logical AND; respectively.
 
-#### Item(field1 = something1, field2 = something2, ...) - e.g. item = Item(title = item.title, filename = `/dev/null`)
+#### Item(field1 = something1, field2 = something2, ...) - e.g. `item = Item(title = item.title, filename = "/dev/null"),`
 
    Constructor for a new item. Each function parameter defines a new field and it's value.
-#### ~`string_format` something -- e.g. ~`{filename}` item
+#### ~"string_format" something -- e.g. `~"{filename}" item,`
 
    Format a value into a string. This behaves differently depending on the value's type: When the value is an Item, the item's corresponding field will replace all `{field}` instances in the format string. When the value is a primitive type (String, Int, Bool, etc.), the value's text equivalent will replace all `{}` instances in the format string. When the value is an iterable operation (Op), the operation's script equivalent will replace all `{}` instances in the format string.
 
+#### file(filepath) -- e.g. `file("~/Music/Romantic Traffic.flac"),`
+
+Load an item from file, populating the item with the song's tags.
+
 
 License: LGPL-2.1-only OR GPL-3.0-only

interpreter/benches/file_parse.rs

@@ -0,0 +1,58 @@
+use muss_interpreter::Interpreter;
+//use mps_interpreter::MpsRunner;
+use std::fs::File;
+use std::io::{BufReader, Read, Seek};
+
+use criterion::{criterion_group, criterion_main, Criterion};
+
+/*fn interpretor_benchmark(c: &mut Criterion) {
+    let f = File::open("benches/lots_of_empty.mps").unwrap();
+    let mut reader = BufReader::with_capacity(1024 * 1024 /* 1 MiB */, f);
+    // read everything into buffer before starting
+    let mut buf = Vec::with_capacity(1024 * 1024);
+    reader.read_to_end(&mut buf).unwrap();
+    drop(buf);
+    c.bench_function("mps lots_of_empty.mps", |b| {
+        b.iter(|| {
+            //let f = File::open("benches/lots_of_empty.mps").unwrap();
+            //let mut reader = BufReader::new(f);
+            reader.rewind().unwrap();
+            let mps = MpsRunner::with_stream(&mut reader);
+            for item in mps {
+                match item {
+                    Err(e) => panic!("{}", e),
+                    Ok(_) => {}
+                }
+            }
+        })
+    });
+}*/
+
+fn faye_benchmark(c: &mut Criterion) {
+    let f = File::open("benches/lots_of_empty.muss").unwrap();
+    let mut reader = BufReader::with_capacity(1024 * 1024 /* 1 MiB */, f);
+    // read everything into buffer before starting
+    let mut buf = Vec::with_capacity(1024 * 1024);
+    reader.read_to_end(&mut buf).unwrap();
+    drop(buf);
+    c.bench_function("muss-faye lots_of_empty.muss", |b| {
+        b.iter(|| {
+            //let f = File::open("benches/lots_of_empty.mps").unwrap();
+            //let mut reader = BufReader::new(f);
+            reader.rewind().unwrap();
+            let mps = Interpreter::with_stream(&mut reader);
+            for item in mps {
+                match item {
+                    Err(e) => panic!("{}", e),
+                    Ok(_) => {}
+                }
+            }
+        })
+    });
+}
+
+criterion_group!(
+    parse_benches,
+    /*interpretor_benchmark,*/ faye_benchmark
+);
+criterion_main!(parse_benches);

interpreter/fuzz/.gitignore

@@ -0,0 +1,3 @@
+target
+corpus
+artifacts

interpreter/fuzz/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "muss-interpreter-fuzz"
+version = "0.0.0"
+authors = ["Automatically generated"]
+publish = false
+edition = "2021"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+
+[dependencies.muss-interpreter]
+path = ".."
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[[bin]]
+name = "faye_fuzz"
+path = "fuzz_targets/faye_fuzz.rs"
+test = false
+doc = false

interpreter/fuzz/fuzz_targets/faye_fuzz.rs

@@ -0,0 +1,20 @@
+#![no_main]
+#[macro_use] extern crate libfuzzer_sys;
+extern crate muss_interpreter;
+
+fuzz_target!(|data: &[u8]| {
+    if let Ok(s) = std::str::from_utf8(data) {
+        print!("len:{},data:{}\n", data.len(), s)
+    } else {
+        print!("len:{},data:<non-ut8>,", data.len());
+    }
+    let mut cursor = std::io::Cursor::new(data);
+    let interpreter = muss_interpreter::Interpreter::with_stream(&mut cursor);
+    for item in interpreter {
+        match item {
+            Err(e) => print!("err:{},", e),
+            Ok(_i) => {},//print!("item:{},", i),
+        }
+    }
+    println!("done.");
+});

interpreter/src/context.rs

@@ -0,0 +1,48 @@
+#[cfg(feature = "advanced")]
+use super::processing::advanced::{DefaultAnalyzer, MusicAnalyzer};
+use super::processing::database::DatabaseQuerier;
+#[cfg(feature = "fakesql")]
+use super::processing::database::SQLiteTranspileExecutor;
+#[cfg(feature = "mpd")]
+use super::processing::database::{MpdExecutor, MpdQuerier};
+use super::processing::general::{
+    FilesystemExecutor, FilesystemQuerier, OpStorage, VariableStorer,
+};
+use std::fmt::{Debug, Display, Error, Formatter};
+
+#[derive(Debug)]
+pub struct Context {
+    pub database: Box<dyn DatabaseQuerier>,
+    pub variables: Box<dyn VariableStorer>,
+    pub filesystem: Box<dyn FilesystemQuerier>,
+    #[cfg(feature = "advanced")]
+    pub analysis: Box<dyn MusicAnalyzer>,
+    #[cfg(feature = "mpd")]
+    pub mpd_database: Box<dyn MpdQuerier>,
+}
+
+impl Default for Context {
+    fn default() -> Self {
+        Self {
+            #[cfg(feature = "fakesql")]
+            database: Box::new(SQLiteTranspileExecutor::default()),
+            #[cfg(all(feature = "sql", not(feature = "fakesql")))]
+            database: Box::new(super::processing::database::SQLiteExecutor::default()),
+            #[cfg(all(not(feature = "sql"), not(feature = "fakesql")))]
+            database: Box::new(super::processing::database::SQLErrExecutor::default()),
+            variables: Box::new(OpStorage::default()),
+            filesystem: Box::new(FilesystemExecutor::default()),
+            #[cfg(feature = "advanced")]
+            analysis: Box::new(DefaultAnalyzer::default()),
+            #[cfg(feature = "mpd")]
+            mpd_database: Box::new(MpdExecutor::default()),
+        }
+    }
+}
+
+impl Display for Context {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "Context{{...}}")?;
+        Ok(())
+    }
+}

interpreter/src/debug.rs

@@ -0,0 +1,41 @@
+use std::iter::Iterator;
+
+use super::tokens::TokenReader;
+use super::{Interpreter, InterpreterItem};
+
+/// Wrapper for InterpreterError with a built-in callback function for every iteration of the interpreter.
+pub struct Debugger<'a, T, F>
+where
+    T: TokenReader,
+    F: Fn(&mut Interpreter<'a, T>, Option<InterpreterItem>) -> Option<InterpreterItem>,
+{
+    interpreter: Interpreter<'a, T>,
+    transmuter: F,
+}
+
+impl<'a, T, F> Debugger<'a, T, F>
+where
+    T: TokenReader,
+    F: Fn(&mut Interpreter<'a, T>, Option<InterpreterItem>) -> Option<InterpreterItem>,
+{
+    /// Create a new instance of Debugger using the provided interpreter and callback.
+    pub fn new(faye: Interpreter<'a, T>, item_handler: F) -> Self {
+        Self {
+            interpreter: faye,
+            transmuter: item_handler,
+        }
+    }
+}
+
+impl<'a, T, F> Iterator for Debugger<'a, T, F>
+where
+    T: TokenReader,
+    F: Fn(&mut Interpreter<'a, T>, Option<InterpreterItem>) -> Option<InterpreterItem>,
+{
+    type Item = InterpreterItem;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let next_item = self.interpreter.next();
+        (self.transmuter)(&mut self.interpreter, next_item)
+    }
+}

interpreter/src/errors.rs

@@ -0,0 +1,50 @@
+use std::convert::From;
+use std::fmt::{Debug, Display, Error, Formatter};
+
+use crate::lang::{LanguageError, RuntimeError, SyntaxError};
+use crate::tokens::ParseError;
+
+#[derive(Debug)]
+pub enum InterpreterError {
+    Syntax(SyntaxError),
+    Runtime(RuntimeError),
+    Parse(ParseError),
+}
+
+impl Display for InterpreterError {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        match self {
+            Self::Syntax(e) => (e as &dyn Display).fmt(f),
+            Self::Runtime(e) => (e as &dyn Display).fmt(f),
+            Self::Parse(e) => (e as &dyn Display).fmt(f),
+        }
+    }
+}
+
+impl LanguageError for InterpreterError {
+    fn set_line(&mut self, line: usize) {
+        match self {
+            Self::Syntax(e) => e.set_line(line),
+            Self::Runtime(e) => e.set_line(line),
+            Self::Parse(e) => e.set_line(line),
+        }
+    }
+}
+
+impl From<SyntaxError> for InterpreterError {
+    fn from(e: SyntaxError) -> Self {
+        Self::Syntax(e)
+    }
+}
+
+impl From<RuntimeError> for InterpreterError {
+    fn from(e: RuntimeError) -> Self {
+        Self::Runtime(e)
+    }
+}
+
+impl From<ParseError> for InterpreterError {
+    fn from(e: ParseError) -> Self {
+        Self::Parse(e)
+    }
+}

interpreter/src/faye.rs

@@ -0,0 +1,210 @@
+use std::collections::VecDeque;
+use std::io::Read;
+use std::iter::Iterator;
+
+use super::lang::{LanguageDictionary, LanguageError, Op};
+use super::tokens::{Token, TokenReader, Tokenizer};
+use super::Context;
+use super::InterpreterError;
+use super::Item;
+
+const DEFAULT_TOKEN_BUFFER_SIZE: usize = 16;
+
+pub enum InterpreterEvent {
+    FileEnd,
+    StatementComplete,
+    NewStatementReady,
+}
+
+/// The script interpreter.
+pub struct Interpreter<'a, T>
+where
+    T: TokenReader,
+{
+    tokenizer: T,
+    buffer: VecDeque<Token>,
+    current_stmt: Box<dyn Op>,
+    vocabulary: LanguageDictionary,
+    callback: &'a dyn Fn(&mut Interpreter<'a, T>, InterpreterEvent) -> Result<(), InterpreterError>,
+}
+
+#[inline]
+fn empty_callback<T: TokenReader>(
+    _s: &mut Interpreter<'_, T>,
+    _d: InterpreterEvent,
+) -> Result<(), InterpreterError> {
+    Ok(())
+}
+
+/*impl <T> Interpreter<'static, T>
+where
+    T: TokenReader,
+{
+    /// Create a new interpreter for the provided token reader, using the standard MPS language.
+    #[inline]
+    pub fn with_standard_vocab(token_reader: T) -> Self {
+        let mut vocab = LanguageDictionary::default();
+        super::interpretor::standard_vocab(&mut vocab);
+        Self::with_vocab(vocab, token_reader)
+    }
+
+    /// Create a new interpreter with the provided vocabulary and token reader.
+    #[inline]
+    pub fn with_vocab(vocab: LanguageDictionary, token_reader: T) -> Self {
+        Self::with(vocab, token_reader, &empty_callback)
+    }
+}*/
+
+impl<'a, R: Read> Interpreter<'a, Tokenizer<R>> {
+    pub fn with_stream(stream: R) -> Self {
+        let tokenizer = Tokenizer::new(stream);
+        Self::with_standard_vocab(tokenizer)
+    }
+
+    pub fn with_stream_and_callback(
+        stream: R,
+        callback: &'a dyn Fn(
+            &mut Interpreter<'a, Tokenizer<R>>,
+            InterpreterEvent,
+        ) -> Result<(), InterpreterError>,
+    ) -> Self {
+        let tokenizer = Tokenizer::new(stream);
+        let vocab = LanguageDictionary::standard();
+        Self::with(vocab, tokenizer, callback)
+    }
+}
+
+impl<'a, T> Interpreter<'a, T>
+where
+    T: TokenReader,
+{
+    #[inline]
+    pub fn with_standard_vocab(token_reader: T) -> Self {
+        let vocab = LanguageDictionary::standard();
+        Self::with_vocab(vocab, token_reader)
+    }
+
+    /// Create a new interpreter with the provided vocabulary and token reader.
+    #[inline]
+    pub fn with_vocab(vocab: LanguageDictionary, token_reader: T) -> Self {
+        Self::with(vocab, token_reader, &empty_callback)
+    }
+
+    /// Create a custom interpreter instance.
+    #[inline]
+    pub fn with(
+        vocab: LanguageDictionary,
+        token_reader: T,
+        callback: &'a dyn Fn(
+            &mut Interpreter<'a, T>,
+            InterpreterEvent,
+        ) -> Result<(), InterpreterError>,
+    ) -> Self {
+        Self {
+            tokenizer: token_reader,
+            buffer: VecDeque::with_capacity(DEFAULT_TOKEN_BUFFER_SIZE),
+            current_stmt: Box::new(crate::lang::vocabulary::empty::EmptyStatement {
+                context: Some(Context::default()),
+            }),
+            vocabulary: vocab,
+            callback: callback,
+        }
+    }
+
+    // build a new statement
+    #[inline]
+    fn new_statement(&mut self) -> Option<Result<Box<dyn Op>, InterpreterError>> {
+        while !self.tokenizer.end_of_file() && self.buffer.is_empty() {
+            let result = self.tokenizer.next_statement(&mut self.buffer);
+            match result {
+                Ok(_) => {}
+                Err(e) => return Some(Err(error_with_ctx(e, self.tokenizer.current_line()))),
+            }
+        }
+        if self.buffer.is_empty() {
+            let callback_result = (self.callback)(self, InterpreterEvent::FileEnd);
+            match callback_result {
+                Ok(_) => {}
+                Err(e) => return Some(Err(e)),
+            }
+            return None;
+        }
+        let result = self.vocabulary.try_build_statement(&mut self.buffer);
+        let stmt = match result {
+            Ok(stmt) => stmt,
+            Err(e) => return Some(Err(error_with_ctx(e, self.tokenizer.current_line()))),
+        };
+        //println!("Final parsed op: {}", stmt);
+        #[cfg(debug_assertions)]
+        if !self.buffer.is_empty() {
+            panic!("Token buffer was not emptied! (rem: {:?})", self.buffer)
+        }
+        Some(Ok(stmt))
+    }
+}
+
+pub type InterpreterItem = Result<Item, InterpreterError>;
+
+impl<'a, T> Iterator for Interpreter<'a, T>
+where
+    T: TokenReader,
+{
+    type Item = InterpreterItem;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            match self.current_stmt.next() {
+                Some(item) => {
+                    return Some(item.map_err(|e| error_with_ctx(e, self.tokenizer.current_line())))
+                }
+                None => {
+                    // current_stmt has terminated
+                    if self.tokenizer.end_of_file() {
+                        // always try to read at least once, in case stream gets new data (e.g. in a REPL)
+                        let result = self.tokenizer.next_statement(&mut self.buffer);
+                        match result {
+                            Ok(_) => {}
+                            Err(e) => {
+                                return Some(Err(error_with_ctx(e, self.tokenizer.current_line())))
+                            }
+                        }
+                    } else {
+                        // notify old statement is complete
+                        let callback_result =
+                            (self.callback)(self, InterpreterEvent::StatementComplete);
+                        match callback_result {
+                            Ok(_) => {}
+                            Err(e) => return Some(Err(e)),
+                        }
+                    }
+                    // build next statement
+                    let result = self.new_statement();
+                    let mut stmt = match result {
+                        Some(Ok(stmt)) => stmt,
+                        Some(Err(e)) => return Some(Err(e)),
+                        None => return None,
+                    };
+                    let ctx = self.current_stmt.escape();
+                    stmt.enter(ctx);
+                    self.current_stmt = stmt;
+                    // notify new statement is ready
+                    let callback_result =
+                        (self.callback)(self, InterpreterEvent::NewStatementReady);
+                    match callback_result {
+                        Ok(_) => {}
+                        Err(e) => return Some(Err(e)),
+                    }
+                }
+            }
+        }
+    }
+}
+
+fn error_with_ctx<T: std::convert::Into<InterpreterError>>(
+    error: T,
+    line: usize,
+) -> InterpreterError {
+    let mut err = error.into();
+    err.set_line(line);
+    err
+}

interpreter/src/interpretor.rs

@@ -0,0 +1,71 @@
+use super::lang::LanguageDictionary;
+
+/// Builder function to add the standard statements parsers for MPS interpreters.
+pub(crate) fn standard_vocab(vocabulary: &mut LanguageDictionary) {
+    vocabulary
+        // filters
+        .add_transform(crate::lang::vocabulary::filters::empty_filter())
+        .add_transform(crate::lang::vocabulary::filters::range_filter())
+        .add_transform(
+            // accepts any .(.something)
+            crate::lang::vocabulary::filters::field::FieldFilterBlockFactory::new()
+                .push(crate::lang::vocabulary::filters::field::FieldFilterComparisonFactory)
+                .push(crate::lang::vocabulary::filters::field::FieldFilterMaybeFactory)
+                .push(crate::lang::vocabulary::filters::field::FieldLikeFilterFactory)
+                .push(crate::lang::vocabulary::filters::field::FieldRegexFilterFactory)
+                .to_statement_factory(),
+        )
+        .add_transform(crate::lang::vocabulary::filters::unique_field_filter())
+        .add_transform(crate::lang::vocabulary::filters::unique_filter())
+        .add_transform(crate::lang::vocabulary::filters::nonempty_filter())
+        .add_transform(crate::lang::vocabulary::filters::index_filter())
+        // sorters
+        .add_transform(crate::lang::vocabulary::sorters::empty_sort())
+        .add_transform(crate::lang::vocabulary::sorters::shuffle_sort()) // accepts ~(~shuffle)
+        .add_transform(crate::lang::vocabulary::sorters::bliss_sort())
+        .add_transform(crate::lang::vocabulary::sorters::bliss_next_sort())
+        .add_transform(crate::lang::vocabulary::sorters::radio_sort())
+        .add_transform(crate::lang::vocabulary::sorters::field_sort()) // accepts any ~(.name)
+        // iter blocks
+        .add_transform(
+