Fix playlist filepath generation to ignore already-absolute paths

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: NGnius
+liberapay: NGnius

.gitignore

@@ -1,5 +1,5 @@
 /target
 **/target
-/*/metadata.mps.sqlite
-metadata.mps.sqlite
+/*/metadata.muss.sqlite
+metadata.muss.sqlite
 **.m3u8

.gitmodules

@@ -4,3 +4,5 @@
 [submodule "bliss-rs"]
 	path = bliss-rs
 	url = https://github.com/NGnius/bliss-rs
+[submodule "bliss-rs/"]
+	url = https://git.ngni.us/NGnius/bliss-rs

Cargo.toml

@@ -1,30 +1,57 @@
 [package]
-name = "mps"
-version = "0.3.0"
+name = "muss"
+version = "0.9.0"
 edition = "2021"
 authors = ["NGnius (Graham) <ngniusness@gmail.com>"]
-description = "Music Playlist Scripting language (MPS)"
-license = "LGPL-2.1-only OR GPL-2.0-or-later"
-repository = "https://github.com/NGnius/mps"
+description = "Music Set Script language (MuSS)"
+license = "LGPL-2.1-only OR GPL-3.0-only"
+repository = "https://git.ngni.us/NGnius/muss"
 keywords = ["audio", "playlist", "scripting", "language"]
 readme = "README.md"
+exclude = ["extras/"]
 
 [workspace]
 members = [
-    "mps-interpreter",
-    "mps-player"
+    "interpreter",
+    "player",
+    "m3u8"
 ]
 
 [dependencies]
 # local
-mps-interpreter = { version = "0.3.0", path = "./mps-interpreter" }
+muss-interpreter = { version = "0.9.0", path = "./interpreter" }
 # external
 clap = { version = "3.0", features = ["derive"] }
-# termios = { version = "^0.3"}
+console = { version = "0.15" }
+lazy_static = { version = "1.4" }
+
+# cli add to playlist functionality
+m3u8-rs = { version = "^3.0.0" }
 
 [target.'cfg(not(target_os = "linux"))'.dependencies]
-mps-player = { version = "0.3.0", path = "./mps-player", default-features = false }
+muss-player = { version = "0.9.0", path = "./player", default-features = false, features = ["mpd"] }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 # TODO fix need to specify OS-specific dependency of mps-player
-mps-player = { version = "0.3.0", path = "./mps-player", features = ["mpris-player"] }
+muss-player = { version = "0.9.0", path = "./player", features = ["mpris-player", "mpd"] }
+
+[profile.release]
+debug = false
+strip = true
+lto = true
+codegen-units = 4
+
+[profile.bench]
+lto = false
+
+[profile.dev.package.bliss-audio-symphonia]
+debug-assertions = false
+overflow-checks = false
+debug = true
+opt-level = 3
+
+[profile.dev.package."*"]
+debug-assertions = false
+overflow-checks = false
+debug = true
+opt-level = 3

README.md

@@ -1,33 +1,61 @@
-# mps
+# muss
 
-A language all about iteration to play your music files.
-This project implements the interpreter (mps-interpreter), music player (mps-player), and CLI interface for MPS (root).
+![repl_demo](https://raw.githubusercontent.com/NGnius/muss/master/extras/demo.png)
+
+Sort, filter and analyse your music to create great playlists.
+This project implements the interpreter (`./interpreter`), music player (`./player`), and CLI interface for Muss (`./`).
 The CLI interface includes a REPL for running scripts.
-The REPL interactive mode also provides more details about using MPS through the `?help` command.
+The REPL interactive mode also provides more details about using Muss through the `?help` command.
 
 ## Usage
-To access the REPL, simply run `cargo run`. You will need the [Rust toolchain installed](https://rustup.rs/).
+To access the REPL, simply run `cargo run`. You will need the [Rust toolchain installed](https://rustup.rs/). For a bit of extra performance, run `cargo run --release` instead.
 
 ## Examples
-For now, check out `./src/tests`, `./mps-player/tests`, and `./mps-interpreter/tests` for examples.
+
+### One-liners
+
+All songs by artist `<artist>` (in your library), sorted by similarity to a random first song by the artist.
+```muss
+files().(.artist? like "<artist>")~(~radio);
+```
+
+All songs with a `.flac` file extension (anywhere in their path -- not necessarily at the end).
+```muss
+files().(.filename? like ".flac");
+```
+
+All songs by artist `<artist1>` or `<artist2>`, sorted by similarity to a random first song by either artist.
+```muss
+files().(.artist? like "<artist1>" || .artist? like "<artist2>")~(~radio);
+```
+
+### Bigger examples
+
+For now, check out `./src/tests`, `./player/tests`, and `./interpreter/tests` for examples.
 One day I'll add pretty REPL example pictures and some script files...
 // TODO
 
 ## FAQ
-### Is MPS Turing-Complete?
-**No**. It can't perform arbitrary calculations (yet), which easily disqualifies MPS from being Turing-complete.
 
-### Can I use MPS right now?
-**Sure!** It's not complete, but MPS is completely useable for basic music queries right now. Hopefully most of the bugs have been ironed out as well...
+### Can I use Muss right now?
+**Sure!** It's never complete, but Muss is completely useable right now. Hopefully most of the bugs have been ironed out as well :)
 
 ### Why write a new language?
-**I thought it would be fun**. I also wanted to be able to play my music without having to be at the whim of someone else's algorithm (and music), and playing just by album or artist was getting boring. I also thought designing a language specifically for iteration would be a novel approach to a language (though every approach is a novel approach for me).
+**I thought it would be fun**. I also wanted to be able to play my music without having to be at the whim of someone else's algorithm (and music), and playing just by album or artist was getting boring. Designing a language specifically for iteration seemed like a cool & novel way of doing it, too (though every approach is a novel approach for me).
 
-### What is MPS?
-**Music Playlist Script (MPS) is technically a query language for music files.** It uses an (auto-generated) SQLite3 database for SQL queries and can also directly query the filesystem. Queries can be modified by using filters, functions, and sorters built-in to MPS (see mps-interpreter's README.md).
+### What is Muss?
+**Music Set Script (MuSS) is a language for describing a playlist of music.** It can execute SQLite select clauses or directly query the filesystem. Queries can be modified by using filters, functions, and sorters built-in to Muss (see interpreter's README.md).
 
-### Is MPS a scripting language?
-**No**. Technically, it was designed to be one, but it doesn't meet the requirements of a scripting language (yet). One day, I would like it be Turing-complete and then it could be considered a scripting language. At the moment it is barely a query language.
+### Is Muss a scripting language?
+**Yes**. It evolved from a simple query language into something that can do arbitrary calculations. Whether it's Turing-complete is still unproven, but it's powerful enough for what I want it to do.
 
 
-License: LGPL-2.1-only OR GPL-2.0-or-later
+## License
+
+LGPL-2.1-only OR GPL-3.0-only
+
+**NOTE**: Advanced features make use of [a fork of bliss-rs](https://github.com/NGnius/bliss-rs) a GPL-3.0 licensed music analysis library.
+
+## Contribution
+
+This is a hobby project, so any contribution may take a while to be acknowledged and accepted.

README.tpl

@@ -0,0 +1,15 @@
+# {{crate}}
+
+![repl_demo](https://raw.githubusercontent.com/NGnius/muss/master/extras/demo.png)
+
+{{readme}}
+
+## License
+
+{{license}}
+
+**NOTE**: Advanced features make use of [a fork of bliss-rs](https://github.com/NGnius/bliss-rs) a GPL-3.0 licensed music analysis library.
+
+## Contribution
+
+This is a hobby project, so any contribution may take a while to be acknowledged and accepted.

bliss-rs

@@ -1 +1 @@
-Subproject commit 637cf29d556bc44b6f7d8aa78430f02759560a63
+Subproject commit 2dcdd5643bc0cd055887128637abbb9bed041f77

interpreter/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "muss-interpreter"
+version = "0.9.0"
+edition = "2021"
+license = "LGPL-2.1-only OR GPL-3.0-only"
+readme = "README.md"
+repository = "https://git.ngni.us/NGnius/muss"
+rust-version = "1.59"
+
+[dependencies]
+rusqlite = { version = "0.27", features = ["bundled"], optional = true }
+sqlparser = { version = "0.23", optional = true }
+symphonia = { version = "0.5", optional = true, features = ["all"] }
+dirs = { version = "4" }
+regex = { version = "1" }
+rand = { version = "0.8" }
+shellexpand = { version = "2", optional = true }
+bliss-audio-symphonia = { version = "0.6", optional = true, path = "../bliss-rs" }
+mpd = { version = "0.1", optional = true }
+unidecode = { version = "0.3.0", optional = true }
+base64 = { version = "0.13", optional = true }
+m3u8-rs = { version = "3.0.0", optional = true }
+
+[dev-dependencies]
+criterion = "0.3"
+
+[[bench]]
+name = "file_parse"
+harness = false
+
+[features]
+default = [ "music_library", "ergonomics", "advanced", "advanced-bliss", "fakesql", "collections" ]
+music_library = [ "symphonia", "mpd", "base64" ] # song metadata parsing and database auto-population
+collections = [ "m3u8-rs" ] # read from m3u8 playlists (and other song collections, eventually)
+ergonomics = ["shellexpand", "unidecode"] # niceties like ~ in paths and unicode string sanitisation
+advanced = [] # advanced language features like music analysis
+advanced-bliss = ["bliss-audio-symphonia"] # bliss audio analysis
+sql = [ "rusqlite" ] # sqlite database for music
+fakesql = [ "sqlparser" ] # transpiled sqlite interpreter

interpreter/README.md

@@ -0,0 +1,254 @@
+# muss-interpreter
+
+All necessary components to interpret and run a Muss script.
+
+Interpreter is the Muss script interpreter.
+Debugger can be used to run scripts within a custom debug harness.
+Since Muss is centered around iterators, script execution is also done by iterating.
+
+
+```rust
+use std::io::Cursor;
+use muss_interpreter::*;
+
+let cursor = Cursor::new(
+"files(folder=`~/Music/`, recursive=true)" // retrieve all files from Music folder
+);
+
+let interpreter = Interpreter::with_stream(cursor);
+
+// warning: my library has ~3800 songs, so this outputs too much information to be useful.
+for result in interpreter {
+    match result {
+        Ok(item) => println!("Got song `{}` (file: `{}`)", item.field("title").unwrap(), item.field("filename").unwrap()),
+        Err(e) => panic!("Got error while executing: {}", e),
+    }
+}
+```
+
+## Standard Vocabulary
+By default, the standard vocabulary is used to parse the stream when iterating the interpreter.
+The standard vocabulary defines the valid statement syntax for Muss and parses syntax into special Rust Iterators which can be used to execute the statement.
+To declare your own vocabulary, use Interpretor::with or Interpreter::with_vocab with a custom vocabulary (I'm not sure why you would, but I'm not going to stop you).
+
+### Oddities
+The Muss standard syntax does a few things that most other languages don't, because I wanted it to.
+
+\` can be used in place of " -- To make it easier to write SQL, string literals may be surrounded by backticks instead of quotation marks.
+
+; -- The REPL will automatically place semicolons when Enter is pressed and it's not inside of brackets or a literal. Muss requires semicolons at the end of every statement, though, so Muss files must use semicolons.
+
+### Filters
+Operations to reduce the items in an iterable: `iterable.(filter);`.
+Filters are statements of the format `something.(predicate)`, where "something" is a variable name or another valid statement, and "predicate" is a valid filter predicate (see below).
+E.g. `files(folder="~/Music/", recursive=true).(title == "Romantic Traffic");` is valid filter syntax to filter all songs in the Music folder for songs named "Romantic Traffic" (probably just one song).
+
+#### .field == something
+
+#### .field like something
+
+#### .field unlike something
+
+#### .field matches some_regex
+
+#### .field != something
+
+#### .field >= something
+
+#### .field > something
+
+#### .field <= something
+
+#### .field < something -- e.g. `iterable.(.title == "Romantic Traffic");`
+
+Compare all items, keeping only those that match the condition. Valid field names change depending on what information is available when the Item is populated, but usually title, artist, album, genre, track, filename are valid fields. Optionally, a ? or ! can be added to the end of the field name to skip items whose field is missing/incomparable, or keep all items whose field is missing/incomparable (respectively).
+
+
+#### start..end -- e.g. `iterable.(0..42);`
+
+Keep only the items that are at the start index up to the end index. Start and/or end may be omitted to start/stop at the iterable's existing start/end (respectively). This stops once the end condition is met, leaving the rest of the iterator unconsumed.
+
+#### start..=end -- e.g. `iterable.(0..=42);`
+
+Keep only the items that are at the start index up to and including the end index. Start may be omitted to start at the iterable's existing start. This stops once the end condition is met, leaving the rest of the iterator unconsumed.
+
+#### index -- e.g. `iterable.(4);`
+
+Keep only the item at the given index. This stops once the index is reached, leaving the rest of the iterator unconsumed.
+
+#### predicate1 || predicate2 -- e.g. `iterable.(4 || 5);`
+
+Keep only the items that meet the criteria of predicate1 or predicate2. This will always consume the full iterator.
+
+#### [empty] -- e.g. `iterable.();`
+
+Matches all items
+
+#### if filter: operation1 else operation2 -- e.g. `iterable.(if title == "Romantic Traffic": repeat(item, 2) else item.());`
+
+Replace items matching the filter with operation1 and replace items not matching the filter with operation2. The `else operation2` part may be omitted to preserve items not matching the filter. To perform operations with the current item, use the special variable `item`. The replacement filter may not contain || -- instead, use multiple filters chained together.
+
+#### unique
+#### unique field -- e.g. `iterable.(unique .title);`
+
+Keep only items which are do not duplicate another item, or keep only items whoes specified field does not duplicate another item's same field. The first non-duplicated instance of an item is always the one that is kept.
+
+#### ?? -- e.g. `iterable.(??);`
+
+Keep only the items that contain at least one field (not including the filename field).
+
+### Functions
+Similar to most other languages: `function_name(param1, param2, etc.);`.
+These always return an iterable which can be manipulated with other syntax (filters, sorters, etc.).
+Functions are statements of the format `function_name(params)`, where "function_name" is one of the function names (below) and params is a valid parameter input for the function.
+Each function is responsible for parsing it's own parameters when the statement is parsed, so this is very flexible.
+E.g. `files(folder="~/Music/", recursive=true);` is valid function syntax to execute the files function with parameters `folder="~/Music/", recursive=true`.
+
+
+#### sql_init(generate = true|false, folder = "path/to/music");
+
+Initialize the SQLite database connection using the provided parameters. This must be performed before any other database operation (otherwise the database will already be connected with default settings). This returns an empty iterable (contains zero items).
+
+#### sql("SQL query here");
+
+Perform a raw SQLite query on the database which Muss auto-generates. An iterator of the results is returned.
+
+#### song("something");
+
+Retrieve all songs in the database with a title like something.
+
+#### album("something");
+
+Retrieve all songs in the database with an album title like something.
+
+#### artist("something");
+
+Retrieve all songs in the database with an artist name like something.
+
+#### genre("something");
+
+Retrieve all songs in the database with a genre title like something.
+
+#### repeat(iterable, count);
+
+Repeat the iterable count times, or infinite times if count is omitted.
+
+#### files(folder = "path/to/music", recursive = true|false, regex = "pattern");
+
+Retrieve all files from a folder, matching a regex pattern.
+
+#### mpd(address, term = value, term2 = value2, ...);
+
+Retrieve songs from a music player daemon at `address`. If compiled without the `music_library` feature, this is equivalent to `empty()`.
+
+#### reset(iterable);
+
+Explicitly reset an iterable. This useful for reusing an iterable variable.
+
+#### interlace(iterable1, iterable2, ...);
+
+Combine multiple iterables in an interleaved pattern. This is a variant of union(...) where the first item in iterable1, then iterable2, ... is returned, then the second item, etc. until all iterables are depleted. There is no limit to the amount of iterables which can be provided as parameters.
+
+#### union(iterable1, iterable2, ...);
+
+Combine multiple iterables in a sequential pattern. All items in iterable1 are returned, then all items in iterable2, ... until all provided iterables are depleted. There is no limit to the amount of iterables which can be provided as parameters.
+
+#### intersection(iterable1, iterable2, ...);
+
+Combine multiple iterables such that only items that exist in iterable1 and iterable2 and ... are returned. The order of items from iterable1 is maintained. There is no limit to the amount of iterables which can be provided as parameters.
+
+#### empty();
+
+Empty iterator containing zero items. Useful for deleting items using replacement filters.
+
+#### empties(count);
+
+Iterate over count empty items. The items in this iterator have no fields (i.e. are empty).
+
+### Sorters
+Operations to sort the items in an iterable: `iterable~(sorter)`.
+
+#### .field -- e.g. `iterable~(.filename);`
+
+Sort by an Item field. Valid field names change depending on what information is available when the Item is populated, but usually title, artist, album, genre, track, filename are valid fields. Items with a missing/incomparable fields will be sorted to the end.
+
+#### ~shuffle
+#### random shuffle -- e.g. `iterable~(~shuffle);`
+
+Shuffle the songs in the iterator. This is random for up to 2^16 items, and then the randomness degrades (but at that point you won't notice). The more verbose syntax is allowed in preparation for future randomisation strategies.
+
+#### ~radio
+#### ~radio qualifier -- e.g. `iterable~(~radio)`
+Sort by musical similarity, starting with a random first song from the iterator. The optional qualifier may be `chroma`, `loudness`, `spectrum`, or `tempo`. When the qualifier is omitted, they are all considered for comparing audio similarity.
+
+#### advanced bliss_first -- e.g. `iterable~(advanced bliss_first);`
+
+Sort by the distance (similarity) from the first song in the iterator. Songs which are more similar (lower distance) to the first song in the iterator will be placed closer to the first song, while less similar songs will be sorted to the end. This uses the [bliss music analyser](https://github.com/polochon-street/bliss-rs), which is a very slow operation and can cause music playback interruptions for large iterators. This requires the `advanced` feature to be enabled (without the feature enabled this is still valid syntax but doesn't change the order).
+
+#### advanced bliss_next -- e.g. `iterable~(advanced bliss_next);`
+
+Sort by the distance (similarity) between the last played song in the iterator. Similar to bliss_first. The song which is the most similar (lower distance) to the previous song in the iterator will be placed next to it, then the process is repeated. This uses the [bliss music analyser](https://github.com/polochon-street/bliss-rs), which is a very slow operation and can cause music playback interruptions for large iterators. This requires the `advanced` feature to be enabled (without the feature enabled this is still valid syntax but doesn't change the order).
+
+### Procedures
+Operations to apply to each item in an iterable: `iterable.{step1, step2, ...}`;
+
+Comma-separated procedure steps will be executed sequentially (like a for loop in regular programming languages). The variable item contains the current item of the iterable.
+
+#### let variable = something -- e.g. `let my_var = 42,`
+
+Declare the variable and (optionally) set the initial value to something. The assignment will only be performed when the variable has not yet been declared. When the initial value (and equals sign) is omitted, the variable is initialized as empty().
+
+#### variable = something -- e.g. `my_var = 42,`
+
+Assign something to the variable. The variable must have already been declared.
+
+#### empty() -- e.g. `empty(),`
+
+The empty or null constant.
+
+#### if condition { something } else { something_else } -- e.g.
+```muss
+if item.title == `Romantic Traffic` {
+
+   } else {
+       remove item
+   }
+```
+
+   Branch based on a boolean condition. Multiple comma-separated procedure steps may be supplied in the if and else branches. This does not currently support if else chains, but they can be nested to accomplish similar behaviour.
+
+#### something1 == something2
+#### something1 != something2
+#### something1 >= something2
+#### something1 > something2
+#### something1 <= something2
+#### something1 < something2 -- e.g. `item.filename != item.title,`
+
+Compare something1 to something2. The result is a boolean which is useful for branch conditions.
+
+#### op iterable_operation -- e.g. `op files().(0..=42)~(shuffle),`
+
+An iterable operation inside of the procedure. When assigned to item, this can be used to replace item with multiple others. Note that iterable operations are never executed inside the procedure; when item is iterable, it will be executed immediately after the end of the procedure for the current item.
+
+#### (something1)
+#### -something1
+#### something1 - something2
+#### something1 + something2
+#### something1 || something2
+#### something1 && something2 -- e.g. `42 + (128 - 64),`
+
+Various algebraic operations: brackets (order of operations), negation, subtraction, addition, logical OR, logical AND; respectively.
+
+#### Item(field1 = something1, field2 = something2, ...) - e.g. `item = Item(title = item.title, filename = "/dev/null"),`
+
+   Constructor for a new item. Each function parameter defines a new field and it's value.
+#### ~"string_format" something -- e.g. `~"{filename}" item,`
+
+   Format a value into a string. This behaves differently depending on the value's type: When the value is an Item, the item's corresponding field will replace all `{field}` instances in the format string. When the value is a primitive type (String, Int, Bool, etc.), the value's text equivalent will replace all `{}` instances in the format string. When the value is an iterable operation (Op), the operation's script equivalent will replace all `{}` instances in the format string.
+
+#### file(filepath) -- e.g. `file("~/Music/Romantic Traffic.flac"),`
+
+Load an item from file, populating the item with the song's tags.
+
+
+License: LGPL-2.1-only OR GPL-3.0-only

interpreter/benches/file_parse.rs

@@ -0,0 +1,58 @@
+use muss_interpreter::Interpreter;
+//use mps_interpreter::MpsRunner;
+use std::fs::File;
+use std::io::{BufReader, Read, Seek};
+
+use criterion::{criterion_group, criterion_main, Criterion};
+
+/*fn interpretor_benchmark(c: &mut Criterion) {
+    let f = File::open("benches/lots_of_empty.mps").unwrap();
+    let mut reader = BufReader::with_capacity(1024 * 1024 /* 1 MiB */, f);
+    // read everything into buffer before starting
+    let mut buf = Vec::with_capacity(1024 * 1024);
+    reader.read_to_end(&mut buf).unwrap();
+    drop(buf);
+    c.bench_function("mps lots_of_empty.mps", |b| {
+        b.iter(|| {
+            //let f = File::open("benches/lots_of_empty.mps").unwrap();
+            //let mut reader = BufReader::new(f);
+            reader.rewind().unwrap();
+            let mps = MpsRunner::with_stream(&mut reader);
+            for item in mps {
+                match item {
+                    Err(e) => panic!("{}", e),
+                    Ok(_) => {}
+                }
+            }
+        })
+    });
+}*/
+
+fn faye_benchmark(c: &mut Criterion) {
+    let f = File::open("benches/lots_of_empty.muss").unwrap();
+    let mut reader = BufReader::with_capacity(1024 * 1024 /* 1 MiB */, f);
+    // read everything into buffer before starting
+    let mut buf = Vec::with_capacity(1024 * 1024);
+    reader.read_to_end(&mut buf).unwrap();
+    drop(buf);
+    c.bench_function("muss-faye lots_of_empty.muss", |b| {
+        b.iter(|| {
+            //let f = File::open("benches/lots_of_empty.mps").unwrap();
+            //let mut reader = BufReader::new(f);
+            reader.rewind().unwrap();
+            let mps = Interpreter::with_stream(&mut reader);
+            for item in mps {
+                match item {
+                    Err(e) => panic!("{}", e),
+                    Ok(_) => {}
+                }
+            }
+        })
+    });
+}
+
+criterion_group!(
+    parse_benches,
+    /*interpretor_benchmark,*/ faye_benchmark
+);
+criterion_main!(parse_benches);

interpreter/fuzz/.gitignore

@@ -0,0 +1,3 @@
+target
+corpus
+artifacts

interpreter/fuzz/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "muss-interpreter-fuzz"
+version = "0.0.0"
+authors = ["Automatically generated"]
+publish = false
+edition = "2021"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+
+[dependencies.muss-interpreter]
+path = ".."
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[[bin]]
+name = "faye_fuzz"
+path = "fuzz_targets/faye_fuzz.rs"
+test = false
+doc = false

interpreter/fuzz/fuzz_targets/faye_fuzz.rs

@@ -0,0 +1,20 @@
+#![no_main]
+#[macro_use] extern crate libfuzzer_sys;
+extern crate muss_interpreter;
+
+fuzz_target!(|data: &[u8]| {
+    if let Ok(s) = std::str::from_utf8(data) {
+        print!("len:{},data:{}\n", data.len(), s)
+    } else {
+        print!("len:{},data:<non-ut8>,", data.len());
+    }
+    let mut cursor = std::io::Cursor::new(data);
+    let interpreter = muss_interpreter::Interpreter::with_stream(&mut cursor);
+    for item in interpreter {
+        match item {
+            Err(e) => print!("err:{},", e),
+            Ok(_i) => {},//print!("item:{},", i),
+        }
+    }
+    println!("done.");
+});

interpreter/src/context.rs

@@ -0,0 +1,48 @@
+#[cfg(feature = "advanced")]
+use super::processing::advanced::{DefaultAnalyzer, MusicAnalyzer};
+use super::processing::database::DatabaseQuerier;
+#[cfg(feature = "fakesql")]
+use super::processing::database::SQLiteTranspileExecutor;
+#[cfg(feature = "mpd")]
+use super::processing::database::{MpdExecutor, MpdQuerier};
+use super::processing::general::{
+    FilesystemExecutor, FilesystemQuerier, OpStorage, VariableStorer,
+};
+use std::fmt::{Debug, Display, Error, Formatter};
+
+#[derive(Debug)]
+pub struct Context {
+    pub database: Box<dyn DatabaseQuerier>,
+    pub variables: Box<dyn VariableStorer>,
+    pub filesystem: Box<dyn FilesystemQuerier>,
+    #[cfg(feature = "advanced")]
+    pub analysis: Box<dyn MusicAnalyzer>,
+    #[cfg(feature = "mpd")]
+    pub mpd_database: Box<dyn MpdQuerier>,
+}
+
+impl Default for Context {
+    fn default() -> Self {
+        Self {
+            #[cfg(feature = "fakesql")]
+            database: Box::new(SQLiteTranspileExecutor::default()),
+            #[cfg(all(feature = "sql", not(feature = "fakesql")))]
+            database: Box::new(super::processing::database::SQLiteExecutor::default()),
+            #[cfg(all(not(feature = "sql"), not(feature = "fakesql")))]
+            database: Box::new(super::processing::database::SQLErrExecutor::default()),
+            variables: Box::new(OpStorage::default()),
+            filesystem: Box::new(FilesystemExecutor::default()),
+            #[cfg(feature = "advanced")]
+            analysis: Box::new(DefaultAnalyzer::default()),
+            #[cfg(feature = "mpd")]
+            mpd_database: Box::new(MpdExecutor::default()),
+        }
+    }
+}
+
+impl Display for Context {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "Context{{...}}")?;
+        Ok(())
+    }
+}

interpreter/src/debug.rs

@@ -0,0 +1,41 @@
+use std::iter::Iterator;
+
+use super::tokens::TokenReader;
+use super::{Interpreter, InterpreterItem};
+
+/// Wrapper for InterpreterError with a built-in callback function for every iteration of the interpreter.
+pub struct Debugger<'a, T, F>
+where
+    T: TokenReader,
+    F: Fn(&mut Interpreter<'a, T>, Option<InterpreterItem>) -> Option<InterpreterItem>,
+{
+    interpreter: Interpreter<'a, T>,
+    transmuter: F,
+}
+
+impl<'a, T, F> Debugger<'a, T, F>
+where
+    T: TokenReader,
+    F: Fn(&mut Interpreter<'a, T>, Option<InterpreterItem>) -> Option<InterpreterItem>,
+{
+    /// Create a new instance of Debugger using the provided interpreter and callback.
+    pub fn new(faye: Interpreter<'a, T>, item_handler: F) -> Self {
+        Self {
+            interpreter: faye,
+            transmuter: item_handler,
+        }
+    }
+}
+
+impl<'a, T, F> Iterator for Debugger<'a, T, F>
+where
+    T: TokenReader,
+    F: Fn(&mut Interpreter<'a, T>, Option<InterpreterItem>) -> Option<InterpreterItem>,
+{
+    type Item = InterpreterItem;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let next_item = self.interpreter.next();
+        (self.transmuter)(&mut self.interpreter, next_item)
+    }
+}

interpreter/src/errors.rs

@@ -0,0 +1,50 @@
+use std::convert::From;
+use std::fmt::{Debug, Display, Error, Formatter};
+
+use crate::lang::{LanguageError, RuntimeError, SyntaxError};
+use crate::tokens::ParseError;
+
+#[derive(Debug)]
+pub enum InterpreterError {
+    Syntax(SyntaxError),
+    Runtime(RuntimeError),
+    Parse(ParseError),
+}
+
+impl Display for InterpreterError {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        match self {
+            Self::Syntax(e) => (e as &dyn Display).fmt(f),
+            Self::Runtime(e) => (e as &dyn Display).fmt(f),
+            Self::Parse(e) => (e as &dyn Display).fmt(f),
+        }
+    }
+}
+
+impl LanguageError for InterpreterError {
+    fn set_line(&mut self, line: usize) {
+        match self {
+            Self::Syntax(e) => e.set_line(line),
+            Self::Runtime(e) => e.set_line(line),
+            Self::Parse(e) => e.set_line(line),
+        }
+    }
+}
+
+impl From<SyntaxError> for InterpreterError {
+    fn from(e: SyntaxError) -> Self {
+        Self::Syntax(e)
+    }
+}
+
+impl From<RuntimeError> for InterpreterError {
+    fn from(e: RuntimeError) -> Self {
+        Self::Runtime(e)
+    }
+}
+
+impl From<ParseError> for InterpreterError {
+    fn from(e: ParseError) -> Self {
+        Self::Parse(e)
+    }
+}

interpreter/src/faye.rs

@@ -0,0 +1,210 @@
+use std::collections::VecDeque;
+use std::io::Read;
+use std::iter::Iterator;
+
+use super::lang::{LanguageDictionary, LanguageError, Op};
+use super::tokens::{Token, TokenReader, Tokenizer};
+use super::Context;
+use super::InterpreterError;
+use super::Item;
+
+const DEFAULT_TOKEN_BUFFER_SIZE: usize = 16;
+
+pub enum InterpreterEvent {
+    FileEnd,
+    StatementComplete,
+    NewStatementReady,
+}
+
+/// The script interpreter.
+pub struct Interpreter<'a, T>
+where
+    T: TokenReader,
+{
+    tokenizer: T,
+    buffer: VecDeque<Token>,
+    current_stmt: Box<dyn Op>,
+    vocabulary: LanguageDictionary,
+    callback: &'a dyn Fn(&mut Interpreter<'a, T>, InterpreterEvent) -> Result<(), InterpreterError>,
+}
+
+#[inline]
+fn empty_callback<T: TokenReader>(
+    _s: &mut Interpreter<'_, T>,
+    _d: InterpreterEvent,
+) -> Result<(), InterpreterError> {
+    Ok(())
+}
+
+/*impl <T> Interpreter<'static, T>
+where
+    T: TokenReader,
+{
+    /// Create a new interpreter for the provided token reader, using the standard MPS language.
+    #[inline]
+    pub fn with_standard_vocab(token_reader: T) -> Self {
+        let mut vocab = LanguageDictionary::default();
+        super::interpretor::standard_vocab(&mut vocab);
+        Self::with_vocab(vocab, token_reader)
+    }
+
+    /// Create a new interpreter with the provided vocabulary and token reader.
+    #[inline]
+    pub fn with_vocab(vocab: LanguageDictionary, token_reader: T) -> Self {
+        Self::with(vocab, token_reader, &empty_callback)
+    }
+}*/
+
+impl<'a, R: Read> Interpreter<'a, Tokenizer<R>> {
+    pub fn with_stream(stream: R) -> Self {
+        let tokenizer = Tokenizer::new(stream);
+        Self::with_standard_vocab(tokenizer)
+    }
+
+    pub fn with_stream_and_callback(
+        stream: R,
+        callback: &'a dyn Fn(
+            &mut Interpreter<'a, Tokenizer<R>>,
+            InterpreterEvent,
+        ) -> Result<(), InterpreterError>,
+    ) -> Self {
+        let tokenizer = Tokenizer::new(stream);
+        let vocab = LanguageDictionary::standard();
+        Self::with(vocab, tokenizer, callback)
+    }
+}
+
+impl<'a, T> Interpreter<'a, T>
+where
+    T: TokenReader,
+{
+    #[inline]
+    pub fn with_standard_vocab(token_reader: T) -> Self {
+        let vocab = LanguageDictionary::standard();
+        Self::with_vocab(vocab, token_reader)
+    }
+
+    /// Create a new interpreter with the provided vocabulary and token reader.
+    #[inline]
+    pub fn with_vocab(vocab: LanguageDictionary, token_reader: T) -> Self {
+        Self::with(vocab, token_reader, &empty_callback)
+    }
+
+    /// Create a custom interpreter instance.
+    #[inline]
+    pub fn with(
+        vocab: LanguageDictionary,
+        token_reader: T,
+        callback: &'a dyn Fn(
+            &mut Interpreter<'a, T>,
+            InterpreterEvent,
+        ) -> Result<(), InterpreterError>,
+    ) -> Self {
+        Self {
+            tokenizer: token_reader,
+            buffer: VecDeque::with_capacity(DEFAULT_TOKEN_BUFFER_SIZE),
+            current_stmt: Box::new(crate::lang::vocabulary::empty::EmptyStatement {
+                context: Some(Context::default()),
+            }),
+            vocabulary: vocab,
+            callback: callback,
+        }
+    }
+
+    // build a new statement
+    #[inline]
+    fn new_statement(&mut self) -> Option<Result<Box<dyn Op>, InterpreterError>> {
+        while !self.tokenizer.end_of_file() && self.buffer.is_empty() {
+            let result = self.tokenizer.next_statement(&mut self.buffer);
+            match result {
+                Ok(_) => {}
+                Err(e) => return Some(Err(error_with_ctx(e, self.tokenizer.current_line()))),
+            }
+        }
+        if self.buffer.is_empty() {
+            let callback_result = (self.callback)(self, InterpreterEvent::FileEnd);
+            match callback_result {
+                Ok(_) => {}
+                Err(e) => return Some(Err(e)),
+            }
+            return None;
+        }
+        let result = self.vocabulary.try_build_statement(&mut self.buffer);
+        let stmt = match result {
+            Ok(stmt) => stmt,
+            Err(e) => return Some(Err(error_with_ctx(e, self.tokenizer.current_line()))),
+        };
+        //println!("Final parsed op: {}", stmt);
+        #[cfg(debug_assertions)]
+        if !self.buffer.is_empty() {
+            panic!("Token buffer was not emptied! (rem: {:?})", self.buffer)
+        }
+        Some(Ok(stmt))
+    }
+}
+
+pub type InterpreterItem = Result<Item, InterpreterError>;
+
+impl<'a, T> Iterator for Interpreter<'a, T>
+where
+    T: TokenReader,
+{
+    type Item = InterpreterItem;
+
+    fn next(&mut self) -> Option<Self::Item> {