Fix playlist filepath generation to ignore already-absolute paths

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: NGnius
+liberapay: NGnius

.gitmodules

@@ -4,3 +4,5 @@
 [submodule "bliss-rs"]
 	path = bliss-rs
 	url = https://github.com/NGnius/bliss-rs
+[submodule "bliss-rs/"]
+	url = https://git.ngni.us/NGnius/bliss-rs

Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "muss"
-version = "0.8.0"
+version = "0.9.0"
 edition = "2021"
 authors = ["NGnius (Graham) <ngniusness@gmail.com>"]
 description = "Music Set Script language (MuSS)"
 license = "LGPL-2.1-only OR GPL-3.0-only"
-repository = "https://github.com/NGnius/mps"
+repository = "https://git.ngni.us/NGnius/muss"
 keywords = ["audio", "playlist", "scripting", "language"]
 readme = "README.md"
 exclude = ["extras/"]
@@ -19,17 +19,21 @@ members = [
 
 [dependencies]
 # local
-muss-interpreter = { version = "0.8.0", path = "./interpreter" }
+muss-interpreter = { version = "0.9.0", path = "./interpreter" }
 # external
 clap = { version = "3.0", features = ["derive"] }
 console = { version = "0.15" }
+lazy_static = { version = "1.4" }
+
+# cli add to playlist functionality
+m3u8-rs = { version = "^3.0.0" }
 
 [target.'cfg(not(target_os = "linux"))'.dependencies]
-muss-player = { version = "0.8.0", path = "./player", default-features = false, features = ["mpd"] }
+muss-player = { version = "0.9.0", path = "./player", default-features = false, features = ["mpd"] }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 # TODO fix need to specify OS-specific dependency of mps-player
-muss-player = { version = "0.8.0", path = "./player", features = ["mpris-player", "mpd"] }
+muss-player = { version = "0.9.0", path = "./player", features = ["mpris-player", "mpd"] }
 
 [profile.release]
 debug = false

README.md

@@ -1,6 +1,6 @@
 # muss
 
-![repl_demo](https://raw.githubusercontent.com/NGnius/mps/master/extras/demo.png)
+![repl_demo](https://raw.githubusercontent.com/NGnius/muss/master/extras/demo.png)
 
 Sort, filter and analyse your music to create great playlists.
 This project implements the interpreter (`./interpreter`), music player (`./player`), and CLI interface for Muss (`./`).
@@ -16,17 +16,17 @@ To access the REPL, simply run `cargo run`. You will need the [Rust toolchain in
 
 All songs by artist `<artist>` (in your library), sorted by similarity to a random first song by the artist.
 ```muss
-files().(artist? like "<artist>")~(shuffle)~(advanced bliss_next);
+files().(.artist? like "<artist>")~(~radio);
 ```
 
 All songs with a `.flac` file extension (anywhere in their path -- not necessarily at the end).
 ```muss
-files().(filename? like ".flac");
+files().(.filename? like ".flac");
 ```
 
 All songs by artist `<artist1>` or `<artist2>`, sorted by similarity to a random first song by either artist.
 ```muss
-files().(artist? like "<artist1>" || artist? like "<artist2>")~(shuffle)~(advanced bliss_next);
+files().(.artist? like "<artist1>" || .artist? like "<artist2>")~(~radio);
 ```
 
 ### Bigger examples
@@ -44,7 +44,7 @@ One day I'll add pretty REPL example pictures and some script files...
 **I thought it would be fun**. I also wanted to be able to play my music without having to be at the whim of someone else's algorithm (and music), and playing just by album or artist was getting boring. Designing a language specifically for iteration seemed like a cool & novel way of doing it, too (though every approach is a novel approach for me).
 
 ### What is Muss?
-**Music Set Script (MuSS) is a language for describing a playlist of music.** It uses an (auto-generated) SQLite3 database for SQL queries and can also directly query the filesystem. Queries can be modified by using filters, functions, and sorters built-in to Muss (see interpreter's README.md).
+**Music Set Script (MuSS) is a language for describing a playlist of music.** It can execute SQLite select clauses or directly query the filesystem. Queries can be modified by using filters, functions, and sorters built-in to Muss (see interpreter's README.md).
 
 ### Is Muss a scripting language?
 **Yes**. It evolved from a simple query language into something that can do arbitrary calculations. Whether it's Turing-complete is still unproven, but it's powerful enough for what I want it to do.
@@ -54,7 +54,7 @@ One day I'll add pretty REPL example pictures and some script files...
 
 LGPL-2.1-only OR GPL-3.0-only
 
-**NOTE**: When advanced features are enabled, GPL-3.0 must be used.
+**NOTE**: Advanced features make use of [a fork of bliss-rs](https://github.com/NGnius/bliss-rs) a GPL-3.0 licensed music analysis library.
 
 ## Contribution
 

README.tpl

@@ -1,6 +1,6 @@
 # {{crate}}
 
-![repl_demo](https://raw.githubusercontent.com/NGnius/mps/master/extras/demo.png)
+![repl_demo](https://raw.githubusercontent.com/NGnius/muss/master/extras/demo.png)
 
 {{readme}}
 
@@ -8,7 +8,7 @@
 
 {{license}}
 
-**NOTE**: When advanced features are enabled, GPL-3.0 must be used.
+**NOTE**: Advanced features make use of [a fork of bliss-rs](https://github.com/NGnius/bliss-rs) a GPL-3.0 licensed music analysis library.
 
 ## Contribution
 

bliss-rs

@@ -1 +1 @@
-Subproject commit 5d66e104235479e38f7bfef6d27980a183ff2142
+Subproject commit 2dcdd5643bc0cd055887128637abbb9bed041f77

interpreter/Cargo.toml

@@ -1,23 +1,25 @@
 [package]
 name = "muss-interpreter"
-version = "0.8.0"
+version = "0.9.0"
 edition = "2021"
 license = "LGPL-2.1-only OR GPL-3.0-only"
 readme = "README.md"
+repository = "https://git.ngni.us/NGnius/muss"
 rust-version = "1.59"
 
 [dependencies]
-rusqlite = { version = "0.26", features = ["bundled"] }
-symphonia = { version = "0.5", optional = true, features = [
-    "aac", "alac", "flac", "mp3", "pcm", "vorbis", "isomp4", "ogg", "wav"
-] }
+rusqlite = { version = "0.27", features = ["bundled"], optional = true }
+sqlparser = { version = "0.23", optional = true }
+symphonia = { version = "0.5", optional = true, features = ["all"] }
 dirs = { version = "4" }
 regex = { version = "1" }
 rand = { version = "0.8" }
 shellexpand = { version = "2", optional = true }
-bliss-audio-symphonia = { version = "0.4", optional = true, path = "../bliss-rs" }
-mpd = { version = "0.0.12", optional = true }
+bliss-audio-symphonia = { version = "0.6", optional = true, path = "../bliss-rs" }
+mpd = { version = "0.1", optional = true }
 unidecode = { version = "0.3.0", optional = true }
+base64 = { version = "0.13", optional = true }
+m3u8-rs = { version = "3.0.0", optional = true }
 
 [dev-dependencies]
 criterion = "0.3"
@@ -27,7 +29,11 @@ name = "file_parse"
 harness = false
 
 [features]
-default = [ "music_library", "ergonomics", "advanced" ]
-music_library = [ "symphonia", "mpd" ] # song metadata parsing and database auto-population
+default = [ "music_library", "ergonomics", "advanced", "advanced-bliss", "fakesql", "collections" ]
+music_library = [ "symphonia", "mpd", "base64" ] # song metadata parsing and database auto-population
+collections = [ "m3u8-rs" ] # read from m3u8 playlists (and other song collections, eventually)
 ergonomics = ["shellexpand", "unidecode"] # niceties like ~ in paths and unicode string sanitisation
-advanced = ["bliss-audio-symphonia"] # advanced language features like bliss playlist generation
+advanced = [] # advanced language features like music analysis
+advanced-bliss = ["bliss-audio-symphonia"] # bliss audio analysis
+sql = [ "rusqlite" ] # sqlite database for music
+fakesql = [ "sqlparser" ] # transpiled sqlite interpreter

interpreter/README.md

@@ -43,23 +43,23 @@ Operations to reduce the items in an iterable: `iterable.(filter);`.
 Filters are statements of the format `something.(predicate)`, where "something" is a variable name or another valid statement, and "predicate" is a valid filter predicate (see below).
 E.g. `files(folder="~/Music/", recursive=true).(title == "Romantic Traffic");` is valid filter syntax to filter all songs in the Music folder for songs named "Romantic Traffic" (probably just one song).
 
-#### field == something
+#### .field == something
 
-#### field like something
+#### .field like something
 
-#### field unlike something
+#### .field unlike something
 
-#### field matches some_regex
+#### .field matches some_regex
 
-#### field != something
+#### .field != something
 
-#### field >= something
+#### .field >= something
 
-#### field > something
+#### .field > something
 
-#### field <= something
+#### .field <= something
 
-#### field < something -- e.g. `iterable.(title == "Romantic Traffic");`
+#### .field < something -- e.g. `iterable.(.title == "Romantic Traffic");`
 
 Compare all items, keeping only those that match the condition. Valid field names change depending on what information is available when the Item is populated, but usually title, artist, album, genre, track, filename are valid fields. Optionally, a ? or ! can be added to the end of the field name to skip items whose field is missing/incomparable, or keep all items whose field is missing/incomparable (respectively).
 
@@ -89,7 +89,7 @@ Matches all items
 Replace items matching the filter with operation1 and replace items not matching the filter with operation2. The `else operation2` part may be omitted to preserve items not matching the filter. To perform operations with the current item, use the special variable `item`. The replacement filter may not contain || -- instead, use multiple filters chained together.
 
 #### unique
-#### unique field -- e.g. `iterable.(unique title);`
+#### unique field -- e.g. `iterable.(unique .title);`
 
 Keep only items which are do not duplicate another item, or keep only items whoes specified field does not duplicate another item's same field. The first non-duplicated instance of an item is always the one that is kept.
 
@@ -166,17 +166,21 @@ Empty iterator containing zero items. Useful for deleting items using replacemen
 Iterate over count empty items. The items in this iterator have no fields (i.e. are empty).
 
 ### Sorters
-Operations to sort the items in an iterable: `iterable~(sorter)` OR `iterable.sort(sorter)`.
+Operations to sort the items in an iterable: `iterable~(sorter)`.
 
-#### field -- e.g. `iterable~(filename);`
+#### .field -- e.g. `iterable~(.filename);`
 
-Sort by a Item field. Valid field names change depending on what information is available when the Item is populated, but usually title, artist, album, genre, track, filename are valid fields. Items with a missing/incomparable fields will be sorted to the end.
+Sort by an Item field. Valid field names change depending on what information is available when the Item is populated, but usually title, artist, album, genre, track, filename are valid fields. Items with a missing/incomparable fields will be sorted to the end.
 
-#### shuffle
-#### random shuffle -- e.g. `iterable~(shuffle);`
+#### ~shuffle
+#### random shuffle -- e.g. `iterable~(~shuffle);`
 
 Shuffle the songs in the iterator. This is random for up to 2^16 items, and then the randomness degrades (but at that point you won't notice). The more verbose syntax is allowed in preparation for future randomisation strategies.
 
+#### ~radio
+#### ~radio qualifier -- e.g. `iterable~(~radio)`
+Sort by musical similarity, starting with a random first song from the iterator. The optional qualifier may be `chroma`, `loudness`, `spectrum`, or `tempo`. When the qualifier is omitted, they are all considered for comparing audio similarity.
+
 #### advanced bliss_first -- e.g. `iterable~(advanced bliss_first);`
 
 Sort by the distance (similarity) from the first song in the iterator. Songs which are more similar (lower distance) to the first song in the iterator will be placed closer to the first song, while less similar songs will be sorted to the end. This uses the [bliss music analyser](https://github.com/polochon-street/bliss-rs), which is a very slow operation and can cause music playback interruptions for large iterators. This requires the `advanced` feature to be enabled (without the feature enabled this is still valid syntax but doesn't change the order).
@@ -244,7 +248,7 @@ Various algebraic operations: brackets (order of operations), negation, subtract
 
 #### file(filepath) -- e.g. `file("~/Music/Romantic Traffic.flac"),`
 
-Load a item from file, populating the item with the song's tags.
+Load an item from file, populating the item with the song's tags.
 
 
 License: LGPL-2.1-only OR GPL-3.0-only

interpreter/benches/file_parse.rs

@@ -1,4 +1,4 @@
-use mps_interpreter::MpsFaye;
+use muss_interpreter::Interpreter;
 //use mps_interpreter::MpsRunner;
 use std::fs::File;
 use std::io::{BufReader, Read, Seek};
@@ -29,18 +29,18 @@ use criterion::{criterion_group, criterion_main, Criterion};
 }*/
 
 fn faye_benchmark(c: &mut Criterion) {
-    let f = File::open("benches/lots_of_empty.mps").unwrap();
+    let f = File::open("benches/lots_of_empty.muss").unwrap();
     let mut reader = BufReader::with_capacity(1024 * 1024 /* 1 MiB */, f);
     // read everything into buffer before starting
     let mut buf = Vec::with_capacity(1024 * 1024);
     reader.read_to_end(&mut buf).unwrap();
     drop(buf);
-    c.bench_function("mps-faye lots_of_empty.mps", |b| {
+    c.bench_function("muss-faye lots_of_empty.muss", |b| {
         b.iter(|| {
             //let f = File::open("benches/lots_of_empty.mps").unwrap();
             //let mut reader = BufReader::new(f);
             reader.rewind().unwrap();
-            let mps = MpsFaye::with_stream(&mut reader);
+            let mps = Interpreter::with_stream(&mut reader);
             for item in mps {
                 match item {
                     Err(e) => panic!("{}", e),
@@ -51,5 +51,8 @@ fn faye_benchmark(c: &mut Criterion) {
     });
 }
 
-criterion_group!(parse_benches, /*interpretor_benchmark,*/ faye_benchmark);
+criterion_group!(
+    parse_benches,
+    /*interpretor_benchmark,*/ faye_benchmark
+);
 criterion_main!(parse_benches);

interpreter/fuzz/Cargo.lock

@@ -8,17 +8,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
-[[package]]
-name = "ahash"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
-dependencies = [
- "getrandom",
- "once_cell",
- "version_check",
-]
-
 [[package]]
 name = "aho-corasick"
 version = "0.7.18"
@@ -57,6 +46,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "base64"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -83,7 +78,7 @@ dependencies = [
 
 [[package]]
 name = "bliss-audio-symphonia"
-version = "0.4.6"
+version = "0.5.0"
 dependencies = [
  "bliss-audio-aubio-rs",
  "crossbeam",
@@ -96,6 +91,7 @@ dependencies = [
  "noisy_float",
  "num_cpus",
  "rayon",
+ "rcue",
  "ripemd160",
  "rustfft",
  "strum",
@@ -344,18 +340,6 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
 
-[[package]]
-name = "fallible-iterator"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
-
-[[package]]
-name = "fallible-streaming-iterator"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
-
 [[package]]
 name = "flate2"
 version = "1.0.24"
@@ -396,30 +380,12 @@ dependencies = [
  "wasi 0.11.0+wasi-snapshot-preview1",
 ]
 
-[[package]]
-name = "hashbrown"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
-dependencies = [
- "ahash",
-]
-
 [[package]]
 name = "hashbrown"
 version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3"
 
-[[package]]
-name = "hashlink"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
-dependencies = [
- "hashbrown 0.11.2",
-]
-
 [[package]]
 name = "heck"
 version = "0.3.3"
@@ -451,7 +417,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
 dependencies = [
  "autocfg",
- "hashbrown 0.12.1",
+ "hashbrown",
 ]
 
 [[package]]
@@ -495,17 +461,6 @@ dependencies = [
  "once_cell",
 ]
 
-[[package]]
-name = "libsqlite3-sys"
-version = "0.23.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2cafc7c74096c336d9d27145f7ebd4f4b6f95ba16aa5a282387267e6925cb58"
-dependencies = [
- "cc",
- "pkg-config",
- "vcpkg",
-]
-
 [[package]]
 name = "log"
 version = "0.4.17"
@@ -515,6 +470,15 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "m3u8-rs"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50fe05791a7f418b59d6cddebdc293d77c9c1f652adbff855c071d4507cd883b"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "maplit"
 version = "1.0.2"
@@ -545,6 +509,12 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.5.3"
@@ -567,15 +537,17 @@ dependencies = [
 
 [[package]]
 name = "muss-interpreter"
-version = "0.8.0"
+version = "0.9.0"
 dependencies = [
+ "base64",
  "bliss-audio-symphonia",
  "dirs",
+ "m3u8-rs",
  "mpd",
  "rand",
  "regex",
- "rusqlite",
  "shellexpand",
+ "sqlparser",
  "symphonia",
  "unidecode",
 ]
@@ -640,6 +612,16 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "nom"
+version = "7.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "num-bigint"
 version = "0.4.3"
@@ -759,12 +741,6 @@ dependencies = [
  "sha-1",
 ]
 
-[[package]]
-name = "pkg-config"
-version = "0.3.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
-
 [[package]]
 name = "ppv-lite86"
 version = "0.2.16"
@@ -871,6 +847,12 @@ dependencies = [
  "num_cpus",
 ]
 
+[[package]]
+name = "rcue"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fca1481d62f18158646de2ec552dd63f8bdc5be6448389b192ba95c939df997e"
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.13"
@@ -919,21 +901,6 @@ dependencies = [
  "opaque-debug 0.3.0",
 ]
 
-[[package]]
-name = "rusqlite"
-version = "0.26.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ba4d3462c8b2e4d7f4fcfcf2b296dc6b65404fbbc7b63daa37fd485c149daf7"
-dependencies = [
- "bitflags",
- "fallible-iterator",
- "fallible-streaming-iterator",
- "hashlink",
- "libsqlite3-sys",
- "memchr",
- "smallvec",
-]
-
 [[package]]
 name = "rustc-serialize"
 version = "0.3.24"
@@ -982,10 +949,13 @@ dependencies = [
 ]
 
 [[package]]
-name = "smallvec"
-version = "1.8.1"
+name = "sqlparser"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc88c725d61fc6c3132893370cac4a0200e3fedf5da8331c570664b1987f5ca2"
+checksum = "0beb13adabbdda01b63d595f38c8bfd19a361e697fd94ce0098a634077bc5b25"
+dependencies = [
+ "log",
+]
 
 [[package]]
 name = "strength_reduce"
@@ -1274,12 +1244,6 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "402bb19d8e03f1d1a7450e2bd613980869438e0666331be3e073089124aa1adc"
 
-[[package]]
-name = "vcpkg"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
-
 [[package]]
 name = "version_check"
 version = "0.9.4"

interpreter/src/context.rs

@@ -1,8 +1,10 @@
 #[cfg(feature = "advanced")]
 use super::processing::advanced::{DefaultAnalyzer, MusicAnalyzer};
-use super::processing::database::{DatabaseQuerier, SQLiteExecutor};
+use super::processing::database::DatabaseQuerier;
+#[cfg(feature = "fakesql")]
+use super::processing::database::SQLiteTranspileExecutor;
 #[cfg(feature = "mpd")]
-use super::processing::database::{MpdQuerier, MpdExecutor};
+use super::processing::database::{MpdExecutor, MpdQuerier};
 use super::processing::general::{
     FilesystemExecutor, FilesystemQuerier, OpStorage, VariableStorer,
 };
@@ -22,7 +24,12 @@ pub struct Context {
 impl Default for Context {
     fn default() -> Self {
         Self {
-            database: Box::new(SQLiteExecutor::default()),
+            #[cfg(feature = "fakesql")]
+            database: Box::new(SQLiteTranspileExecutor::default()),
+            #[cfg(all(feature = "sql", not(feature = "fakesql")))]
+            database: Box::new(super::processing::database::SQLiteExecutor::default()),
+            #[cfg(all(not(feature = "sql"), not(feature = "fakesql")))]
+            database: Box::new(super::processing::database::SQLErrExecutor::default()),
             variables: Box::new(OpStorage::default()),
             filesystem: Box::new(FilesystemExecutor::default()),
             #[cfg(feature = "advanced")]

interpreter/src/debug.rs

@@ -1,32 +1,25 @@
 use std::iter::Iterator;
 
 use super::tokens::TokenReader;
-use super::{InterpreterError, Interpreter, Item};
+use super::{Interpreter, InterpreterItem};
 
 /// Wrapper for InterpreterError with a built-in callback function for every iteration of the interpreter.
-pub struct Debugger<'a, 'b, T>
+pub struct Debugger<'a, T, F>
 where
     T: TokenReader,
+    F: Fn(&mut Interpreter<'a, T>, Option<InterpreterItem>) -> Option<InterpreterItem>,
 {
     interpreter: Interpreter<'a, T>,
-    transmuter: &'b dyn Fn(
-        &mut Interpreter<'a, T>,
-        Option<Result<Item, InterpreterError>>,
-    ) -> Option<Result<Item, InterpreterError>>,
+    transmuter: F,
 }
 
-impl<'a, 'b, T> Debugger<'a, 'b, T>
+impl<'a, T, F> Debugger<'a, T, F>
 where
     T: TokenReader,
+    F: Fn(&mut Interpreter<'a, T>, Option<InterpreterItem>) -> Option<InterpreterItem>,
 {
     /// Create a new instance of Debugger using the provided interpreter and callback.
-    pub fn new(
-        faye: Interpreter<'a, T>,
-        item_handler: &'b dyn Fn(
-            &mut Interpreter<'a, T>,
-            Option<Result<Item, InterpreterError>>,
-        ) -> Option<Result<Item, InterpreterError>>,
-    ) -> Self {
+    pub fn new(faye: Interpreter<'a, T>, item_handler: F) -> Self {
         Self {
             interpreter: faye,
             transmuter: item_handler,
@@ -34,15 +27,15 @@ where
     }
 }
 
-impl<'a, 'b, T> Iterator for Debugger<'a, 'b, T>
+impl<'a, T, F> Iterator for Debugger<'a, T, F>
 where
     T: TokenReader,
+    F: Fn(&mut Interpreter<'a, T>, Option<InterpreterItem>) -> Option<InterpreterItem>,
 {
-    type Item = Result<Item, InterpreterError>;
+    type Item = InterpreterItem;
 
     fn next(&mut self) -> Option<Self::Item> {
         let next_item = self.interpreter.next();
-        let transmuted_next = (self.transmuter)(&mut self.interpreter, next_item);
-        transmuted_next
+        (self.transmuter)(&mut self.interpreter, next_item)
     }
 }

interpreter/src/faye.rs

@@ -29,8 +29,8 @@ where
 }
 
 #[inline]
-fn empty_callback<'a, T: TokenReader>(
-    _s: &mut Interpreter<'a, T>,
+fn empty_callback<T: TokenReader>(
+    _s: &mut Interpreter<'_, T>,
     _d: InterpreterEvent,
 ) -> Result<(), InterpreterError> {
     Ok(())
@@ -60,6 +60,18 @@ impl<'a, R: Read> Interpreter<'a, Tokenizer<R>> {
         let tokenizer = Tokenizer::new(stream);
         Self::with_standard_vocab(tokenizer)
     }
+
+    pub fn with_stream_and_callback(
+        stream: R,
+        callback: &'a dyn Fn(
+            &mut Interpreter<'a, Tokenizer<R>>,
+            InterpreterEvent,
+        ) -> Result<(), InterpreterError>,
+    ) -> Self {
+        let tokenizer = Tokenizer::new(stream);
+        let vocab = LanguageDictionary::standard();
+        Self::with(vocab, tokenizer, callback)
+    }
 }
 
 impl<'a, T> Interpreter<'a, T>
@@ -83,7 +95,10 @@ where
     pub fn with(
         vocab: LanguageDictionary,
         token_reader: T,
-        callback: &'a dyn Fn(&mut Interpreter<'a, T>, InterpreterEvent) -> Result<(), InterpreterError>,
+        callback: &'a dyn Fn(
+            &mut Interpreter<'a, T>,
+            InterpreterEvent,
+        ) -> Result<(), InterpreterError>,
     ) -> Self {
         Self {
             tokenizer: token_reader,
@@ -119,6 +134,7 @@ where
             Ok(stmt) => stmt,
             Err(e) => return Some(Err(error_with_ctx(e, self.tokenizer.current_line()))),
         };
+        //println!("Final parsed op: {}", stmt);
         #[cfg(debug_assertions)]
         if !self.buffer.is_empty() {
             panic!("Token buffer was not emptied! (rem: {:?})", self.buffer)
@@ -127,11 +143,13 @@ where
     }
 }
 
+pub type InterpreterItem = Result<Item, InterpreterError>;
+
 impl<'a, T> Iterator for Interpreter<'a, T>
 where
     T: TokenReader,
 {
-    type Item = Result<Item, InterpreterError>;
+    type Item = InterpreterItem;
 
     fn next(&mut self) -> Option<Self::Item> {
         loop {
@@ -182,7 +200,10 @@ where
     }
 }
 
-fn error_with_ctx<T: std::convert::Into<InterpreterError>>(error: T, line: usize) -> InterpreterError {
+fn error_with_ctx<T: std::convert::Into<InterpreterError>>(
+    error: T,
+    line: usize,
+) -> InterpreterError {
     let mut err = error.into();
     err.set_line(line);
     err

interpreter/src/interpretor.rs

@@ -4,45 +4,52 @@ use super::lang::LanguageDictionary;
 pub(crate) fn standard_vocab(vocabulary: &mut LanguageDictionary) {
     vocabulary
         // filters
-        .add(crate::lang::vocabulary::filters::empty_filter())
-        .add(crate::lang::vocabulary::filters::unique_filter()) // accepts .(unique)
-        .add(crate::lang::vocabulary::filters::field_filter()) // accepts any .(something)
-        .add(crate::lang::vocabulary::filters::field_filter_maybe())
-        .add(crate::lang::vocabulary::filters::index_filter())
-        .add(crate::lang::vocabulary::filters::range_filter())
-        .add(crate::lang::vocabulary::filters::field_like_filter())
-        .add(crate::lang::vocabulary::filters::field_re_filter())
-        .add(crate::lang::vocabulary::filters::unique_field_filter())
-        .add(crate::lang::vocabulary::filters::nonempty_filter())
+        .add_transform(crate::lang::vocabulary::filters::empty_filter())
+        .add_transform(crate::lang::vocabulary::filters::range_filter())
+        .add_transform(
+            // accepts any .(.something)
+            crate::lang::vocabulary::filters::field::FieldFilterBlockFactory::new()
+                .push(crate::lang::vocabulary::filters::field::FieldFilterComparisonFactory)
+                .push(crate::lang::vocabulary::filters::field::FieldFilterMaybeFactory)
+                .push(crate::lang::vocabulary::filters::field::FieldLikeFilterFactory)
+                .push(crate::lang::vocabulary::filters::field::FieldRegexFilterFactory)
+                .to_statement_factory(),
+        )
+        .add_transform(crate::lang::vocabulary::filters::unique_field_filter())
+        .add_transform(crate::lang::vocabulary::filters::unique_filter())
+        .add_transform(crate::lang::vocabulary::filters::nonempty_filter())
+        .add_transform(crate::lang::vocabulary::filters::index_filter())
         // sorters
-        .add(crate::lang::vocabulary::sorters::empty_sort())
-        .add(crate::lang::vocabulary::sorters::shuffle_sort()) // accepts ~(shuffle)
-        .add(crate::lang::vocabulary::sorters::bliss_sort())
-        .add(crate::lang::vocabulary::sorters::bliss_next_sort())
-        .add(crate::lang::vocabulary::sorters::field_sort()) // accepts any ~(something)
+        .add_transform(crate::lang::vocabulary::sorters::empty_sort())
+        .add_transform(crate::lang::vocabulary::sorters::shuffle_sort()) // accepts ~(~shuffle)
+        .add_transform(crate::lang::vocabulary::sorters::bliss_sort())
+        .add_transform(crate::lang::vocabulary::sorters::bliss_next_sort())
+        .add_transform(crate::lang::vocabulary::sorters::radio_sort())
+        .add_transform(crate::lang::vocabulary::sorters::field_sort()) // accepts any ~(.name)
         // iter blocks
-        .add(
+        .add_transform(
             crate::lang::ItemBlockFactory::new()
-                .add(crate::lang::vocabulary::item_ops::ConstantItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::VariableAssignItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::FieldAssignItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::FileItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::VariableDeclareItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::InterpolateStringItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::BranchItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::IterItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::ConstructorItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::EmptyItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::RemoveItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::VariableRetrieveItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::NegateItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::NotItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::CompareItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::AddItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::SubtractItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::OrItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::AndItemOpFactory)
-                .add(crate::lang::vocabulary::item_ops::BracketsItemOpFactory),
+                .push(crate::lang::vocabulary::item_ops::FieldAssignItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::VariableAssignItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::VariableDeclareItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::FileItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::InterpolateStringItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::BranchItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::IterItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::ConstructorItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::EmptyItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::RemoveItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::NotItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::CompareItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::NegateItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::AddItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::SubtractItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::OrItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::AndItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::BracketsItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::FieldRetrieveItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::ConstantItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::VariableRetrieveItemOpFactory),
         )
         // functions and misc
         // functions don't enforce bracket coherence
@@ -54,9 +61,11 @@ pub(crate) fn standard_vocab(vocabulary: &mut LanguageDictionary) {
         .add(crate::lang::vocabulary::AssignStatementFactory)
         .add(crate::lang::vocabulary::sql_init_function_factory())
         .add(crate::lang::vocabulary::files_function_factory())
+        .add(crate::lang::vocabulary::playlist_function_factory())
         .add(crate::lang::vocabulary::empty_function_factory())
         .add(crate::lang::vocabulary::empties_function_factory())
         .add(crate::lang::vocabulary::reset_function_factory())
         .add(crate::lang::vocabulary::union_function_factory())
-        .add(crate::lang::vocabulary::intersection_function_factory());
+        .add(crate::lang::vocabulary::intersection_function_factory())
+        .add(crate::lang::vocabulary::VariableRetrieveStatementFactory);
 }

interpreter/src/item.rs

@@ -45,6 +45,10 @@ impl Item {
     pub fn len(&self) -> usize {
         self.fields.len()
     }
+
+    pub fn is_empty(&self) -> bool {
+        self.fields.is_empty()
+    }
 }
 
 impl Display for Item {

interpreter/src/lang/db_items.rs

@@ -1,15 +1,20 @@
+#[cfg(feature = "sql")]
 use std::path::Path;
 
+#[cfg(feature = "sql")]
 pub const DEFAULT_SQLITE_FILEPATH: &str = "metadata.muss.sqlite";
 
 pub trait DatabaseObj: Sized {
+    #[cfg(feature = "sql")]
     fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self>;
 
+    #[cfg(feature = "sql")]
     fn to_params(&self) -> Vec<&'_ dyn rusqlite::ToSql>;
 
     fn id(&self) -> u64;
 }
 
+#[cfg(feature = "sql")]
 pub fn generate_default_db() -> rusqlite::Result<rusqlite::Connection> {
     generate_db(
         super::utility::music_folder(),
@@ -18,6 +23,7 @@ pub fn generate_default_db() -> rusqlite::Result<rusqlite::Connection> {
     )
 }
 
+#[cfg(feature = "sql")]
 pub fn generate_db<P1: AsRef<Path>, P2: AsRef<Path>>(
     music_path: P1,
     sqlite_path: P2,
@@ -172,6 +178,7 @@ pub struct DbMusicItem {
 }
 
 impl DatabaseObj for DbMusicItem {
+    #[cfg(feature = "sql")]
     fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self> {
         Ok(Self {
             song_id: row.get(0)?,
@@ -184,6 +191,7 @@ impl DatabaseObj for DbMusicItem {
         })
     }
 
+    #[cfg(feature = "sql")]
     fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> {
         vec![
             &self.song_id,
@@ -212,6 +220,7 @@ pub struct DbMetaItem {
 }
 
 impl DatabaseObj for DbMetaItem {
+    #[cfg(feature = "sql")]
     fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self> {
         Ok(Self {
             meta_id: row.get(0)?,
@@ -223,6 +232,7 @@ impl DatabaseObj for DbMetaItem {
         })
     }
 
+    #[cfg(feature = "sql")]
     fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> {
         vec![
             &self.meta_id,
@@ -247,6 +257,7 @@ pub struct DbArtistItem {
 }
 
 impl DatabaseObj for DbArtistItem {
+    #[cfg(feature = "sql")]
     fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self> {
         Ok(Self {
             artist_id: row.get(0)?,
@@ -255,6 +266,7 @@ impl DatabaseObj for DbArtistItem {
         })
     }
 
+    #[cfg(feature = "sql")]
     fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> {
         vec![&self.artist_id, &self.name, &self.genre]
     }
@@ -274,6 +286,7 @@ pub struct DbAlbumItem {
 }
 
 impl DatabaseObj for DbAlbumItem {
+    #[cfg(feature = "sql")]
     fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self> {
         Ok(Self {
             album_id: row.get(0)?,
@@ -284,6 +297,7 @@ impl DatabaseObj for DbAlbumItem {
         })
     }
 
+    #[cfg(feature = "sql")]
     fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> {
         vec![
             &self.album_id,
@@ -306,6 +320,7 @@ pub struct DbGenreItem {
 }
 
 impl DatabaseObj for DbGenreItem {
+    #[cfg(feature = "sql")]
     fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self> {
         Ok(Self {
             genre_id: row.get(0)?,
@@ -313,6 +328,7 @@ impl DatabaseObj for DbGenreItem {
         })
     }
 
+    #[cfg(feature = "sql")]
     fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> {
         vec![&self.genre_id, &self.title]
     }

interpreter/src/lang/dictionary.rs

@@ -1,26 +1,33 @@
 use std::collections::VecDeque;
 
 use super::SyntaxError;
-use super::{BoxedOpFactory, Op};
+use super::{BoxedOpFactory, BoxedTransformOpFactory, Op};
 use crate::tokens::Token;
 
 pub struct LanguageDictionary {
-    vocabulary: Vec<Box<dyn BoxedOpFactory>>,
+    root_vocabulary: Vec<Box<dyn BoxedOpFactory>>,
+    transform_vocabulary: Vec<Box<dyn BoxedTransformOpFactory>>,
 }
 
 impl LanguageDictionary {
     pub fn add<T: BoxedOpFactory + 'static>(&mut self, factory: T) -> &mut Self {
-        self.vocabulary
+        self.root_vocabulary
             .push(Box::new(factory) as Box<dyn BoxedOpFactory>);
         self
     }
 
-    pub fn try_build_statement(
+    pub fn add_transform<T: BoxedTransformOpFactory + 'static>(&mut self, factory: T) -> &mut Self {
+        self.transform_vocabulary
+            .push(Box::new(factory) as Box<dyn BoxedTransformOpFactory>);
+        self
+    }
+
+    fn try_build_root_statement(
         &self,
         tokens: &mut VecDeque<Token>,
     ) -> Result<Box<dyn Op>, SyntaxError> {
-        //println!("try_build_statement with tokens {:?}", tokens);
-        for factory in &self.vocabulary {
+        //println!("building root op with tokens {:?}", tokens);
+        for factory in &self.root_vocabulary {
             if factory.is_op_boxed(tokens) {
                 return factory.build_op_boxed(tokens, self);
             }
@@ -40,9 +47,47 @@ impl LanguageDictionary {
         })
     }
 
+    fn try_build_transformed_statement(
+        &self,
+        mut op: Box<dyn Op>,
+        tokens: &mut VecDeque<Token>,
+    ) -> Result<Box<dyn Op>, SyntaxError> {
+        //println!("building transformer for op {} with tokens {:?}", op, tokens);
+        let mut op_found = true;
+        while op_found && !tokens.is_empty() {
+            (op, op_found) = self.try_build_one_transform(op, tokens)?;
+        }
+        //println!("built transformed op {}, remaining tokens {:?}", op, tokens);
+        Ok(op)
+    }
+
+    pub fn try_build_one_transform(
+        &self,
+        mut op: Box<dyn Op>,
+        tokens: &mut VecDeque<Token>,
+    ) -> Result<(Box<dyn Op>, bool), SyntaxError> {
+        for factory in &self.transform_vocabulary {
+            if factory.is_transform_op(tokens) {
+                op = factory.build_transform_op(tokens, self, op)?;
+                return Ok((op, true));
+            }
+        }
+        Ok((op, false))
+    }
+
+    pub fn try_build_statement(
+        &self,
+        tokens: &mut VecDeque<Token>,
+    ) -> Result<Box<dyn Op>, SyntaxError> {
+        let root = self.try_build_root_statement(tokens)?;
+        //println!("built root op {}, remaining tokens {:?}", root, tokens);
+        self.try_build_transformed_statement(root, tokens)
+    }
+
     pub fn new() -> Self {
         Self {
-            vocabulary: Vec::new(),
+            root_vocabulary: Vec::new(),
+            transform_vocabulary: Vec::new(),
         }
     }
 
@@ -53,10 +98,12 @@ impl LanguageDictionary {
     }
 }
 
+#[allow(clippy::derivable_impls)]
 impl Default for LanguageDictionary {
     fn default() -> Self {
         Self {
-            vocabulary: Vec::new(),
+            root_vocabulary: Vec::new(),
+            transform_vocabulary: Vec::new(),
         }
     }
 }

interpreter/src/lang/filter.rs

@@ -3,11 +3,11 @@ use std::fmt::{Debug, Display, Error, Formatter};
 use std::iter::Iterator;
 use std::marker::PhantomData;
 
-use crate::lang::utility::{assert_name, assert_token, assert_token_raw, check_name};
+use crate::lang::utility::{assert_name, assert_token_raw, check_name};
 use crate::lang::FilterReplaceStatement;
 use crate::lang::LanguageDictionary;
 use crate::lang::SingleItem;