Create initial language functionality and framework

.gitignore

@@ -0,0 +1,5 @@
+/target
+**/target
+/*/metadata.mps.sqlite
+metadata.mps.sqlite
+**.m3u8

Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "mps"
+version = "0.1.0"
+edition = "2021"
+authors = ["NGnius (Graham) <ngniusness@gmail.com>"]
+description = "Music Playlist Scripting language (MPS)"
+
+[workspace]
+members = [
+    "mps-interpreter",
+    "mps-player"
+]
+
+[dependencies]
+# local
+mps-interpreter = { path = "./mps-interpreter" }
+mps-player = { path = "./mps-player" }

mps-interpreter/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "mps-interpreter"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+rusqlite = { version = "0.26.1" }
+symphonia = { version = "0.4.0", optional = true, features = [
+    "aac", "flac", "mp3", "pcm", "vorbis", "isomp4", "ogg", "wav"
+] }
+dirs = { version = "4.0.0", optional = true}
+
+[features]
+default = [ "music_library" ]
+music_library = [ "symphonia", "dirs" ] # song metadata parsing and database auto-population

mps-interpreter/src/context.rs

@@ -0,0 +1,29 @@
+use std::fmt::{Debug, Display, Formatter, Error};
+
+#[derive(Debug)]
+pub struct MpsContext {
+    pub sqlite_connection: Option<rusqlite::Connection>,
+}
+
+impl Default for MpsContext {
+    fn default() -> Self {
+        Self {
+            sqlite_connection: None, // initialized by first SQL statement instead
+        }
+    }
+}
+
+impl std::clone::Clone for MpsContext {
+    fn clone(&self) -> Self {
+        Self {
+            sqlite_connection: None,
+        }
+    }
+}
+
+impl Display for MpsContext {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "MpsContext")?;
+        Ok(())
+    }
+}

mps-interpreter/src/interpretor.rs

@@ -0,0 +1,140 @@
+use std::iter::Iterator;
+use std::collections::VecDeque;
+use std::path::Path;
+use std::fs::File;
+
+use super::MpsMusicItem;
+use super::MpsContext;
+use super::tokens::MpsToken;
+use super::lang::{MpsOp, MpsLanguageError, MpsLanguageDictionary};
+
+pub struct MpsInterpretor<T> where T: crate::tokens::MpsTokenReader {
+    tokenizer: T,
+    buffer: VecDeque<MpsToken>,
+    current_stmt: Option<Box<dyn MpsOp>>,
+    vocabulary: MpsLanguageDictionary,
+    context: Option<MpsContext>,
+}
+
+pub fn interpretor<R: std::io::Read>(stream: R) -> MpsInterpretor<crate::tokens::MpsTokenizer<R>> {
+    let tokenizer = crate::tokens::MpsTokenizer::new(stream);
+    MpsInterpretor::with_standard_vocab(tokenizer)
+}
+
+impl<T> MpsInterpretor<T>
+    where T: crate::tokens::MpsTokenReader
+{
+    pub fn with_vocab(tokenizer: T, vocab: MpsLanguageDictionary) -> Self {
+        Self {
+            tokenizer: tokenizer,
+            buffer: VecDeque::new(),
+            current_stmt: None,
+            vocabulary: vocab,
+            context: None,
+        }
+    }
+
+    pub fn with_standard_vocab(tokenizer: T) -> Self {
+        let mut result = Self {
+            tokenizer: tokenizer,
+            buffer: VecDeque::new(),
+            current_stmt: None,
+            vocabulary: MpsLanguageDictionary::default(),
+            context: None,
+        };
+        standard_vocab(&mut result.vocabulary);
+        result
+    }
+
+    pub fn context(&mut self, ctx: MpsContext) {
+        self.context = Some(ctx)
+    }
+
+    pub fn is_done(&self) -> bool {
+        self.tokenizer.end_of_file() && self.current_stmt.is_none()
+    }
+}
+
+impl MpsInterpretor<crate::tokens::MpsTokenizer<File>> {
+    pub fn standard_file<P: AsRef<Path>>(path: P) -> std::io::Result<Self> {
+        let file = File::open(path)?;
+        let tokenizer = crate::tokens::MpsTokenizer::new(file);
+        let mut result = Self {
+            tokenizer: tokenizer,
+            buffer: VecDeque::new(),
+            current_stmt: None,
+            vocabulary: MpsLanguageDictionary::default(),
+            context: None,
+        };
+        standard_vocab(&mut result.vocabulary);
+        Ok(result)
+    }
+}
+
+impl<T> Iterator for MpsInterpretor<T>
+    where T: crate::tokens::MpsTokenReader
+{
+    type Item = Result<MpsMusicItem, Box<dyn MpsLanguageError>>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let mut is_stmt_done = false;
+        let result = if let Some(stmt) = &mut self.current_stmt {
+            let next_item = stmt.next();
+            if next_item.is_none() {
+                is_stmt_done = true;
+            }
+            match next_item {
+                Some(item) => Some(item.map_err(|e| box_error_with_ctx(
+                    e, self.tokenizer.current_line()
+                ))),
+                None => None
+            }
+        } else {
+            if self.tokenizer.end_of_file() { return None; }
+            // build new statement
+            let token_result = self.tokenizer.next_statements(1, &mut self.buffer)
+                .map_err(|e| box_error_with_ctx(e, self.tokenizer.current_line()));
+            match token_result {
+                Ok(_) => {},
+                Err(x) => return Some(Err(x))
+            }
+            if self.tokenizer.end_of_file() && self.buffer.len() == 0 { return None; }
+            let stmt = self.vocabulary.try_build_statement(&mut self.buffer);
+            match stmt {
+                Ok(mut stmt) => {
+                    stmt.enter(self.context.take().unwrap_or_else(|| MpsContext::default()));
+                    self.current_stmt = Some(stmt);
+                    let next_item = self.current_stmt.as_mut().unwrap().next();
+                    if next_item.is_none() {
+                        is_stmt_done = true;
+                    }
+                    match next_item {
+                        Some(item) => Some(item.map_err(|e| box_error_with_ctx(
+                            e,
+                            self.tokenizer.current_line()
+                        ))),
+                        None => None
+                    }
+                },
+                Err(e) => Some(Err(e).map_err(|e| box_error_with_ctx(
+                    e,
+                    self.tokenizer.current_line()
+                )))
+            }
+        };
+        if is_stmt_done {
+            self.context = Some(self.current_stmt.take().unwrap().escape());
+        }
+        result
+    }
+}
+
+fn box_error_with_ctx<E: MpsLanguageError + 'static>(mut error: E, line: usize) -> Box<dyn MpsLanguageError> {
+    error.set_line(line);
+    Box::new(error) as Box<dyn MpsLanguageError>
+}
+
+pub(crate) fn standard_vocab(vocabulary: &mut MpsLanguageDictionary) {
+    vocabulary
+            .add(crate::lang::vocabulary::SqlStatementFactory);
+}

mps-interpreter/src/lang/db_items.rs

@@ -0,0 +1,287 @@
+pub const DEFAULT_SQLITE_FILEPATH: &str = "metadata.mps.sqlite";
+
+pub trait DatabaseObj: Sized {
+    fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self>;
+
+    fn to_params(&self) -> Vec<&'_ dyn rusqlite::ToSql>;
+
+    fn id(&self) -> u64;
+}
+
+pub fn generate_default_db() -> rusqlite::Result<rusqlite::Connection> {
+    let db_exists = std::path::Path::new(DEFAULT_SQLITE_FILEPATH).exists();
+    let conn = rusqlite::Connection::open(DEFAULT_SQLITE_FILEPATH)?;
+    // skip db building if SQLite file already exists
+    // TODO do a more exhaustive db check to make sure it's actually the correct file
+    if db_exists {return Ok(conn);}
+    // build db tables
+    conn.execute_batch(
+        "BEGIN;
+        CREATE TABLE IF NOT EXISTS songs (
+            song_id INTEGER NOT NULL PRIMARY KEY,
+            title TEXT NOT NULL,
+            artist INTEGER NOT NULL,
+            album INTEGER,
+            filename TEXT,
+            metadata INTEGER NOT NULL,
+            genre INTEGER NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS artists (
+            artist_id INTEGER NOT NULL PRIMARY KEY,
+            name TEXT NOT NULL,
+            genre INTEGER NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS albums (
+            album_id INTEGER NOT NULL PRIMARY KEY,
+            title TEXT,
+            metadata INTEGER NOT NULL,
+            artist INTEGER NOT NULL,
+            genre INTEGER NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS metadata (
+            meta_id INTEGER NOT NULL PRIMARY KEY,
+            plays INTEGER NOT NULL DEFAULT 0,
+            track INTEGER NOT NULL DEFAULT 1,
+            disc INTEGER NOT NULL DEFAULT 1,
+            duration INTEGER,
+            date INTEGER
+        );
+        CREATE TABLE IF NOT EXISTS genres (
+            genre_id INTEGER NOT NULL PRIMARY KEY,
+            title TEXT
+        );
+        COMMIT;"
+    )?;
+    // generate data and store in db
+    #[cfg(feature = "music_library")]
+    {
+        let music_path = super::utility::music_folder();
+        match crate::music::build_library(&music_path) {
+            Ok(lib) => {
+                let mut song_insert = conn.prepare(
+                    "INSERT OR REPLACE INTO songs (
+                        song_id,
+                        title,
+                        artist,
+                        album,
+                        filename,
+                        metadata,
+                        genre
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?)"
+                )?;
+                for song in lib.all_songs() {
+                    song_insert.execute(song.to_params().as_slice())?;
+                }
+
+                let mut metadata_insert = conn.prepare(
+                    "INSERT OR REPLACE INTO metadata (
+                        meta_id,
+                        plays,
+                        track,
+                        disc,
+                        duration,
+                        date
+                    ) VALUES (?, ?, ?, ?, ?, ?)"
+                )?;
+                for meta in lib.all_metadata() {
+                    metadata_insert.execute(meta.to_params().as_slice())?;
+                }
+
+                let mut artist_insert = conn.prepare(
+                    "INSERT OR REPLACE INTO artists (
+                        artist_id,
+                        name,
+                        genre
+                    ) VALUES (?, ?, ?)"
+                )?;
+                for artist in lib.all_artists() {
+                    artist_insert.execute(artist.to_params().as_slice())?;
+                }
+
+                let mut album_insert = conn.prepare(
+                    "INSERT OR REPLACE INTO albums (
+                        album_id,
+                        title,
+                        metadata,
+                        artist,
+                        genre
+                    ) VALUES (?, ?, ?, ?, ?)"
+                )?;
+                for album in lib.all_albums() {
+                    album_insert.execute(album.to_params().as_slice())?;
+                }
+
+                let mut genre_insert = conn.prepare(
+                    "INSERT OR REPLACE INTO genres (
+                        genre_id,
+                        title
+                    ) VALUES (?, ?)"
+                )?;
+                for genre in lib.all_genres() {
+                    genre_insert.execute(genre.to_params().as_slice())?;
+                }
+            },
+            Err(e) => println!("Unable to load music from {}: {}", music_path.display(), e)
+        }
+    }
+    Ok(conn)
+}
+
+#[derive(Clone, Debug)]
+pub struct DbMusicItem {
+    pub song_id: u64,
+    pub title: String,
+    pub artist: u64,
+    pub album: Option<u64>,
+    pub filename: String,
+    pub metadata: u64,
+    pub genre: u64,
+}
+
+impl DatabaseObj for DbMusicItem {
+    fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self> {
+        Ok(Self{
+            song_id: row.get(0)?,
+            title: row.get(1)?,
+            artist: row.get(2)?,
+            album: row.get(3)?,
+            filename: row.get(4)?,
+            metadata: row.get(5)?,
+            genre: row.get(6)?,
+        })
+    }
+
+    fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> {
+        vec![
+            &self.song_id,
+            &self.title,
+            &self.artist,
+            &self.album,
+            &self.filename,
+            &self.metadata,
+            &self.genre,
+        ]
+    }
+
+    fn id(&self) -> u64 {self.song_id}
+}
+
+#[derive(Clone, Debug)]
+pub struct DbMetaItem {
+    pub meta_id: u64,
+    pub plays: u64,
+    pub track: u64,
+    pub disc: u64,
+    pub duration: u64, // seconds
+    pub date: u64, // year
+}
+
+impl DatabaseObj for DbMetaItem {
+    fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self> {
+        Ok(Self{
+            meta_id: row.get(0)?,
+            plays: row.get(1)?,
+            track: row.get(2)?,
+            disc: row.get(3)?,
+            duration: row.get(4)?,
+            date: row.get(5)?,
+        })
+    }
+
+    fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> {
+        vec![
+            &self.meta_id,
+            &self.plays,
+            &self.track,
+            &self.disc,
+            &self.duration,
+            &self.date,
+        ]
+    }
+
+    fn id(&self) -> u64 {self.meta_id}
+}
+
+#[derive(Clone, Debug)]
+pub struct DbArtistItem {
+    pub artist_id: u64,
+    pub name: String,
+    pub genre: u64,
+}
+
+impl DatabaseObj for DbArtistItem {
+    fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self> {
+        Ok(Self{
+            artist_id: row.get(0)?,
+            name: row.get(1)?,
+            genre: row.get(2)?,
+        })
+    }
+
+    fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> {
+        vec![
+            &self.artist_id,
+            &self.name,
+            &self.genre,
+        ]
+    }
+
+    fn id(&self) -> u64 {self.artist_id}
+}
+
+#[derive(Clone, Debug)]
+pub struct DbAlbumItem {
+    pub album_id: u64,
+    pub title: String,
+    pub metadata: u64,
+    pub artist: u64,
+    pub genre: u64,
+}
+
+impl DatabaseObj for DbAlbumItem {
+    fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self> {
+        Ok(Self{
+            album_id: row.get(0)?,
+            title: row.get(1)?,
+            metadata: row.get(2)?,
+            artist: row.get(3)?,
+            genre: row.get(4)?,
+        })
+    }
+
+    fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> {
+        vec![
+            &self.album_id,
+            &self.title,
+            &self.metadata,
+            &self.artist,
+            &self.genre,
+        ]
+    }
+
+    fn id(&self) -> u64 {self.album_id}
+}
+
+#[derive(Clone, Debug)]
+pub struct DbGenreItem {
+    pub genre_id: u64,
+    pub title: String,
+}
+
+impl DatabaseObj for DbGenreItem {
+    fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self> {
+        Ok(Self{
+            genre_id: row.get(0)?,
+            title: row.get(1)?,
+        })
+    }
+
+    fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> {
+        vec![
+            &self.genre_id,
+            &self.title,
+        ]
+    }
+
+    fn id(&self) -> u64 {self.genre_id}
+}

mps-interpreter/src/lang/dictionary.rs

@@ -0,0 +1,42 @@
+use std::collections::VecDeque;
+
+use crate::tokens::MpsToken;
+use super::{BoxedMpsOpFactory, MpsOp};
+use super::SyntaxError;
+
+pub struct MpsLanguageDictionary {
+    vocabulary: Vec<Box<dyn BoxedMpsOpFactory>>
+}
+
+impl MpsLanguageDictionary {
+    pub fn add<T: BoxedMpsOpFactory + 'static>(&mut self, factory: T) -> &mut Self {
+        self.vocabulary.push(Box::new(factory) as Box<dyn BoxedMpsOpFactory>);
+        self
+    }
+
+    pub fn try_build_statement(&self, tokens: &mut VecDeque<MpsToken>) -> Result<Box<dyn MpsOp>, SyntaxError> {
+        for factory in &self.vocabulary {
+            if factory.is_op_boxed(tokens) {
+                return factory.build_op_boxed(tokens);
+            }
+        }
+        Err(SyntaxError {
+            line: 0,
+            token: tokens.pop_front().unwrap()
+        })
+    }
+
+    pub fn new() -> Self {
+        Self {
+            vocabulary: Vec::new(),
+        }
+    }
+}
+
+impl Default for MpsLanguageDictionary {
+    fn default() -> Self {
+        Self {
+            vocabulary: Vec::new(),
+        }
+    }
+}

mps-interpreter/src/lang/error.rs

@@ -0,0 +1,41 @@
+use std::fmt::{Debug, Display, Formatter, Error};
+
+use crate::tokens::MpsToken;
+use super::MpsOp;
+
+#[derive(Debug)]
+pub struct SyntaxError {
+    pub line: usize,
+    pub token: MpsToken,
+}
+
+impl Display for SyntaxError {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "SyntaxError (line {}): Unexpected {}", &self.line, &self.token)
+    }
+}
+
+impl MpsLanguageError for SyntaxError {
+    fn set_line(&mut self, line: usize) {self.line = line}
+}
+
+#[derive(Debug)]
+pub struct RuntimeError {
+    pub line: usize,
+    pub op: Box<dyn MpsOp>,
+    pub msg: String,
+}
+
+impl Display for RuntimeError {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "{} (line {}): {}", &self.msg, &self.line, &self.op)
+    }
+}
+
+impl MpsLanguageError for RuntimeError {
+    fn set_line(&mut self, line: usize) {self.line = line}
+}
+
+pub trait MpsLanguageError: Display + Debug {
+    fn set_line(&mut self, line: usize);
+}

mps-interpreter/src/lang/mod.rs

@@ -0,0 +1,29 @@
+mod db_items;
+mod dictionary;
+mod error;
+mod operation;
+mod sql_query;
+//mod statement;
+pub(crate) mod utility;
+
+pub use dictionary::MpsLanguageDictionary;
+pub use error::{SyntaxError, RuntimeError, MpsLanguageError};
+pub use operation::{MpsOp, MpsOpFactory, BoxedMpsOpFactory};
+//pub(crate) use statement::MpsStatement;
+
+pub mod vocabulary {
+    pub use super::sql_query::{SqlStatement, SqlStatementFactory};
+}
+
+pub mod db {
+    pub use super::db_items::{DEFAULT_SQLITE_FILEPATH, generate_default_db, DatabaseObj, DbMusicItem, DbAlbumItem, DbArtistItem, DbMetaItem, DbGenreItem};
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn db_build_test() -> rusqlite::Result<()> {
+        super::db::generate_default_db()?;
+        Ok(())
+    }
+}

mps-interpreter/src/lang/operation.rs

@@ -0,0 +1,31 @@
+use std::iter::Iterator;
+use std::collections::VecDeque;
+use std::fmt::{Debug, Display};
+
+use crate::MpsMusicItem;
+use crate::MpsContext;
+use crate::tokens::MpsToken;
+use super::{SyntaxError, RuntimeError};
+
+pub trait MpsOpFactory<T: MpsOp + 'static> {
+    fn is_op(&self, tokens: &VecDeque<MpsToken>) -> bool;
+
+    fn build_op(&self, tokens: &mut VecDeque<MpsToken>) -> Result<T, SyntaxError>;
+
+    #[inline]
+    fn build_box(&self, tokens: &mut VecDeque<MpsToken>) -> Result<Box<dyn MpsOp>, SyntaxError> {
+        Ok(Box::new(self.build_op(tokens)?))
+    }
+}
+
+pub trait BoxedMpsOpFactory {
+    fn build_op_boxed(&self, tokens: &mut VecDeque<MpsToken>) -> Result<Box<dyn MpsOp>, SyntaxError>;
+
+    fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool;
+}
+
+pub trait MpsOp: Iterator<Item=Result<MpsMusicItem, RuntimeError>> + Debug + Display {
+    fn enter(&mut self, ctx: MpsContext);
+
+    fn escape(&mut self) -> MpsContext;
+}

mps-interpreter/src/lang/sql_query.rs

@@ -0,0 +1,166 @@
+use std::iter::Iterator;
+use std::collections::VecDeque;
+use std::fmt::{Debug, Display, Formatter, Error};
+
+use crate::MpsContext;
+use crate::MpsMusicItem;
+use crate::tokens::MpsToken;
+use super::{MpsOp, MpsOpFactory, BoxedMpsOpFactory};
+use super::{SyntaxError, RuntimeError};
+use super::utility::{assert_token, assert_token_raw};
+use super::db::*;
+
+#[derive(Debug)]
+pub struct SqlStatement {
+    query: String,
+    context: Option<MpsContext>,
+    rows: Option<Vec<rusqlite::Result<MpsMusicItem>>>,
+    current: usize,
+}
+
+impl SqlStatement {
+    fn map_item(&mut self, increment: bool) -> Option<Result<MpsMusicItem, RuntimeError>> {
+        if let Some(rows) = &self.rows {
+            if increment {
+                if self.current == rows.len() {
+                    return None
+                }
+                self.current += 1;
+            }
+            if self.current >= rows.len() {
+                None
+            } else {
+                match &rows[self.current] {
+                    Ok(item) => Some(Ok(item.clone())),
+                    Err(e) => Some(Err(RuntimeError {
+                        line: 0,
+                        op: Box::new(self.clone()),
+                        msg: format!("SQL music item mapping error: {}", e).into(),
+                    }))
+                }
+            }
+        } else {
+            Some(Err(RuntimeError {
+                line: 0,
+                op: Box::new(self.clone()),
+                msg: format!("Context error: rows is None").into(),
+            }))
+        }
+
+    }
+}
+
+impl MpsOp for SqlStatement {
+    fn enter(&mut self, ctx: MpsContext){
+        self.context = Some(ctx)
+    }
+
+    fn escape(&mut self) -> MpsContext {
+        self.context.take().unwrap()
+    }
+}
+
+impl std::clone::Clone for SqlStatement {
+    fn clone(&self) -> Self {
+        Self {
+            query: self.query.clone(),
+            context: self.context.clone(),
+            rows: None, // TODO use different Result type so this is cloneable
+            current: self.current,
+        }
+    }
+}
+
+impl Iterator for SqlStatement {
+    type Item = Result<MpsMusicItem, RuntimeError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.rows.is_some() {
+            // query has executed, return another result
+            self.map_item(true)
+        } else {
+            let ctx = self.context.as_mut().unwrap();
+            // query has not been executed yet
+            if let None = ctx.sqlite_connection {
+                // connection needs to be created
+                match generate_default_db() {
+                    Ok(conn) => ctx.sqlite_connection = Some(conn),
+                    Err(e) => return Some(Err(RuntimeError{
+                        line: 0,
+                        op: Box::new(self.clone()),
+                        msg: format!("SQL connection error: {}", e).into()
+                    }))
+                }
+            }
+            let conn = ctx.sqlite_connection.as_mut().unwrap();
+            // execute query
+            match perform_query(conn, &self.query) {
+                Ok(items) => {
+                    self.rows = Some(items);
+                    self.map_item(false)
+                }
+                Err(e) => Some(Err(RuntimeError{
+                    line: 0,
+                    op: Box::new(self.clone()),
+                    msg: e
+                }))
+            }
+        }
+    }
+}
+
+impl Display for SqlStatement {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "sql(`{}`)", &self.query)
+    }
+}
+
+pub struct SqlStatementFactory;
+
+impl MpsOpFactory<SqlStatement> for SqlStatementFactory {
+    #[inline]
+    fn is_op(&self, tokens: &VecDeque<MpsToken>) -> bool {
+        tokens.len() > 3 && tokens[0].is_sql()
+    }
+
+    #[inline]
+    fn build_op(&self, tokens: &mut VecDeque<MpsToken>) -> Result<SqlStatement, SyntaxError> {
+        // sql ( `some query` )
+        assert_token_raw(MpsToken::Sql, tokens)?;
+        assert_token_raw(MpsToken::OpenBracket, tokens)?;
+        let literal = assert_token(|t| {
+            match t {
+                MpsToken::Literal(query) => Some(query),
+                _ => None
+            }
+        }, MpsToken::Literal("".into()), tokens)?;
+        assert_token_raw(MpsToken::CloseBracket, tokens)?;
+        Ok(SqlStatement {
+            query: literal,
+            context: None,
+            current: 0,
+            rows: None,
+        })
+    }
+}
+
+impl BoxedMpsOpFactory for SqlStatementFactory {
+    fn build_op_boxed(&self, tokens: &mut VecDeque<MpsToken>) -> Result<Box<dyn MpsOp>, SyntaxError> {
+        self.build_box(tokens)
+    }
+
+    fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool {
+        self.is_op(tokens)
+    }
+}
+
+fn perform_query(
+    conn: &mut rusqlite::Connection,
+    query: &str
+) -> Result<Vec<rusqlite::Result<MpsMusicItem>>, String> {
+    let mut stmt = conn.prepare(query)
+        .map_err(|e| format!("SQLite query error: {}", e))?;
+    let iter = stmt.query_map([], MpsMusicItem::map_row)
+        .map_err(|e| format!("SQLite item mapping error: {}", e))?;
+    Ok(iter.collect())
+}

mps-interpreter/src/lang/statement.rs

@@ -0,0 +1,40 @@
+use std::iter::Iterator;
+use std::fmt::{Debug, Display, Formatter, Error};
+
+use std::collections::VecDeque;
+
+use crate::tokens::MpsToken;
+use crate::MpsMusicItem;
+
+use super::SqlStatement;
+use super::{SyntaxError, RuntimeError};
+use super::MpsLanguageDictionary;
+
+#[derive(Debug)]
+pub enum MpsStatement {
+    Sql(SqlStatement),
+}
+
+impl MpsStatement {
+    pub fn eat_some(tokens: &mut VecDeque<MpsToken>, vocab: MpsLanguageDictionary) -> Result<Self, SyntaxError> {
+        vocab.try_build_statement(tokens)
+    }
+}
+
+impl Iterator for MpsStatement {
+    type Item = Result<MpsMusicItem, RuntimeError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self {
+            MpsStatement::Sql(s) => s.next(),
+        }
+    }
+}
+
+impl Display for MpsStatement {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        match self {
+            Self::Sql(s) => write!(f, "{}", s),
+        }
+    }
+}

mps-interpreter/src/lang/utility.rs

@@ -0,0 +1,41 @@
+use std::collections::VecDeque;
+#[cfg(feature = "music_library")]
+use std::path::PathBuf;
+
+use crate::tokens::MpsToken;
+use super::SyntaxError;
+
+pub fn assert_token<T, F: FnOnce(MpsToken) -> Option<T>>(
+    caster: F,
+    token: MpsToken,
+    tokens: &mut VecDeque<MpsToken>
+) -> Result<T, SyntaxError> {
+    if let Some(out) = caster(tokens.pop_front().unwrap()) {
+        Ok(out)
+    } else {
+        Err(SyntaxError{
+            line: 0,
+            token: token,
+        })
+    }
+}
+
+pub fn assert_token_raw(
+    token: MpsToken,
+    tokens: &mut VecDeque<MpsToken>
+) -> Result<MpsToken, SyntaxError> {
+    let result = tokens.pop_front().unwrap();
+    if std::mem::discriminant(&token) == std::mem::discriminant(&result) {
+        Ok(result)
+    } else {
+        Err(SyntaxError {
+            line: 0,
+            token: token,
+        })
+    }
+}
+
+#[cfg(feature = "music_library")]
+pub fn music_folder() -> PathBuf {
+    dirs::home_dir().unwrap_or_else(|| PathBuf::from("./")).join("Music")
+}

mps-interpreter/src/lib.rs

@@ -0,0 +1,16 @@
+mod context;
+mod interpretor;
+mod runner;
+mod music_item;
+pub mod lang;
+#[cfg(feature = "music_library")]
+pub mod music;
+pub mod tokens;
+
+pub use context::MpsContext;
+pub use interpretor::{MpsInterpretor, interpretor};
+pub use runner::MpsRunner;
+pub use music_item::MpsMusicItem;
+
+#[cfg(test)]
+mod tests {}

mps-interpreter/src/music/build_library.rs

@@ -0,0 +1,9 @@
+use std::path::Path;
+
+use super::MpsLibrary;
+
+pub fn build_library<P: AsRef<Path>>(path: P) -> std::io::Result<MpsLibrary> {
+    let mut result = MpsLibrary::new();
+    result.read_path(path, 10)?;
+    Ok(result)
+}

mps-interpreter/src/music/library.rs

@@ -0,0 +1,156 @@
+use std::collections::HashMap;
+use std::path::Path;
+
+use symphonia::core::io::MediaSourceStream;
+use symphonia::core::probe::Hint;
+
+use crate::lang::db::*;
+use super::tag::Tags;
+
+#[derive(Clone, Default)]
+pub struct MpsLibrary {
+    songs: HashMap<u64, DbMusicItem>,
+    metadata: HashMap<u64, DbMetaItem>,
+    artists: HashMap<String, DbArtistItem>,
+    albums: HashMap<String, DbAlbumItem>,
+    genres: HashMap<String, DbGenreItem>,
+}
+
+impl MpsLibrary {
+    pub fn new() -> Self {
+        Self {
+            songs: HashMap::new(),
+            metadata: HashMap::new(),
+            artists: HashMap::new(),
+            albums: HashMap::new(),
+            genres: HashMap::new(),
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.songs.len()
+    }
+
+    pub fn all_songs<'a>(&'a self) -> Vec<&'a DbMusicItem> {
+        self.songs.values().collect()
+    }
+
+    pub fn all_metadata<'a>(&'a self) -> Vec<&'a DbMetaItem> {
+        self.metadata.values().collect()
+    }
+
+    pub fn all_artists<'a>(&'a self) -> Vec<&'a DbArtistItem> {
+        self.artists.values().collect()
+    }
+
+    pub fn all_albums<'a>(&'a self) -> Vec<&'a DbAlbumItem> {
+        self.albums.values().collect()
+    }
+
+    pub fn all_genres<'a>(&'a self) -> Vec<&'a DbGenreItem> {
+        self.genres.values().collect()
+    }
+
+    pub fn read_path<P: AsRef<Path>>(&mut self, path: P, depth: usize) -> std::io::Result<()> {
+        let path = path.as_ref();
+        if path.is_dir() && depth != 0 {
+            for entry in path.read_dir()? {
+                self.read_path(entry?.path(), depth-1)?;
+            }
+        } else if path.is_file() {
+            self.read_file(path)?;
+        }
+        Ok(())
+    }
+
+    fn read_file<P: AsRef<Path>>(&mut self, path: P) -> std::io::Result<()> {
+        let path = path.as_ref();
+        let file = Box::new(std::fs::File::open(path)?);
+        // use symphonia to get metadata
+        let mss = MediaSourceStream::new(file, Default::default() /* options */);
+        let probed = symphonia::default::get_probe().format(
+            &Hint::new(),
+            mss,
+            &Default::default(),
+            &Default::default()
+        );
+        // process audio file, ignoring any processing errors (skip file on error)
+        if let Ok(mut probed) = probed {
+            let mut tags = Tags::new(path);
+            // collect metadata
+            if let Some(metadata) = probed.metadata.get() {
+                if let Some(rev) = metadata.current() {
+                    for tag in rev.tags() {
+                        //println!("(pre) metadata tag ({},{})", tag.key, tag.value);
+                        tags.add(tag.key.clone(), &tag.value);
+                    }
+                }
+            }
+            if let Some(rev) = probed.format.metadata().current() {
+                for tag in rev.tags() {
+                    //println!("(post) metadata tag ({},{})", tag.key, tag.value);
+                    tags.add(tag.key.clone(), &tag.value);
+                }
+            }
+            self.generate_entries(&tags);
+        }
+        Ok(())
+    }
+
+    /// generate data structures and links
+    fn generate_entries(&mut self, tags: &Tags) {
+        if tags.len() == 0 { return; } // probably not a valid song, let's skip it
+        let song_id = self.songs.len() as u64; // guaranteed to be created
+        let meta_id = self.metadata.len() as u64; // guaranteed to be created
+        self.metadata.insert(meta_id, tags.meta(meta_id)); // definitely necessary
+        // genre has no links to others, so find that first
+        let mut genre = tags.genre(0);
+        genre.genre_id = Self::find_or_gen_id(&self.genres, &genre.title);
+        if genre.genre_id == self.genres.len() as u64 {
+            self.genres.insert(Self::sanitise_key(&genre.title), genre.clone());
+        }
+        // artist only links to genre, so that can be next
+        let mut artist = tags.artist(0, genre.genre_id);
+        artist.artist_id = Self::find_or_gen_id(&self.artists, &artist.name);
+        if artist.artist_id == self.artists.len() as u64 {
+            self.artists.insert(Self::sanitise_key(&artist.name), artist.clone());
+        }
+        // same with album artist
+        let mut album_artist = tags.album_artist(0, genre.genre_id);
+        album_artist.artist_id = Self::find_or_gen_id(&self.artists, &album_artist.name);
+        if album_artist.artist_id == self.artists.len() as u64 {
+            self.artists.insert(Self::sanitise_key(&album_artist.name), album_artist.clone());
+        }
+        // album now has all links ready
+        let mut album = tags.album(0, 0, album_artist.artist_id, genre.genre_id);
+        album.album_id = Self::find_or_gen_id(&self.albums, &album.title);
+        if album.album_id == self.albums.len() as u64 {
+            let album_meta = tags.album_meta(self.metadata.len() as u64);
+            album.metadata = album_meta.meta_id;
+            self.albums.insert(Self::sanitise_key(&album.title), album.clone());
+            self.metadata.insert(album_meta.meta_id, album_meta);
+        }
+        //let meta_album_id = self.metadata.len() as u64;
+        //let album = tags.album(album_id, meta_album_id);
+        self.songs.insert(song_id, tags.song(song_id, artist.artist_id, Some(album.album_id), meta_id, genre.genre_id));
+    }
+
+    #[inline]
+    fn find_or_gen_id<'a, D: DatabaseObj>(map: &'a HashMap<String, D>, key: &str) -> u64 {
+        if let Some(obj) = Self::find_by_key(map, key) {
+            obj.id()
+        } else {
+            map.len() as u64
+        }
+    }
+
+    #[inline(always)]
+    fn find_by_key<'a, D: DatabaseObj>(map: &'a HashMap<String, D>, key: &str) -> Option<&'a D> {
+        map.get(&Self::sanitise_key(key))
+    }
+
+    #[inline(always)]
+    fn sanitise_key(key: &str) -> String {
+        key.trim().to_lowercase()
+    }
+}

mps-interpreter/src/music/mod.rs

@@ -0,0 +1,6 @@
+mod build_library;
+mod library;
+mod tag;
+
+pub use build_library::build_library;
+pub use library::MpsLibrary;

mps-interpreter/src/music/tag.rs

@@ -0,0 +1,145 @@
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+use symphonia::core::meta::Value;
+
+use crate::lang::db::*;
+
+pub struct Tags {
+    data: HashMap<String, TagType>,
+    filename: PathBuf,
+}
+
+impl Tags {
+    pub fn new<P: AsRef<Path>>(path: P) -> Self {
+        Self {
+            data: HashMap::new(),
+            filename: path.as_ref().canonicalize().unwrap(),
+        }
+    }
+
+    pub fn add(&mut self, key: String, value: &Value) {
+        if let Some(tag_type) = TagType::from_symphonia_value(value) {
+            self.data.insert(key.trim().to_uppercase(), tag_type);
+        }
+    }
+
+    pub fn len(&self) -> usize {self.data.len()}
+
+    pub fn song(&self, id: u64, artist_id: u64, album_id: Option<u64>, meta_id: u64, genre_id: u64) -> DbMusicItem {
+        let default_title = || {
+            let extension = self.filename.extension().and_then(|ext| ext.to_str()).unwrap_or("");
+            self.filename.file_name()
+                .and_then(|file| file.to_str())
+                .and_then(|file| Some(file.replacen(extension, "", 1)))
+                .unwrap_or("Unknown Title".into())
+        };
+        DbMusicItem {
+            song_id: id,
+            title:  self.data.get("TITLE")
+                .unwrap_or(&TagType::Unknown).str()
+                .and_then(|s| Some(s.to_string()))
+                .unwrap_or_else(default_title),
+            artist: artist_id,
+            album: album_id,
+            filename: self.filename.to_str().unwrap_or("").into(),
+            metadata: meta_id,
+            genre: genre_id,
+        }
+    }
+
+    pub fn meta(&self, id: u64) -> DbMetaItem {
+        DbMetaItem {
+            meta_id: id,
+            plays: self.data.get("PLAYS").unwrap_or(&TagType::Unknown).uint().unwrap_or(0),
+            track: self.data.get("TRACKNUMBER").unwrap_or(&TagType::Unknown).uint().unwrap_or(id),
+            disc: self.data.get("DISCNUMBER").unwrap_or(&TagType::Unknown).uint().unwrap_or(1),
+            duration: self.data.get("DURATION").unwrap_or(&TagType::Unknown).uint().unwrap_or(0),
+            date: self.data.get("DATE").unwrap_or(&TagType::Unknown).uint().unwrap_or(0),
+        }
+    }
+
+    pub fn artist(&self, id: u64, genre_id: u64) -> DbArtistItem {
+        DbArtistItem {
+            artist_id: id,
+            name: self.data.get("ARTIST").unwrap_or(&TagType::Unknown).str().unwrap_or("Unknown Artist").into(),
+            genre: genre_id,
+        }
+    }
+
+    pub fn album_artist(&self, id: u64, genre_id: u64) -> DbArtistItem {
+        DbArtistItem {
+            artist_id: id,
+            name: self.data.get("ALBUMARTIST").unwrap_or(&TagType::Unknown).str().unwrap_or("Unknown Artist").into(),
+            genre: genre_id,
+        }
+    }
+
+    pub fn album(&self, id: u64, meta_id: u64, artist_id: u64, genre_id: u64) -> DbAlbumItem {
+        DbAlbumItem {
+            album_id: id,
+            title: self.data.get("ALBUM").unwrap_or(&TagType::Unknown).str().unwrap_or("Unknown Album").into(),
+            metadata: meta_id,
+            artist: artist_id,
+            genre: genre_id,
+        }
+    }
+
+    pub fn album_meta(&self, id: u64) -> DbMetaItem {
+        DbMetaItem {
+            meta_id: id,
+            plays: self.data.get("PLAYS").unwrap_or(&TagType::Unknown).uint().unwrap_or(0),
+            track: self.data.get("TRACKTOTAL").unwrap_or(&TagType::Unknown).uint().unwrap_or(0),
+            disc: self.data.get("DISCTOTAL").unwrap_or(&TagType::Unknown).uint().unwrap_or(1),
+            duration: 0,
+            date: self.data.get("DATE").unwrap_or(&TagType::Unknown).uint().unwrap_or(0),
+        }
+    }
+
+    pub fn genre(&self, id: u64) -> DbGenreItem {
+        DbGenreItem {
+            genre_id: id,
+            title: self.data.get("GENRE").unwrap_or(&TagType::Unknown).str().unwrap_or("Unknown Genre").into(),
+        }
+    }
+}
+
+#[derive(Clone)]
+enum TagType {
+    Boolean(bool),
+    Flag,
+    I64(i64),
+    U64(u64),
+    Str(String),
+    Unknown
+}
+
+impl TagType {
+    fn from_symphonia_value(value: &Value) -> Option<Self> {
+        match value {
+            Value::Binary(_val) => None,
+            Value::Boolean(b) => Some(Self::Boolean(*b)),
+            Value::Flag => Some(Self::Flag),
+            Value::Float(_val) => None,
+            Value::SignedInt(i) => Some(Self::I64(*i)),
+            Value::String(s) => Some(Self::Str(s.clone())),
+            Value::UnsignedInt(u) => Some(Self::U64(*u)),
+        }
+    }
+
+    fn str(&self) -> Option<&str> {
+        match self {
+            Self::Str(s) => Some(&s),
+            _ => None
+        }
+    }
+
+    fn uint(&self) -> Option<u64> {
+        match self {
+            Self::I64(i) => (*i).try_into().ok(),
+            Self::U64(u) => Some(*u),
+            Self::Str(s) => s.parse::<u64>().ok(),
+            _ => None
+        }
+    }
+}

mps-interpreter/src/music_item.rs

@@ -0,0 +1,17 @@
+use super::lang::db::{DatabaseObj, DbMusicItem};
+
+#[derive(Clone, Debug)]
+pub struct MpsMusicItem {
+    pub title: String,
+    pub filename: String,
+}
+
+impl MpsMusicItem {
+    pub fn map_row(row: &rusqlite::Row) -> rusqlite::Result<Self> {
+        let item = DbMusicItem::map_row(row)?;
+        Ok(Self{
+            title: item.title,
+            filename: item.filename,
+        })
+    }
+}

mps-interpreter/src/runner.rs

@@ -0,0 +1,70 @@
+use std::iter::Iterator;
+use std::io::Read;
+
+use super::{MpsInterpretor, MpsContext, MpsMusicItem};
+use super::tokens::{MpsTokenReader, MpsTokenizer};
+use super::lang::{MpsLanguageDictionary, MpsLanguageError};
+
+pub struct MpsRunnerSettings<T: MpsTokenReader> {
+    pub vocabulary: MpsLanguageDictionary,
+    pub tokenizer: T,
+    pub context: Option<MpsContext>,
+}
+
+impl<T: MpsTokenReader> MpsRunnerSettings<T> {
+    pub fn default_with_tokenizer(token_reader: T) -> Self {
+        let mut vocab = MpsLanguageDictionary::default();
+        super::interpretor::standard_vocab(&mut vocab);
+        Self {
+            vocabulary: vocab,
+            tokenizer: token_reader,
+            context: None,
+        }
+    }
+}
+
+pub struct MpsRunner<T: MpsTokenReader> {
+    interpretor: MpsInterpretor<T>,
+    new_statement: bool,
+}
+
+impl<T: MpsTokenReader> MpsRunner<T> {
+    pub fn with_settings(settings: MpsRunnerSettings<T>) -> Self {
+        let mut interpretor = MpsInterpretor::with_vocab(settings.tokenizer, settings.vocabulary);
+        if let Some(ctx) = settings.context {
+            interpretor.context(ctx);
+        }
+        Self {
+            interpretor: interpretor,
+            new_statement: true,
+        }
+    }
+
+    pub fn is_new_statement(&self) -> bool {
+        self.new_statement
+    }
+}
+
+impl<R: Read> MpsRunner<MpsTokenizer<R>> {
+    pub fn with_stream(stream: R) -> Self {
+        let tokenizer = MpsTokenizer::new(stream);
+        Self {
+            interpretor: MpsInterpretor::with_standard_vocab(tokenizer),
+            new_statement: true,
+        }
+    }
+}
+
+impl<T: MpsTokenReader> Iterator for MpsRunner<T> {
+    type Item = Result<MpsMusicItem, Box<dyn MpsLanguageError>>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let mut item = self.interpretor.next();
+        self.new_statement = false;
+        while item.is_none() && !self.interpretor.is_done() {
+            item = self.interpretor.next();
+            self.new_statement = true;
+        }
+        item
+    }
+}

mps-interpreter/src/tokens/error.rs

@@ -0,0 +1,39 @@
+use std::fmt::{Debug, Display, Formatter, Error};
+
+use crate::lang::MpsLanguageError;
+
+#[derive(Debug)]
+pub struct ParseError {
+    pub line: usize,
+    pub column: usize,
+    pub item: String,
+}
+
+impl Display for ParseError {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "ParseError (line {}, column {}): Unexpected {}", &self.line, &self.column, &self.item)
+    }
+}
+
+impl MpsTokenError for ParseError {
+    fn set_line(&mut self, line: usize) {self.line = line}
+
+    fn set_column(&mut self, column: usize) {self.column = column}
+}
+
+pub trait MpsTokenError: Display + Debug {
+    fn set_line(&mut self, line: usize);
+
+    fn set_column(&mut self, column: usize);
+
+    fn set_location(&mut self, line: usize, column: usize) {
+        self.set_line(line);
+        self.set_column(column);
+    }
+}
+
+impl<T: MpsTokenError> MpsLanguageError for T {
+    fn set_line(&mut self, line: usize) {
+        (self as &mut dyn MpsTokenError).set_line(line);
+    }
+}

mps-interpreter/src/tokens/mod.rs

@@ -0,0 +1,7 @@
+mod error;
+mod token_enum;
+mod tokenizer;
+
+pub use error::{ParseError, MpsTokenError};
+pub use token_enum::MpsToken;
+pub use tokenizer::{MpsTokenizer, MpsTokenReader};

mps-interpreter/src/tokens/token_enum.rs

@@ -0,0 +1,59 @@
+use std::fmt::{Debug, Display, Formatter, Error};
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum MpsToken {
+    Sql,
+    OpenBracket,
+    CloseBracket,
+    Literal(String),
+}
+
+impl MpsToken {
+    pub fn parse_from_string(s: String) -> Result<Self, String> {
+        match &s as &str {
+            "sql" => Ok(Self::Sql),
+            "(" => Ok(Self::OpenBracket),
+            ")" => Ok(Self::CloseBracket),
+            _ => Err(s),
+        }
+    }
+
+    pub fn is_sql(&self) -> bool {
+        match self {
+            Self::Sql => true,
+            _ => false
+        }
+    }
+
+    pub fn is_open_bracket(&self) -> bool {
+        match self {
+            Self::OpenBracket => true,
+            _ => false
+        }
+    }
+
+    pub fn is_close_bracket(&self) -> bool {
+        match self {
+            Self::CloseBracket => true,
+            _ => false
+        }
+    }
+
+    pub fn is_literal(&self) -> bool {
+        match self {
+            Self::Literal(_) => true,
+            _ => false
+        }
+    }
+}
+
+impl Display for MpsToken {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        match self {
+            Self::Sql => write!(f, "sql"),
+            Self::OpenBracket => write!(f, "("),
+            Self::CloseBracket => write!(f, ")"),
+            Self::Literal(s) => write!(f, "\"{}\"", s),
+        }
+    }
+}

mps-interpreter/src/tokens/tokenizer.rs

@@ -0,0 +1,249 @@
+use std::collections::VecDeque;
+
+use super::ParseError;
+use super::MpsToken;
+
+pub trait MpsTokenReader {
+    fn current_line(&self) -> usize;
+
+    fn current_column(&self) -> usize;
+
+    fn next_statements(&mut self, count: usize, token_buffer: &mut VecDeque<MpsToken>) -> Result<(), ParseError>;
+
+    fn end_of_file(&self) -> bool;
+}
+
+pub struct MpsTokenizer<R> where R: std::io::Read {
+    reader: R,
+    fsm: ReaderStateMachine,
+    line: usize,
+    column: usize,
+}
+
+impl<R> MpsTokenizer<R> where R: std::io::Read {
+    pub fn new(reader: R) -> Self {
+        Self {
+            reader: reader,
+            fsm: ReaderStateMachine::Start{},
+            line: 0,
+            column: 0,
+        }
+    }
+
+    pub fn read_line(&mut self, buf: &mut VecDeque<MpsToken>) -> Result<(), ParseError> {
+        let mut byte_buf = [0_u8];
+        // first read special case
+        // always read before checking if end of statement
+        // since FSM could be from previous (already ended) statement
+        if self.reader.read(&mut byte_buf).map_err(|e| self.error(format!("IO read error: {}", e)))? == 0 {
+            byte_buf[0] = 0; // clear to null char (nothing read is assumed to mean end of file)
+        }
+        self.do_tracking(byte_buf[0]);
+        self.fsm = self.fsm.next_state(byte_buf[0]);
+        let mut bigger_buf: Vec<u8> = Vec::new();
+        while !(self.fsm.is_end_statement() || self.fsm.is_end_of_file()) {
+            // keep token's bytes
+            if let Some(out) = self.fsm.output() {
+                bigger_buf.push(out);
+            }
+            // handle parse endings
+            match self.fsm {
+                ReaderStateMachine::EndLiteral{} => {
+                    let literal = String::from_utf8(bigger_buf.clone())
+                        .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?;
+                    buf.push_back(MpsToken::Literal(literal));
+                    bigger_buf.clear();
+                },
+                ReaderStateMachine::EndToken{} => {
+                    let token = String::from_utf8(bigger_buf.clone())
+                        .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?;
+                    buf.push_back(
+                        MpsToken::parse_from_string(token)
+                            .map_err(|e| self.error(format!("Invalid token {}", e)))?
+                    );
+                    bigger_buf.clear();
+                },
+                ReaderStateMachine::Bracket{..} => {
+                    let out = bigger_buf.pop().unwrap(); // bracket token
+                    if bigger_buf.len() != 0 { // bracket tokens can be beside other tokens, without separator
+                        let token = String::from_utf8(bigger_buf.clone())
+                            .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?;
+                        buf.push_back(
+                            MpsToken::parse_from_string(token)
+                                .map_err(|e| self.error(format!("Invalid token {}", e)))?
+                        );
+                        bigger_buf.clear();
+                    }
+                    // process bracket token
+                    bigger_buf.push(out);
+                    let token = String::from_utf8(bigger_buf.clone())
+                        .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?;
+                    buf.push_back(
+                        MpsToken::parse_from_string(token)
+                            .map_err(|e| self.error(format!("Invalid token {}", e)))?
+                    );
+                    bigger_buf.clear();
+                },
+                ReaderStateMachine::EndStatement{} => {
+                    // unnecessary; loop will have already exited
+                },
+                ReaderStateMachine::EndOfFile{} => {
+                    // unnecessary; loop will have already exited
+                },
+                _ => {},
+            }
+            if self.reader.read(&mut byte_buf).map_err(|e| self.error(format!("IO read error: {}", e)))? == 0 {
+                byte_buf[0] = 0; // clear to null char (nothing read is assumed to mean end of file)
+            }
+            self.do_tracking(byte_buf[0]);
+            self.fsm = self.fsm.next_state(byte_buf[0]);
+        }
+        // handle end statement
+        if bigger_buf.len() != 0 { // also end of token
+            // note: never also end of literal, since those have explicit closing characters
+            let token = String::from_utf8(bigger_buf.clone())
+                .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?;
+            buf.push_back(
+                MpsToken::parse_from_string(token)
+                    .map_err(|e| self.error(format!("Invalid token {}", e)))?
+            );
+            bigger_buf.clear();
+        }
+        Ok(())
+    }
+
+    /// track line and column locations
+    fn do_tracking(&mut self, input: u8) {
+        if input as char == '\n' {
+            self.line += 1;
+        }
+        self.column += 1; // TODO correctly track columns with utf-8 characters longer than one byte
+    }
+
+    /// error factory (for ergonomics/DRY)
+    fn error(&self, item: String) -> ParseError {
+        ParseError {
+            line: self.current_line(),
+            column: self.current_column(),
+            item: item,
+        }
+    }
+}
+
+impl<R> MpsTokenReader for MpsTokenizer<R>
+where
+    R: std::io::Read
+{
+    fn current_line(&self) -> usize {
+        self.line
+    }
+
+    fn current_column(&self) -> usize {
+        self.column
+    }
+
+    fn next_statements(&mut self, count: usize, buf: &mut VecDeque<MpsToken>) -> Result<(), ParseError> {
+        for _ in 0..count {
+            self.read_line(buf)?;
+        }
+        Ok(())
+    }
+
+    fn end_of_file(&self) -> bool {
+        self.fsm.is_end_of_file()
+    }
+}
+
+#[derive(Copy, Clone)]
+enum ReaderStateMachine {
+    Start{}, // beginning of machine, no parsing has occured
+    Regular{
+        out: u8,
+    }, // standard
+    Escaped{
+        inside: char, // literal
+    }, // escape character; applied to next character
+    StartTickLiteral{},
+    StartQuoteLiteral{},
+    InsideTickLiteral{
+        out: u8,
+    },
+    InsideQuoteLiteral{
+        out: u8,
+    },
+    Bracket {
+        out: u8,
+    },
+    EndLiteral{},
+    EndToken{},
+    EndStatement{},
+    EndOfFile{},
+}
+
+impl ReaderStateMachine {
+    pub fn next_state(self, input: u8) -> Self {
+        let input_char = input as char;
+        match self {
+            Self::Start{}
+            | Self::Regular{..}
+            | Self::Bracket{..}
+            | Self::EndLiteral{}
+            | Self::EndToken{}
+            | Self::EndStatement{} =>
+                match input_char {
+                    '\\' => Self::Escaped{inside: '_'},
+                    '`' => Self::StartTickLiteral{},
+                    '"' => Self::StartQuoteLiteral{},
+                    ' ' => Self::EndToken{},
+                    '\n' | '\r' | ';' => Self::EndStatement{},
+                    '\0' => Self::EndOfFile{},
+                    '(' | ')' => Self::Bracket{out: input},
+                    _ => Self::Regular{out: input},
+                },
+            Self::Escaped{inside} => match inside {
+                '`' => Self::InsideTickLiteral{out: input},
+                '"' => Self::InsideQuoteLiteral{out: input},
+                '_' | _ => Self::Regular{out: input}
+            },
+            Self::StartTickLiteral{}
+            | Self::InsideTickLiteral{..} =>
+                match input_char {
+                    '\\' => Self::Escaped{inside: '`'},
+                    '`' => Self::EndLiteral{},
+                    _ => Self::InsideTickLiteral{out: input},
+                },
+            Self::StartQuoteLiteral{}
+            | Self::InsideQuoteLiteral{..} =>
+                match input_char {
+                    '\\' => Self::Escaped{inside: '"'},
+                    '"' => Self::EndLiteral{},
+                    _ => Self::InsideQuoteLiteral{out: input},
+                },
+            Self::EndOfFile{} => Self::EndOfFile{},
+        }
+    }
+
+    pub fn is_end_statement(&self) -> bool {
+        match self {
+            Self::EndStatement{} => true,
+            _ => false
+        }
+    }
+
+    pub fn is_end_of_file(&self) -> bool {
+        match self {
+            Self::EndOfFile{} => true,
+            _ => false
+        }
+    }
+
+    pub fn output(&self) -> Option<u8> {
+        match self {
+            Self::Regular{ out, ..}
+            | Self::Bracket{ out, ..}
+            | Self::InsideTickLiteral{ out, ..}
+            | Self::InsideQuoteLiteral{ out, ..} => Some(*out),
+            _ => None
+        }
+    }
+}

mps-interpreter/tests/music_lib.rs

@@ -0,0 +1,11 @@
+#[cfg(feature = "music_library")]
+mod music_lib_test {
+    use mps_interpreter::music::*;
+
+    #[test]
+    fn generate_library() {
+        let mut lib = MpsLibrary::new();
+        lib.read_path("/home/ngnius/Music", 10).unwrap();
+        println!("generated library size: {}", lib.len());
+    }
+}

mps-interpreter/tests/single_line.rs

@@ -0,0 +1,57 @@
+use std::io::Cursor;
+use std::collections::VecDeque;
+use mps_interpreter::*;
+use mps_interpreter::lang::MpsLanguageError;
+use mps_interpreter::tokens::{ParseError, MpsToken, MpsTokenizer};
+
+#[test]
+fn parse_line() -> Result<(), ParseError> {
+    let cursor = Cursor::new("sql(`SELECT * FROM songs;`)");
+    let correct_tokens: Vec<MpsToken> = vec![
+        MpsToken::Sql,
+        MpsToken::OpenBracket,
+        MpsToken::Literal("SELECT * FROM songs;".into()),
+        MpsToken::CloseBracket,
+    ];
+
+    let mut tokenizer = MpsTokenizer::new(cursor);
+    let mut buf = VecDeque::<MpsToken>::new();
+    tokenizer.read_line(&mut buf)?; // operation being tested
+
+    // debug output
+    println!("Token buffer:");
+    for i in 0..buf.len() {
+        println!("  Token #{}: {}", i, &buf[i]);
+    }
+
+    // validity tests
+    assert_eq!(buf.len(), correct_tokens.len());
+    for i in 0..buf.len() {
+        assert_eq!(buf[i], correct_tokens[i]);
+    }
+
+    tokenizer.read_line(&mut buf)?; // this should immediately return
+    Ok(())
+}
+
+#[test]
+fn execute_line() -> Result<(), Box<dyn MpsLanguageError>> {
+    let cursor = Cursor::new("sql(`SELECT * FROM songs ORDER BY artist;`)");
+
+    let tokenizer = MpsTokenizer::new(cursor);
+    let interpreter = MpsInterpretor::with_standard_vocab(tokenizer);
+
+    let mut count = 0;
+    for result in interpreter {
+        if let Ok(item) = result {
+            count +=1;
+            if count > 100 {continue;} // no need to spam the rest of the songs
+            println!("Got song `{}` (file: `{}`)", item.title, item.filename);
+        } else {
+            println!("Got error while iterating (executing)");
+            result?;
+        }
+    }
+    assert_ne!(count, 0); // database is populated
+    Ok(())
+}

mps-player/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "mps-player"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+rodio = { version = "^0.14"}
+m3u8-rs = { version = "^3.0.0" }
+
+# local
+mps-interpreter = { path = "../mps-interpreter" }

mps-player/src/errors.rs

@@ -0,0 +1,20 @@
+use std::fmt::{Debug, Display, Formatter, Error};
+
+#[derive(Debug)]
+pub struct PlaybackError {
+    pub(crate) msg: String
+}
+
+impl PlaybackError {
+    pub fn from_err<E: Display>(err: E) -> Self {
+        Self {
+            msg: format!("{}", err),
+        }
+    }
+}
+
+impl Display for PlaybackError {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "PlaybackError: {}", &self.msg)
+    }
+}

mps-player/src/lib.rs

@@ -0,0 +1,8 @@
+mod errors;
+mod player;
+
+pub use errors::PlaybackError;
+pub use player::MpsPlayer;
+
+#[cfg(test)]
+mod tests {}

mps-player/src/player.rs

@@ -0,0 +1,135 @@
+use std::io;
+use std::fs;
+
+use rodio::{decoder::Decoder, OutputStream, Sink};
+
+use m3u8_rs::{MediaPlaylist, MediaSegment};
+
+use mps_interpreter::{MpsRunner, tokens::MpsTokenReader};
+
+use super::PlaybackError;
+
+pub struct MpsPlayer<T: MpsTokenReader> {
+    runner: MpsRunner<T>,
+    sink: Sink,
+    #[allow(dead_code)]
+    output_stream: OutputStream, // this is required for playback, so it must live as long as this struct instance
+}
+
+impl<T: MpsTokenReader> MpsPlayer<T> {
+    pub fn new(runner: MpsRunner<T>) -> Result<Self, PlaybackError> {
+        let (stream, output_handle) = OutputStream::try_default().map_err(PlaybackError::from_err)?;
+        Ok(Self{
+            runner: runner,
+            sink: Sink::try_new(&output_handle).map_err(PlaybackError::from_err)?,
+            output_stream: stream,
+        })
+    }
+
+    pub fn play_all(&mut self) -> Result<(), PlaybackError> {
+        for item in &mut self.runner {
+            self.sink.sleep_until_end();
+            match item {
+                Ok(music) => {
+                    let file = fs::File::open(music.filename).map_err(PlaybackError::from_err)?;
+                    let stream = io::BufReader::new(file);
+                    let source = Decoder::new(stream).map_err(PlaybackError::from_err)?;
+                    self.sink.append(source);
+                    //self.sink.play(); // idk if this is necessary
+                    Ok(())
+                },
+                Err(e) => Err(PlaybackError::from_err(e))
+            }?;
+        }
+        self.sink.sleep_until_end();
+        Ok(())
+    }
+
+    pub fn enqueue_all(&mut self) -> Result<(), PlaybackError> {
+        for item in &mut self.runner {
+            match item {
+                Ok(music) => {
+                    let file = fs::File::open(music.filename).map_err(PlaybackError::from_err)?;
+                    let stream = io::BufReader::new(file);
+                    let source = Decoder::new(stream).map_err(PlaybackError::from_err)?;
+                    self.sink.append(source);
+                    self.sink.play(); // idk if this is necessary
+                    Ok(())
+                },
+                Err(e) => Err(PlaybackError::from_err(e))
+            }?;
+        }
+        Ok(())
+    }
+
+    pub fn resume(&self) {
+        self.sink.play()
+    }
+
+    pub fn pause(&self) {
+        self.sink.pause()
+    }
+
+    pub fn stop(&self) {
+        self.sink.stop()
+    }
+
+    pub fn sleep_until_end(&self) {
+        self.sink.sleep_until_end()
+    }
+
+    pub fn queue_len(&self) -> usize {
+        self.sink.len()
+    }
+
+    pub fn save_m3u8<W: io::Write>(&mut self, w: &mut W) -> Result<(), PlaybackError> {
+        let mut playlist = MediaPlaylist {
+            version: 6,
+            ..Default::default()
+        };
+        // generate
+        for item in &mut self.runner {
+            match item {
+                Ok(music) => {
+                    playlist.segments.push(
+                        MediaSegment {
+                            uri: music.filename,
+                            title: Some(music.title),
+                            ..Default::default()
+                        }
+                    );
+                    Ok(())
+                },
+                Err(e) => Err(PlaybackError::from_err(e))
+            }?;
+        }
+        playlist.write_to(w).map_err(PlaybackError::from_err)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io;
+    use mps_interpreter::MpsRunner;
+    use super::*;
+
+    #[allow(dead_code)]
+    #[test]
+    fn play_cursor() -> Result<(), PlaybackError> {
+        let cursor = io::Cursor::new("sql(`SELECT * FROM songs JOIN artists ON songs.artist = artists.artist_id WHERE artists.name like 'thundercat'`);");
+        let runner = MpsRunner::with_stream(cursor);
+        let mut player = MpsPlayer::new(runner)?;
+        player.play_all()
+    }
+
+    #[test]
+    fn playlist() -> Result<(), PlaybackError> {
+        let cursor = io::Cursor::new("sql(`SELECT * FROM songs JOIN artists ON songs.artist = artists.artist_id WHERE artists.name like 'thundercat'`);");
+        let runner = MpsRunner::with_stream(cursor);
+        let mut player = MpsPlayer::new(runner)?;
+
+        let output_file = std::fs::File::create("playlist.m3u8").unwrap();
+        let mut buffer = std::io::BufWriter::new(output_file);
+        player.save_m3u8(&mut buffer)
+    }
+}

mps-player/src/utility.rs

@@ -0,0 +1,15 @@
+use std::path::Path;
+use std::io;
+use std::fs;
+
+use mps_interpreter::{MpsRunner, tokens::MpsTokenReader};
+
+use super::{MpsPlayer, PlaybackError};
+
+pub fn play_script<P: AsRef<Path>>(p: P) -> Result<(), PlaybackError> {
+    let file = fs::File::open(music.filename).map_err(PlaybackError::from_err)?;
+    let stream = io::BufReader::new(file);
+    let runner = MpsRunner::with_stream(stream);
+    let mut player = MpsPlayer::new(runner);
+    player.play_all()
+}

src/main.rs

@@ -0,0 +1,15 @@
+use std::io;
+use mps_interpreter::MpsRunner;
+use mps_player::{MpsPlayer, PlaybackError};
+
+#[allow(dead_code)]
+fn play_cursor() -> Result<(), PlaybackError> {
+    let cursor = io::Cursor::new("sql(`SELECT * FROM songs JOIN artists ON songs.artist = artists.artist_id WHERE artists.name like 'thundercat'`);");
+    let runner = MpsRunner::with_stream(cursor);
+    let mut player = MpsPlayer::new(runner)?;
+    player.play_all()
+}
+
+fn main() {
+    play_cursor().unwrap();
+}