Refactor vocab and add simple filters

mps-interpreter/Cargo.toml

@@ -8,8 +8,8 @@ rusqlite = { version = "0.26.3" }
 symphonia = { version = "0.4.0", optional = true, features = [
     "aac", "flac", "mp3", "pcm", "vorbis", "isomp4", "ogg", "wav"
 ] }
-dirs = { version = "4.0.0", optional = true}
+dirs = { version = "4.0.0" }
 
 [features]
 default = [ "music_library" ]
-music_library = [ "symphonia", "dirs" ] # song metadata parsing and database auto-population
+music_library = [ "symphonia" ] # song metadata parsing and database auto-population

mps-interpreter/src/context.rs

@@ -1,23 +1,18 @@
 use super::processing::database::{MpsDatabaseQuerier, MpsSQLiteExecutor};
+use super::processing::general::{MpsVariableStorer, MpsOpStorage};
 use std::fmt::{Debug, Display, Error, Formatter};
 
 #[derive(Debug)]
 pub struct MpsContext {
     pub database: Box<dyn MpsDatabaseQuerier>,
+    pub variables: Box<dyn MpsVariableStorer>,
 }
 
 impl Default for MpsContext {
     fn default() -> Self {
         Self {
             database: Box::new(MpsSQLiteExecutor::default()),
-        }
-    }
-}
-
-impl std::clone::Clone for MpsContext {
-    fn clone(&self) -> Self {
-        Self {
-            database: Box::new(MpsSQLiteExecutor::default()),
+            variables: Box::new(MpsOpStorage::default()),
         }
     }
 }

mps-interpreter/src/interpretor.rs

@@ -113,6 +113,8 @@ where
             let stmt = self.vocabulary.try_build_statement(&mut self.buffer);
             match stmt {
                 Ok(mut stmt) => {
+                    #[cfg(debug_assertions)]
+                    if self.buffer.len() != 0 {panic!("Token buffer was not emptied! (rem: {:?})", self.buffer)}
                     stmt.enter(self.context.take().unwrap_or_else(|| MpsContext::default()));
                     self.current_stmt = Some(stmt);
                     let next_item = self.current_stmt.as_mut().unwrap().next();
@@ -148,7 +150,14 @@ fn box_error_with_ctx<E: MpsLanguageError + 'static>(
 
 pub(crate) fn standard_vocab(vocabulary: &mut MpsLanguageDictionary) {
     vocabulary
+        // high-priority vocabulary (low-priority may accept this, but will not execute as expected)
+        .add(crate::lang::vocabulary::filters::empty_filter())
+        .add(crate::lang::vocabulary::filters::field_filter())
+        // low-priority (more forgiving statements which may not parse complete statement)
         .add(crate::lang::vocabulary::SqlStatementFactory)
         .add(crate::lang::vocabulary::SimpleSqlStatementFactory)
-        .add(crate::lang::vocabulary::CommentStatementFactory);
+        .add(crate::lang::vocabulary::CommentStatementFactory)
+        .add(crate::lang::vocabulary::RepeatStatementFactory)
+        .add(crate::lang::vocabulary::AssignStatementFactory)
+        .add(crate::lang::vocabulary::SqlInitStatementFactory);
 }

mps-interpreter/src/lang/db_items.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 pub const DEFAULT_SQLITE_FILEPATH: &str = "metadata.mps.sqlite";
 
 pub trait DatabaseObj: Sized {
@@ -9,11 +11,25 @@ pub trait DatabaseObj: Sized {
 }
 
 pub fn generate_default_db() -> rusqlite::Result<rusqlite::Connection> {
-    let db_exists = std::path::Path::new(DEFAULT_SQLITE_FILEPATH).exists();
-    let conn = rusqlite::Connection::open(DEFAULT_SQLITE_FILEPATH)?;
+    generate_db(super::utility::music_folder(), DEFAULT_SQLITE_FILEPATH, true)
+}
+
+pub fn generate_db<P1: AsRef<Path>, P2: AsRef<Path>>(
+    music_path: P1,
+    sqlite_path: P2,
+    generate: bool
+) -> rusqlite::Result<rusqlite::Connection> {
+    let music_path = music_path.as_ref();
+    let sqlite_path = sqlite_path.as_ref();
+    let db_exists = std::path::Path::new(sqlite_path).exists();
+    #[cfg(not(feature = "music_library"))]
+    let conn = rusqlite::Connection::open(sqlite_path)?;
+    #[cfg(feature = "music_library")]
+    let mut conn = rusqlite::Connection::open(sqlite_path)?;
     // skip db building if SQLite file already exists
-    // TODO do a more exhaustive db check to make sure it's actually the correct file
-    if db_exists {
+    // TODO do a more exhaustive db check to make sure it's actually the correct file and database structure
+    #[cfg(not(feature = "music_library"))]
+    if db_exists && !generate {
         return Ok(conn);
     }
     // build db tables
@@ -56,11 +72,18 @@ pub fn generate_default_db() -> rusqlite::Result<rusqlite::Connection> {
     )?;
     // generate data and store in db
     #[cfg(feature = "music_library")]
+    if generate {
+
+        let mut lib = crate::music::MpsLibrary::new();
+        if db_exists {
+            crate::music::build_library_from_sqlite(&conn, &mut lib)?;
+        }
+        lib.clear_modified();
+        match crate::music::build_library_from_files(&music_path, &mut lib) {
+            Ok(_) => if lib.is_modified() {
+                let transaction = conn.transaction()?;
                 {
-        let music_path = super::utility::music_folder();
-        match crate::music::build_library(&music_path) {
-            Ok(lib) => {
-                let mut song_insert = conn.prepare(
+                let mut song_insert = transaction.prepare(
                     "INSERT OR REPLACE INTO songs (
                         song_id,
                         title,
@@ -75,7 +98,7 @@ pub fn generate_default_db() -> rusqlite::Result<rusqlite::Connection> {
                     song_insert.execute(song.to_params().as_slice())?;
                 }
 
-                let mut metadata_insert = conn.prepare(
+                let mut metadata_insert = transaction.prepare(
                     "INSERT OR REPLACE INTO metadata (
                         meta_id,
                         plays,
@@ -89,7 +112,7 @@ pub fn generate_default_db() -> rusqlite::Result<rusqlite::Connection> {
                     metadata_insert.execute(meta.to_params().as_slice())?;
                 }
 
-                let mut artist_insert = conn.prepare(
+                let mut artist_insert = transaction.prepare(
                     "INSERT OR REPLACE INTO artists (
                         artist_id,
                         name,
@@ -100,7 +123,7 @@ pub fn generate_default_db() -> rusqlite::Result<rusqlite::Connection> {
                     artist_insert.execute(artist.to_params().as_slice())?;
                 }
 
-                let mut album_insert = conn.prepare(
+                let mut album_insert = transaction.prepare(
                     "INSERT OR REPLACE INTO albums (
                         album_id,
                         title,
@@ -113,7 +136,7 @@ pub fn generate_default_db() -> rusqlite::Result<rusqlite::Connection> {
                     album_insert.execute(album.to_params().as_slice())?;
                 }
 
-                let mut genre_insert = conn.prepare(
+                let mut genre_insert = transaction.prepare(
                     "INSERT OR REPLACE INTO genres (
                         genre_id,
                         title
@@ -123,6 +146,8 @@ pub fn generate_default_db() -> rusqlite::Result<rusqlite::Connection> {
                     genre_insert.execute(genre.to_params().as_slice())?;
                 }
                 }
+                transaction.commit()?;
+            },
             Err(e) => println!("Unable to load music from {}: {}", music_path.display(), e),
         }
     }

mps-interpreter/src/lang/dictionary.rs

@@ -19,6 +19,7 @@ impl MpsLanguageDictionary {
         &self,
         tokens: &mut VecDeque<MpsToken>,
     ) -> Result<Box<dyn MpsOp>, SyntaxError> {
+        //println!("try_build_statement with tokens {:?}", tokens);
         for factory in &self.vocabulary {
             if factory.is_op_boxed(tokens) {
                 return factory.build_op_boxed(tokens, self);

mps-interpreter/src/lang/error.rs

@@ -1,6 +1,6 @@
 use std::fmt::{Debug, Display, Error, Formatter};
 
-use super::MpsOp;
+use super::PseudoOp;
 use crate::tokens::MpsToken;
 
 #[derive(Debug)]
@@ -28,7 +28,7 @@ impl MpsLanguageError for SyntaxError {
 #[derive(Debug)]
 pub struct RuntimeError {
     pub line: usize,
-    pub op: Box<dyn MpsOp>,
+    pub op: PseudoOp,
     pub msg: String,
 }
 

mps-interpreter/src/lang/filter.rs

@@ -0,0 +1,281 @@
+use std::collections::VecDeque;
+use std::fmt::{Debug, Display, Error, Formatter};
+use std::iter::Iterator;
+use std::marker::PhantomData;
+
+use crate::MpsContext;
+use crate::MpsMusicItem;
+use crate::tokens::MpsToken;
+use crate::lang::{MpsOp, PseudoOp, BoxedMpsOpFactory};
+use crate::lang::{RuntimeError, SyntaxError};
+use crate::lang::MpsLanguageDictionary;
+use crate::processing::general::MpsType;
+use crate::lang::utility::{assert_token_raw, assert_token};
+use crate::processing::OpGetter;
+
+pub trait MpsFilterPredicate: Clone + Debug + Display {
+    fn matches(&mut self, item: &MpsMusicItem, ctx: &mut MpsContext, op: &mut OpGetter) -> Result<bool, RuntimeError>;
+}
+
+pub trait MpsFilterFactory<P: MpsFilterPredicate + 'static> {
+    fn is_filter(&self, tokens: &VecDeque<&MpsToken>) -> bool;
+
+    fn build_filter(
+        &self,
+        tokens: &mut VecDeque<MpsToken>,
+        dict: &MpsLanguageDictionary,
+    ) -> Result<P, SyntaxError>;
+}
+
+#[derive(Debug, Clone)]
+enum VariableOrOp {
+    Variable(String),
+    Op(PseudoOp),
+}
+
+impl Display for VariableOrOp {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        match self {
+            Self::Variable(s) => write!(f, "{}", s),
+            Self::Op(op) => write!(f, "{}", op)
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct MpsFilterStatement<P: MpsFilterPredicate + 'static> {
+    predicate: P,
+    iterable: VariableOrOp,
+    context: Option<MpsContext>,
+}
+
+impl<P: MpsFilterPredicate + 'static> std::clone::Clone for MpsFilterStatement<P> {
+    fn clone(&self) -> Self {
+        Self {
+            predicate: self.predicate.clone(),
+            iterable: self.iterable.clone(),
+            context: None,
+        }
+    }
+}
+
+impl<P: MpsFilterPredicate + 'static> Display for MpsFilterStatement<P> {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "{}.({})", self.iterable, self.predicate)
+    }
+}
+
+impl<P: MpsFilterPredicate + 'static> MpsOp for MpsFilterStatement<P> {
+    fn enter(&mut self, ctx: MpsContext) {
+        self.context = Some(ctx)
+    }
+
+    fn escape(&mut self) -> MpsContext {
+        self.context.take().unwrap()
+    }
+}
+
+impl<P: MpsFilterPredicate + 'static> Iterator for MpsFilterStatement<P> {
+    type Item = Result<MpsMusicItem, RuntimeError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let self_clone = self.clone();
+        let self_clone2 = self_clone.clone();
+        let mut op_getter = move || (Box::new(self_clone.clone()) as Box<dyn MpsOp>).into();
+        //let ctx = self.context.as_mut().unwrap();
+        match &mut self.iterable {
+            VariableOrOp::Op(op) =>
+                match op.try_real() {
+                    Ok(real_op) => {
+                        let ctx = self.context.take().unwrap();
+                        real_op.enter(ctx);
+                        let mut maybe_result = None;
+                        while let Some(item) = real_op.next() {
+                            let mut ctx = real_op.escape();
+                            match item {
+                                Err(e) => {
+                                    //self.context = Some(op.escape());
+                                    maybe_result = Some(Err(e));
+                                    self.context = Some(ctx);
+                                    break;
+                                },
+                                Ok(item) => {
+                                    let matches_result = self.predicate.matches(&item, &mut ctx, &mut op_getter);
+                                    let matches = match matches_result {
+                                        Err(e) => {
+                                            maybe_result = Some(Err(e));
+                                            self.context = Some(ctx);
+                                            break;
+                                        },
+                                        Ok(b) => b,
+                                    };
+                                    if matches {
+                                        //self.context = Some(op.escape());
+                                        maybe_result = Some(Ok(item));
+                                        self.context = Some(ctx);
+                                        break;
+                                    }
+                                }
+                            }
+                            real_op.enter(ctx);
+                        }
+                        if self.context.is_none() {
+                            self.context = Some(real_op.escape());
+                        }
+                        maybe_result
+                    },
+                    Err(e) => return Some(Err(e)),
+                },
+            VariableOrOp::Variable(variable_name) => {
+                let mut variable = match self.context.as_mut().unwrap().variables.remove(
+                    &variable_name,
+                    &mut op_getter
+                ) {
+                    Ok(MpsType::Op(op)) => op,
+                    Ok(x) => return Some(Err(RuntimeError {
+                        line: 0,
+                        op: (Box::new(self_clone2.clone()) as Box<dyn MpsOp>).into(),
+                        msg: format!("Expected operation/iterable type in variable {}, got {}", &variable_name, x)
+                    })),
+                    Err(e) => return Some(Err(e))
+                };
+                let mut maybe_result = None;
+                let ctx = self.context.take().unwrap();
+                variable.enter(ctx);
+                while let Some(item) = variable.next() {
+                    let mut ctx = variable.escape();
+                    match item {
+                        Err(e) => {
+                           maybe_result = Some(Err(e));
+                           self.context = Some(ctx);
+                           break;
+                        },
+                        Ok(item) => {
+                            let matches_result = self.predicate.matches(&item, &mut ctx, &mut op_getter);
+                            let matches = match matches_result {
+                                Err(e) => {
+                                    maybe_result = Some(Err(e));
+                                    self.context = Some(ctx);
+                                    break;
+                                },
+                                Ok(b) => b,
+                            };
+                            if matches {
+                                maybe_result = Some(Ok(item));
+                                self.context = Some(ctx);
+                                break;
+                            }
+                        }
+                    }
+                    variable.enter(ctx);
+                }
+                if self.context.is_none() {
+                    self.context = Some(variable.escape());
+                }
+                match self.context.as_mut().unwrap().variables.declare(
+                    &variable_name,
+                    MpsType::Op(variable),
+                    &mut op_getter) {
+                    Err(e) => return Some(Err(e)),
+                    Ok(_) => maybe_result
+                }
+            }
+        }
+    }
+}
+
+pub struct MpsFilterStatementFactory<P: MpsFilterPredicate + 'static, F: MpsFilterFactory<P> + 'static> {
+    filter_factory: F,
+    idc: PhantomData<P>,
+}
+
+impl<P: MpsFilterPredicate + 'static, F: MpsFilterFactory<P> + 'static> MpsFilterStatementFactory<P, F> {
+    pub fn new(factory: F) -> Self {
+        Self {
+            filter_factory: factory,
+            idc: PhantomData::default(),
+        }
+    }
+}
+
+impl<P: MpsFilterPredicate + 'static, F: MpsFilterFactory<P> + 'static> BoxedMpsOpFactory for MpsFilterStatementFactory<P, F> {
+    fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool {
+        let tokens_len = tokens.len();
+        if last_open_bracket_is_after_dot(tokens) {
+            let start_of_predicate = last_dot_before_open_bracket(tokens) + 2; // .(predicate)
+            if start_of_predicate > tokens_len - 1 {
+                false
+            } else {
+                let tokens2: VecDeque<&MpsToken> = VecDeque::from_iter(tokens.range(start_of_predicate..tokens_len-1));
+                self.filter_factory.is_filter(&tokens2)
+            }
+        } else {
+            false
+        }
+    }
+
+    fn build_op_boxed(
+        &self,
+        tokens: &mut VecDeque<MpsToken>,
+        dict: &MpsLanguageDictionary,
+    ) -> Result<Box<dyn MpsOp>, SyntaxError> {
+        let start_of_op = last_dot_before_open_bracket(tokens);
+        let op;
+        if start_of_op == 1 && tokens[0].is_name() {
+            // variable_name.(predicate)
+            let variable_name = assert_token(|t| match t {
+                MpsToken::Name(s) => Some(s),
+                _ => None
+            }, MpsToken::Name("variable_name".into()), tokens)?;
+            op = VariableOrOp::Variable(variable_name);
+        } else {
+            // <some other op>.(predicate)
+            //let mut new_tokens = tokens.range(0..start_of_op).map(|x| x.to_owned()).collect();
+            let end_tokens = tokens.split_off(start_of_op);
+            let inner_op = dict.try_build_statement(tokens)?;
+            tokens.extend(end_tokens);
+            op = VariableOrOp::Op(inner_op.into());
+        }
+        assert_token_raw(MpsToken::Dot, tokens)?;
+        assert_token_raw(MpsToken::OpenBracket, tokens)?;
+        let filter = self.filter_factory.build_filter(tokens, dict)?;
+        assert_token_raw(MpsToken::CloseBracket, tokens)?;
+        Ok(Box::new(MpsFilterStatement {
+            predicate: filter,
+            iterable: op,
+            context: None,
+        }))
+    }
+}
+
+fn last_open_bracket_is_after_dot(tokens: &VecDeque<MpsToken>) -> bool {
+    let mut open_bracket_found = false;
+    for i in (0..tokens.len()).rev() {
+        if tokens[i].is_open_bracket() {
+            open_bracket_found = true;
+        } else if open_bracket_found {
+            if tokens[i].is_dot() {
+                return true
+            } else {
+                return false
+            }
+        }
+    }
+    false
+}
+
+fn last_dot_before_open_bracket(tokens: &VecDeque<MpsToken>) -> usize {
+    let mut open_bracket_found = false;
+    for i in (0..tokens.len()).rev() {
+        if tokens[i].is_open_bracket() {
+            open_bracket_found = true;
+        } else if open_bracket_found {
+            if tokens[i].is_dot() {
+                return i
+            } else {
+                return 0
+            }
+        }
+    }
+    0
+}

mps-interpreter/src/lang/mod.rs

@@ -1,27 +1,28 @@
-mod comment;
 mod db_items;
 mod dictionary;
 mod error;
+mod filter;
 mod operation;
-mod sql_query;
-mod sql_simple_query;
+mod pseudo_op;
+mod repeated_meme;
 //mod statement;
+mod type_primitives;
 pub(crate) mod utility;
 
 pub use dictionary::MpsLanguageDictionary;
 pub use error::{MpsLanguageError, RuntimeError, SyntaxError};
+pub use filter::{MpsFilterFactory, MpsFilterPredicate, MpsFilterStatement, MpsFilterStatementFactory};
 pub use operation::{BoxedMpsOpFactory, MpsOp, MpsOpFactory, SimpleMpsOpFactory};
+pub use pseudo_op::PseudoOp;
+pub use repeated_meme::{RepeatedTokens, repeated_tokens};
 //pub(crate) use statement::MpsStatement;
+pub use type_primitives::MpsTypePrimitive;
 
-pub mod vocabulary {
-    pub use super::sql_query::{SqlStatement, SqlStatementFactory};
-    pub use super::sql_simple_query::{SimpleSqlStatement, SimpleSqlStatementFactory};
-    pub use super::comment::{CommentStatement, CommentStatementFactory};
-}
+pub mod vocabulary;
 
 pub mod db {
     pub use super::db_items::{
-        generate_default_db, DatabaseObj, DbAlbumItem, DbArtistItem, DbGenreItem, DbMetaItem,
+        generate_default_db, generate_db, DatabaseObj, DbAlbumItem, DbArtistItem, DbGenreItem, DbMetaItem,
         DbMusicItem, DEFAULT_SQLITE_FILEPATH,
     };
 }

mps-interpreter/src/lang/pseudo_op.rs

@@ -0,0 +1,78 @@
+use std::fmt::{Debug, Display, Error, Formatter};
+
+use super::MpsOp;
+use super::RuntimeError;
+
+/// Mps operation where clones of it emulate the Display behaviour without cloning the data
+#[derive(Debug)]
+pub enum PseudoOp {
+    Real(Box<dyn MpsOp>),
+    Fake(String)
+}
+
+impl PseudoOp {
+    pub fn try_real(&mut self) -> Result<&mut Box<dyn MpsOp>, RuntimeError> {
+        match self {
+            Self::Real(op) => Ok(op),
+            Self::Fake(_) => Err(RuntimeError {
+                line: 0,
+                op: self.clone(),
+                msg: "PseudoOp::Fake is not a real MpsOp".into(),
+            })
+        }
+    }
+
+    pub fn unwrap_real(self) -> Result<Box<dyn MpsOp>, RuntimeError> {
+        match self {
+            Self::Real(op) => {
+                let result = Ok(op);
+                result
+            },
+            Self::Fake(_) => Err(RuntimeError {
+                line: 0,
+                op: self.clone(),
+                msg: "PseudoOp::Fake is not a real MpsOp".into(),
+            })
+        }
+    }
+
+    #[inline]
+    pub fn is_real(&self) -> bool {
+        match self {
+            Self::Real(_) => true,
+            _ => false,
+        }
+    }
+
+    #[inline]
+    pub fn is_fake(&self) -> bool {
+        match self {
+            Self::Fake(_) => true,
+            _ => false,
+        }
+    }
+}
+
+impl Clone for PseudoOp {
+    fn clone(&self) -> Self {
+        match self {
+            Self::Real(op) => Self::Fake(format!("{}", op)),
+            Self::Fake(s) => Self::Fake(s.clone())
+        }
+    }
+}
+
+impl Display for PseudoOp {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        match self {
+            Self::Real(op) => write!(f, "{}", op),
+            Self::Fake(s) => write!(f, "{}", s)
+        }
+    }
+}
+
+impl std::convert::From<Box<dyn MpsOp>> for PseudoOp {
+    fn from(item: Box<dyn MpsOp>) -> Self {
+        Self::Real(item)
+    }
+}

mps-interpreter/src/lang/repeated_meme.rs

@@ -0,0 +1,55 @@
+use std::collections::VecDeque;
+
+use crate::lang::SyntaxError;
+use crate::lang::utility::{check_token_raw, assert_token_raw};
+use crate::tokens::MpsToken;
+
+/// Convenient parser for repeated patterns of tokens
+pub struct RepeatedTokens<
+    X: 'static,
+    F1: FnMut(&mut VecDeque<MpsToken>) -> Result<Option<X>, SyntaxError>,
+    F2: FnMut(&mut VecDeque<MpsToken>) -> Result<bool, SyntaxError> >
+{
+    pattern_ingest: F1,
+    separator_ingest: F2,
+}
+
+impl<
+    X: 'static,
+    F1: FnMut(&mut VecDeque<MpsToken>) -> Result<Option<X>, SyntaxError>,
+    F2: FnMut(&mut VecDeque<MpsToken>) -> Result<bool, SyntaxError> >
+    RepeatedTokens<X, F1, F2>
+{
+    pub fn ingest_all(&mut self, tokens: &mut VecDeque<MpsToken>) -> Result<Vec<X>, SyntaxError> {
+        let mut result = Vec::<X>::new();
+        match (self.pattern_ingest)(tokens)? {
+            Some(x) => result.push(x),
+            None => return Ok(result),
+        }
+        while (self.separator_ingest)(tokens)? {
+            match (self.pattern_ingest)(tokens)? {
+                Some(x) => result.push(x),
+                None => break
+            }
+        }
+        Ok(result)
+    }
+}
+
+pub fn repeated_tokens<X, F1: FnMut(&mut VecDeque<MpsToken>) -> Result<Option<X>, SyntaxError>>(
+    ingestor: F1,
+    separator: MpsToken,
+) -> RepeatedTokens<X, F1, impl FnMut(&mut VecDeque<MpsToken>) -> Result<bool, SyntaxError>> {
+    RepeatedTokens {
+        pattern_ingest: ingestor,
+        separator_ingest: move |tokens| {
+            if tokens.len() > 0 && check_token_raw(separator.clone(), &tokens[0]) {
+                assert_token_raw(separator.clone(), tokens)?;
+                Ok(true)
+            } else {
+                Ok(false)
+            }
+        },
+        //parsed: Vec::new(),
+    }
+}

mps-interpreter/src/lang/type_primitives.rs

@@ -0,0 +1,98 @@
+//! Basic types for MPS
+
+use std::fmt::{Debug, Display, Error, Formatter};
+
+#[derive(Debug, Clone)]
+pub enum MpsTypePrimitive {
+    String(String),
+    Int(i64),
+    UInt(u64),
+    Float(f64),
+    Bool(bool),
+}
+
+impl MpsTypePrimitive {
+    #[inline]
+    pub fn compare(&self, other: &Self) -> Result<i8, String> {
+        let result = match self {
+            Self::String(s1) => {
+                match other {
+                    Self::String(s2) => Some(map_ordering(s1.cmp(s2))),
+                    _ => None,
+                }
+            },
+            Self::Int(i1) => {
+                match other {
+                    Self::Int(i2) => Some(map_ordering(i1.cmp(i2))),
+                    Self::UInt(i2) => Some(map_ordering((*i1 as i128).cmp(&(*i2 as i128)))),
+                    Self::Float(i2) => Some(
+                        map_ordering((*i1 as f64).partial_cmp(&(*i2 as f64)).unwrap_or(std::cmp::Ordering::Less))
+                    ),
+                    _ => None,
+                }
+            },
+            Self::UInt(u1) => {
+                match other {
+                    Self::UInt(u2) => Some(map_ordering(u1.cmp(u2))),
+                    Self::Int(u2) => Some(map_ordering((*u1 as i128).cmp(&(*u2 as i128)))),
+                    Self::Float(u2) => Some(
+                        map_ordering((*u1 as f64).partial_cmp(&(*u2 as f64)).unwrap_or(std::cmp::Ordering::Less))
+                    ),
+                    _ => None,
+                }
+            },
+            Self::Float(f1) => {
+                match other {
+                    Self::Float(f2) => Some(map_ordering(f1.partial_cmp(f2).unwrap_or(std::cmp::Ordering::Less))),
+                    Self::Int(f2) => Some(
+                        map_ordering(f1.partial_cmp(&(*f2 as f64)).unwrap_or(std::cmp::Ordering::Less))
+                    ),
+                    Self::UInt(f2) => Some(
+                        map_ordering(f1.partial_cmp(&(*f2 as f64)).unwrap_or(std::cmp::Ordering::Less))
+                    ),
+                    _ => None
+                }
+            },
+            Self::Bool(b1) => {
+                match other {
+                    Self::Bool(b2) => {
+                        if *b2 == *b1 {
+                            Some(0)
+                        } else if *b1{
+                            Some(1)
+                        } else {
+                            Some(-1)
+                        }
+                    },
+                    _ => None
+                }
+            }
+        };
+        match result {
+            Some(x) => Ok(x),
+            None => Err(format!("Cannot compare {} to {}: incompatible types", self, other))
+        }
+    }
+}
+
+impl Display for MpsTypePrimitive {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        match self {
+            Self::String(s) => write!(f, "(String) {}", s),
+            Self::Int(i) => write!(f, "(Int) {}", *i),
+            Self::UInt(u) => write!(f, "(UInt) {}", *u),
+            Self::Float(f_) => write!(f, "(Float) {}", *f_),
+            Self::Bool(b) => write!(f, "(Bool) {}", *b),
+        }
+    }
+}
+
+#[inline]
+fn map_ordering(ordering: std::cmp::Ordering) -> i8 {
+    match ordering {
+        std::cmp::Ordering::Less => -1,
+        std::cmp::Ordering::Equal => 0,
+        std::cmp::Ordering::Greater => 1,
+    }
+}
+

mps-interpreter/src/lang/utility.rs

@@ -2,18 +2,19 @@ use std::collections::VecDeque;
 #[cfg(feature = "music_library")]
 use std::path::PathBuf;
 
-use crate::tokens::MpsToken;
 use super::SyntaxError;
+use crate::tokens::MpsToken;
+use super::MpsTypePrimitive;
 
 pub fn assert_token<T, F: FnOnce(MpsToken) -> Option<T>>(
     caster: F,
     token: MpsToken,
-    tokens: &mut VecDeque<MpsToken>
+    tokens: &mut VecDeque<MpsToken>,
 ) -> Result<T, SyntaxError> {
     if let Some(out) = caster(tokens.pop_front().unwrap()) {
         Ok(out)
     } else {
-        Err(SyntaxError{
+        Err(SyntaxError {
             line: 0,
             token: token,
         })
@@ -22,7 +23,7 @@ pub fn assert_token<T, F: FnOnce(MpsToken) -> Option<T>>(
 
 pub fn assert_token_raw(
     token: MpsToken,
-    tokens: &mut VecDeque<MpsToken>
+    tokens: &mut VecDeque<MpsToken>,
 ) -> Result<MpsToken, SyntaxError> {
     let result = tokens.pop_front().unwrap();
     if std::mem::discriminant(&token) == std::mem::discriminant(&result) {
@@ -35,7 +36,85 @@ pub fn assert_token_raw(
     }
 }
 
-#[cfg(feature = "music_library")]
-pub fn music_folder() -> PathBuf {
-    dirs::home_dir().unwrap_or_else(|| PathBuf::from("./")).join("Music")
+pub fn check_token_raw(
+    token: MpsToken,
+    token_target: &MpsToken,
+) -> bool {
+    std::mem::discriminant(&token) == std::mem::discriminant(token_target)
 }
+
+pub fn assert_name(name: &str, tokens: &mut VecDeque<MpsToken>) -> Result<String, SyntaxError> {
+    match tokens.pop_front().unwrap() {
+        MpsToken::Name(n) => {
+            if n == name {
+                Ok(n)
+            } else {
+                Err(
+                    SyntaxError {
+                        line: 0,
+                        token: MpsToken::Name(name.to_owned()),
+                    }
+                )
+            }
+        },
+        _token => Err(SyntaxError {
+            line: 0,
+            token: MpsToken::Name(name.to_owned()),
+        })
+    }
+}
+
+pub fn check_name(name: &str, token: &MpsToken) -> bool {
+    match token {
+        MpsToken::Name(n) => n == name,
+        _ => false
+    }
+}
+
+pub fn check_is_type(token: &MpsToken) -> bool {
+    match token {
+        MpsToken::Literal(_) => true,
+        MpsToken::Name(s) =>
+            s.parse::<f64>().is_ok()
+            || s.parse::<i64>().is_ok()
+            || s.parse::<u64>().is_ok()
+            || s == "false"
+            || s == "true",
+        _ => false
+    }
+}
+
+pub fn assert_type(tokens: &mut VecDeque<MpsToken>) -> Result<MpsTypePrimitive, SyntaxError> {
+    match tokens.pop_front().unwrap() {
+        MpsToken::Literal(s) => Ok(MpsTypePrimitive::String(s)),
+        MpsToken::Name(s) => {
+            if let Ok(f) = s.parse::<f64>() {
+                Ok(MpsTypePrimitive::Float(f))
+            } else if let Ok(i) = s.parse::<i64>() {
+                Ok(MpsTypePrimitive::Int(i))
+            } else if let Ok(u) = s.parse::<u64>() {
+                Ok(MpsTypePrimitive::UInt(u))
+            } else if s == "false" {
+                Ok(MpsTypePrimitive::Bool(false))
+            } else if s == "true" {
+                Ok(MpsTypePrimitive::Bool(true))
+            } else {
+                Err(SyntaxError {
+                    line: 0,
+                    token: MpsToken::Name("Float | UInt | Int | Bool".into()),
+                })
+            }
+        },
+        _token => Err(SyntaxError {
+            line: 0,
+            token: MpsToken::Name("Float | UInt | Int | Bool | \"String\"".into()),
+        })
+    }
+}
+
+pub fn music_folder() -> PathBuf {
+    dirs::home_dir()
+        .unwrap_or_else(|| PathBuf::from("./"))
+        .join("Music")
+}
+

mps-interpreter/src/lang/vocabulary/comment.rs

@@ -6,15 +6,15 @@ use crate::MpsContext;
 use crate::MpsMusicItem;
 use crate::tokens::MpsToken;
 
-use super::{RuntimeError, SyntaxError};
-use super::{MpsOp, SimpleMpsOpFactory, MpsOpFactory, BoxedMpsOpFactory};
-use super::MpsLanguageDictionary;
-use super::utility::assert_token;
+use crate::lang::{RuntimeError, SyntaxError};
+use crate::lang::{MpsOp, SimpleMpsOpFactory, MpsOpFactory, BoxedMpsOpFactory};
+use crate::lang::MpsLanguageDictionary;
+use crate::lang::utility::assert_token;
 
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct CommentStatement {
     comment: String,
-    context: Option<MpsContext>
+    context: Option<MpsContext>,
 }
 
 impl CommentStatement {
@@ -35,6 +35,15 @@ impl Display for CommentStatement {
     }
 }
 
+impl std::clone::Clone for CommentStatement {
+    fn clone(&self) -> Self {
+        Self {
+            comment: self.comment.clone(),
+            context: None,
+        }
+    }
+}
+
 impl Iterator for CommentStatement {
     type Item = Result<MpsMusicItem, RuntimeError>;
 

mps-interpreter/src/lang/vocabulary/filters/empty_filter.rs

@@ -0,0 +1,48 @@
+use std::collections::VecDeque;
+use std::fmt::{Debug, Display, Error, Formatter};
+
+use crate::MpsContext;
+use crate::MpsMusicItem;
+use crate::tokens::MpsToken;
+use crate::lang::{MpsFilterPredicate, MpsFilterFactory, MpsFilterStatementFactory};
+use crate::lang::{SyntaxError, RuntimeError};
+use crate::lang::MpsLanguageDictionary;
+use crate::processing::OpGetter;
+
+#[derive(Debug, Clone)]
+pub struct EmptyFilter;
+
+impl Display for EmptyFilter {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "[empty]")
+    }
+}
+
+impl MpsFilterPredicate for EmptyFilter {
+    fn matches(&mut self, _item: &MpsMusicItem, _ctx: &mut MpsContext, _op: &mut OpGetter) -> Result<bool, RuntimeError> {
+        Ok(true)
+    }
+}
+
+pub struct EmptyFilterFactory;
+
+impl MpsFilterFactory<EmptyFilter> for EmptyFilterFactory {
+    fn is_filter(&self, tokens: &VecDeque<&MpsToken>) -> bool {
+        tokens.len() == 0
+    }
+
+    fn build_filter(
+        &self,
+        _tokens: &mut VecDeque<MpsToken>,
+        _dict: &MpsLanguageDictionary,
+    ) -> Result<EmptyFilter, SyntaxError> {
+        Ok(EmptyFilter)
+    }
+}
+
+pub type EmptyFilterStatementFactory = MpsFilterStatementFactory<EmptyFilter, EmptyFilterFactory>;
+
+#[inline(always)]
+pub fn empty_filter() -> EmptyFilterStatementFactory {
+    EmptyFilterStatementFactory::new(EmptyFilterFactory)
+}

mps-interpreter/src/lang/vocabulary/filters/field_filter.rs

@@ -0,0 +1,136 @@
+use std::collections::VecDeque;
+use std::fmt::{Debug, Display, Error, Formatter};
+
+use crate::MpsContext;
+use crate::MpsMusicItem;
+use crate::tokens::MpsToken;
+use crate::lang::{MpsFilterPredicate, MpsFilterFactory, MpsFilterStatementFactory};
+use crate::lang::{SyntaxError, RuntimeError};
+use crate::lang::MpsLanguageDictionary;
+use crate::lang::MpsTypePrimitive;
+use crate::lang::utility::{assert_token, assert_type, check_is_type};
+use super::utility::{item_to_primitive_lut, assert_comparison_operator};
+use crate::processing::OpGetter;
+use crate::processing::general::MpsType;
+
+#[derive(Debug, Clone)]
+enum VariableOrValue {
+    Variable(String),
+    Value(MpsTypePrimitive),
+}
+
+#[derive(Debug, Clone)]
+pub struct FieldFilter {
+    field_name: String,
+    val: VariableOrValue,
+    comparison: [i8; 2]
+}
+
+impl Display for FieldFilter {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        // TODO display other comparison operators correctly
+        match &self.val {
+            VariableOrValue::Variable(name) => write!(f, "{} == {}", self.field_name, name),
+            VariableOrValue::Value(t) => write!(f, "{} == {}", self.field_name, t),
+        }
+
+    }
+}
+
+impl MpsFilterPredicate for FieldFilter {
+    fn matches(&mut self, item: &MpsMusicItem, ctx: &mut MpsContext, op: &mut OpGetter) -> Result<bool, RuntimeError> {
+        let music_item_lut = item_to_primitive_lut(item.to_owned());
+        let variable = match &self.val {
+            VariableOrValue::Variable(name) => match ctx.variables.get(&name, op)? {
+                MpsType::Primitive(t) => Ok(t),
+                _ => Err(RuntimeError {
+                    line: 0,
+                    op: op(),
+                    msg: format!("Variable {} is not comparable", name),
+                })
+            },
+            VariableOrValue::Value(val) => Ok(val)
+        }?;
+        if let Some(field) = music_item_lut.get(&self.field_name) {
+            let compare = field.compare(variable)
+                .map_err(|e| RuntimeError {
+                    line: 0,
+                    op: op(),
+                    msg: e,
+                })?;
+            let mut is_match = false;
+            for comparator in self.comparison {
+                if comparator == compare {
+                    is_match = true;
+                    break;
+                }
+            }
+            Ok(is_match)
+        } else {
+            Err(RuntimeError {
+                line: 0,
+                op: op(),
+                msg: format!("Field {} does not exist", &self.field_name),
+            })
+        }
+
+    }
+}
+
+pub struct FieldFilterFactory;
+
+impl MpsFilterFactory<FieldFilter> for FieldFilterFactory {
+    fn is_filter(&self, tokens: &VecDeque<&MpsToken>) -> bool {
+        let tokens_len = tokens.len();
+        (tokens_len == 3 // field > variable OR field < variable
+            && tokens[0].is_name()
+            && (tokens[1].is_open_angle_bracket() || tokens[1].is_close_angle_bracket())
+            && (tokens[2].is_name() || check_is_type(&tokens[2]))
+        )
+        || (tokens_len == 4 // field >= variable OR field <= variable
+            && tokens[0].is_name()
+            && (tokens[1].is_open_angle_bracket() || tokens[1].is_close_angle_bracket() || tokens[1].is_equals())
+            && tokens[2].is_equals()
+            && (tokens[3].is_name() || check_is_type(&tokens[3]))
+        )
+    }
+
+    fn build_filter(
+        &self,
+        tokens: &mut VecDeque<MpsToken>,
+        _dict: &MpsLanguageDictionary,
+    ) -> Result<FieldFilter, SyntaxError> {
+        let field = assert_token(|t| match t {
+            MpsToken::Name(n) => Some(n),
+            _ => None
+        }, MpsToken::Name("field_name".into()), tokens)?;
+        let compare_operator = assert_comparison_operator(tokens)?;
+        if check_is_type(&tokens[0]) {
+            let value = VariableOrValue::Value(assert_type(tokens)?);
+            Ok(FieldFilter{
+                field_name: field,
+                val: value,
+                comparison: compare_operator,
+            })
+        } else {
+            let variable = VariableOrValue::Variable(
+                assert_token(|t| match t {
+                    MpsToken::Name(n) => Some(n),
+                    _ => None
+                }, MpsToken::Name("variable_name".into()), tokens)?
+            );
+            Ok(FieldFilter{
+                field_name: field,
+                val: variable,
+                comparison: compare_operator,
+            })
+        }
+    }
+}
+
+pub type FieldFilterStatementFactory = MpsFilterStatementFactory<FieldFilter, FieldFilterFactory>;
+
+#[inline(always)]
+pub fn field_filter() -> FieldFilterStatementFactory {
+    FieldFilterStatementFactory::new(FieldFilterFactory)
+}

mps-interpreter/src/lang/vocabulary/filters/mod.rs

@@ -0,0 +1,6 @@
+mod empty_filter;
+mod field_filter;
+pub(crate) mod utility;
+
+pub use empty_filter::{EmptyFilter, EmptyFilterFactory, EmptyFilterStatementFactory, empty_filter};
+pub use field_filter::{FieldFilter, FieldFilterFactory, FieldFilterStatementFactory, field_filter};

mps-interpreter/src/lang/vocabulary/filters/utility.rs

@@ -0,0 +1,57 @@
+use std::collections::VecDeque;
+use std::collections::HashMap;
+
+use crate::MpsMusicItem;
+use crate::lang::SyntaxError;
+use crate::lang::MpsTypePrimitive;
+use crate::lang::utility::assert_token_raw;
+use crate::tokens::MpsToken;
+
+pub fn item_to_primitive_lut(item: MpsMusicItem) -> HashMap<String, MpsTypePrimitive> {
+    let mut result = HashMap::new();
+    result.insert("title".into(), MpsTypePrimitive::String(item.title));
+    result.insert("artist".into(), MpsTypePrimitive::String(item.artist.unwrap_or("".to_owned())));
+    result.insert("album".into(), MpsTypePrimitive::String(item.album.unwrap_or("".to_owned())));
+    result.insert("filename".into(), MpsTypePrimitive::String(item.filename));
+    result.insert("genre".into(), MpsTypePrimitive::String(item.genre.unwrap_or("".to_owned())));
+    result.insert("track".into(), MpsTypePrimitive::UInt(item.track.unwrap_or(0)));
+    result.insert("year".into(), MpsTypePrimitive::UInt(item.year.unwrap_or(0)));
+    result
+}
+
+pub fn assert_comparison_operator(tokens: &mut VecDeque<MpsToken>) -> Result<[i8; 2], SyntaxError> {
+    let token1 = tokens.pop_front().unwrap();
+    match token1 {
+        MpsToken::Equals => {
+            if tokens.len() != 0 && tokens[0].is_equals() { // tokens: ==
+                assert_token_raw(MpsToken::Equals, tokens)?;
+                Ok([0, 0])
+            } else {
+                Err(SyntaxError {
+                    line: 0,
+                    token: MpsToken::Equals,
+                })
+            }
+        },
+        MpsToken::OpenAngleBracket => {
+            if tokens.len() != 0 && tokens[0].is_equals() { // tokens: <=
+                assert_token_raw(MpsToken::Equals, tokens)?;
+                Ok([0, -1])
+            } else { // token: <
+                Ok([-1, -1])
+            }
+        },
+        MpsToken::CloseAngleBracket => {
+            if tokens.len() != 0 && tokens[0].is_equals() { // tokens: >=
+                assert_token_raw(MpsToken::Equals, tokens)?;
+                Ok([0, 1])
+            } else { // token: >
+                Ok([1, 1])
+            }
+        },
+        _ => Err(SyntaxError {
+            line: 0,
+            token: MpsToken::Equals, // TODO this can be < > or =
+        })
+    }
+}

mps-interpreter/src/lang/vocabulary/mod.rs

@@ -0,0 +1,14 @@
+mod comment;
+mod repeat;
+mod sql_init;
+mod sql_query;
+mod sql_simple_query;
+mod variable_assign;
+
+pub use sql_query::{SqlStatement, SqlStatementFactory};
+pub use sql_simple_query::{SimpleSqlStatement, SimpleSqlStatementFactory};
+pub use comment::{CommentStatement, CommentStatementFactory};
+pub use repeat::{RepeatStatement, RepeatStatementFactory};
+pub use variable_assign::{AssignStatement, AssignStatementFactory};
+pub use sql_init::{SqlInitStatement, SqlInitStatementFactory};
+pub mod filters;

mps-interpreter/src/lang/vocabulary/repeat.rs

@@ -0,0 +1,169 @@
+use std::collections::VecDeque;
+use std::fmt::{Debug, Display, Error, Formatter};
+use std::iter::Iterator;
+
+use crate::MpsContext;
+use crate::MpsMusicItem;
+use crate::tokens::MpsToken;
+
+use crate::lang::{RuntimeError, SyntaxError};
+use crate::lang::{MpsOp, PseudoOp, MpsOpFactory, BoxedMpsOpFactory};
+use crate::lang::MpsLanguageDictionary;
+use crate::lang::utility::{assert_name, assert_token_raw, assert_token, check_name};
+
+#[derive(Debug)]
+pub struct RepeatStatement {
+    inner_statement: PseudoOp,
+    inner_done: bool,
+    context: Option<MpsContext>,
+    cache: Vec<MpsMusicItem>,
+    cache_position: usize,
+    repetitions: usize,
+    loop_forever: bool,
+}
+
+impl Display for RepeatStatement {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "repeat({})", self.inner_statement)
+    }
+}
+
+impl std::clone::Clone for RepeatStatement {
+    fn clone(&self) -> Self {
+        Self {
+            inner_statement: self.inner_statement.clone(),
+            inner_done: self.inner_done,
+            context: None,
+            cache: self.cache.clone(),
+            cache_position: self.cache_position,
+            repetitions: self.repetitions,
+            loop_forever: self.loop_forever,
+        }
+    }
+}
+
+impl Iterator for RepeatStatement {
+    type Item = Result<MpsMusicItem, RuntimeError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if !self.inner_done {
+            let real_op = match self.inner_statement.try_real() {
+                Err(e) => return Some(Err(e)),
+                Ok(real) => real
+            };
+            if self.context.is_some() {
+                let ctx = self.context.take().unwrap();
+                real_op.enter(ctx);
+            }
+            let inner_item = real_op.next();
+            match inner_item {
+                Some(x) => {
+                    return match x {
+                        Ok(music) => {
+                            self.cache.push(music.clone());
+                            Some(Ok(music))
+                        },
+                        Err(e) => Some(Err(e))
+                    }
+                },
+                None => {
+                    self.inner_done = true;
+                    self.context = Some(real_op.escape());
+                },
+            }
+        }
+        // inner is done
+        if self.repetitions == 0 && !self.loop_forever {
+            None
+        } else {
+            if self.cache.len() == 0 {
+                if self.loop_forever {
+                        Some(Err(RuntimeError {
+                        line: 0,
+                        op: (Box::new(self.clone()) as Box<dyn MpsOp>).into(),
+                        msg: "Cannot repeat nothing".into()
+                    }))
+                } else {
+                    None
+                }
+            } else {
+                let music_item = self.cache[self.cache_position].clone();
+                self.cache_position += 1;
+                if self.cache_position == self.cache.len() {
+                    if self.repetitions != 0 { self.repetitions -= 1; }
+                    self.cache_position = 0;
+                }
+                Some(Ok(music_item))
+            }
+        }
+    }
+}
+
+impl MpsOp for RepeatStatement {
+    fn enter(&mut self, ctx: MpsContext) {
+        self.context = Some(ctx)
+    }
+
+    fn escape(&mut self) -> MpsContext {
+        if self.context.is_some() {
+            self.context.take().unwrap()
+        } else {
+            self.inner_statement.try_real().unwrap().escape()
+        }
+    }
+}
+
+pub struct RepeatStatementFactory;
+
+impl MpsOpFactory<RepeatStatement> for RepeatStatementFactory {
+    fn is_op(&self, tokens: &VecDeque<MpsToken>) -> bool {
+        tokens.len() >= 3
+        && check_name("repeat", &tokens[0])
+        && tokens[1].is_open_bracket()
+    }
+
+    fn build_op(
+        &self,
+        tokens: &mut VecDeque<MpsToken>,
+        dict: &MpsLanguageDictionary,
+    ) -> Result<RepeatStatement, SyntaxError> {
+        // repeat(query) or repeat(query, repetitions)
+        assert_name("repeat", tokens)?;
+        assert_token_raw(MpsToken::OpenBracket, tokens)?;
+        let inner_statement = dict.try_build_statement(tokens)?;
+        let mut count: Option<usize> = None;
+        if tokens[0].is_close_bracket() { // no repititions
+            assert_token_raw(MpsToken::CloseBracket, tokens)?;
+        } else if tokens[0].is_comma() { // repetitions specified
+            assert_token_raw(MpsToken::Comma, tokens)?;
+            count = Some(assert_token(|t| match t {
+                MpsToken::Name(n) => n.parse::<usize>().map(|d| d - 1).ok(),
+                _ => None
+            }, MpsToken::Name("usize".into()), tokens)?);
+            assert_token_raw(MpsToken::CloseBracket, tokens)?;
+        }
+        Ok(RepeatStatement {
+            inner_statement: inner_statement.into(),
+            inner_done: false,
+            context: None,
+            cache: Vec::new(),
+            cache_position: 0,
+            repetitions: count.unwrap_or(0),
+            loop_forever: count.is_none()
+        })
+    }
+}
+
+impl BoxedMpsOpFactory for RepeatStatementFactory {
+    fn build_op_boxed(
+        &self,
+        tokens: &mut VecDeque<MpsToken>,
+        dict: &MpsLanguageDictionary,
+    ) -> Result<Box<dyn MpsOp>, SyntaxError> {
+        self.build_box(tokens, dict)
+    }
+
+    fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool {
+        self.is_op(tokens)
+    }
+}

mps-interpreter/src/lang/vocabulary/sql_init.rs

@@ -0,0 +1,115 @@
+use std::collections::VecDeque;
+use std::fmt::{Debug, Display, Error, Formatter};
+use std::iter::Iterator;
+use std::collections::HashMap;
+
+use crate::MpsContext;
+use crate::MpsMusicItem;
+use crate::tokens::MpsToken;
+
+use crate::lang::{RuntimeError, SyntaxError};
+use crate::lang::{MpsOp, SimpleMpsOpFactory, MpsOpFactory, BoxedMpsOpFactory};
+use crate::lang::MpsLanguageDictionary;
+use crate::lang::utility::{assert_token_raw, check_name, assert_name, assert_token};
+use crate::lang::repeated_tokens;
+
+#[derive(Debug)]
+pub struct SqlInitStatement {
+    context: Option<MpsContext>,
+    params: HashMap<String, String>,
+}
+
+impl Display for SqlInitStatement {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "sql_init(")?;
+        for (key, val) in self.params.iter() {
+            write!(f, "{} = {},", key, val)?;
+        }
+        write!(f, ")")
+    }
+}
+
+impl std::clone::Clone for SqlInitStatement {
+    fn clone(&self) -> Self {
+        Self {
+            context: None,
+            params: HashMap::new(),
+        }
+    }
+}
+
+impl Iterator for SqlInitStatement {
+    type Item = Result<MpsMusicItem, RuntimeError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let pseudo_clone = self.clone();
+        // execute
+        match self.context.as_mut().unwrap().database.init_with_params(&self.params,
+            &mut move || (Box::new(pseudo_clone.clone()) as Box<dyn MpsOp>).into()) {
+            Ok(_) => None,
+            Err(e) => Some(Err(e))
+        }
+    }
+}
+
+impl MpsOp for SqlInitStatement {
+    fn enter(&mut self, ctx: MpsContext) {
+        self.context = Some(ctx)
+    }
+
+    fn escape(&mut self) -> MpsContext {
+        self.context.take().unwrap()
+    }
+}
+
+pub struct SqlInitStatementFactory;
+
+impl SimpleMpsOpFactory<SqlInitStatement> for SqlInitStatementFactory {
+    fn is_op_simple(&self, tokens: &VecDeque<MpsToken>) -> bool {
+        tokens.len() >= 3
+        && check_name("sql_init", &tokens[0])
+        && tokens[1].is_open_bracket()
+    }
+
+    fn build_op_simple(
+        &self,
+        tokens: &mut VecDeque<MpsToken>,
+    ) -> Result<SqlInitStatement, SyntaxError> {
+        assert_name("sql_init", tokens)?;
+        assert_token_raw(MpsToken::OpenBracket, tokens)?;
+        let ingest = |tokens2: &mut VecDeque<MpsToken>| {
+            if tokens2.len() < 3 {return Ok(None);} // nothing wrong, nothing left to ingest
+            let param_name = assert_token(|t| match t {
+                MpsToken::Name(s) => Some(s),
+                _ => None,
+            }, MpsToken::Name("param".into()), tokens2)?;
+            assert_token_raw(MpsToken::Equals, tokens2)?;
+            let param_val = assert_token(|t| match t {
+                MpsToken::Name(s) => Some(s),
+                MpsToken::Literal(s) => Some(s),
+                _ => None,
+            }, MpsToken::Name("value".into()), tokens2)?;
+            Ok(Some((param_name, param_val))) // successfully ingested one phrase
+        };
+        let params = repeated_tokens(ingest, MpsToken::Comma).ingest_all(tokens)?;
+        assert_token_raw(MpsToken::CloseBracket, tokens)?;
+        Ok(SqlInitStatement {
+            context: None,
+            params: HashMap::from_iter(params),
+        })
+    }
+}
+
+impl BoxedMpsOpFactory for SqlInitStatementFactory {
+    fn build_op_boxed(
+        &self,
+        tokens: &mut VecDeque<MpsToken>,
+        dict: &MpsLanguageDictionary,
+    ) -> Result<Box<dyn MpsOp>, SyntaxError> {
+        self.build_box(tokens, dict)
+    }
+
+    fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool {
+        self.is_op(tokens)
+    }
+}

mps-interpreter/src/lang/vocabulary/sql_query.rs

@@ -2,9 +2,9 @@ use std::collections::VecDeque;
 use std::fmt::{Debug, Display, Error, Formatter};
 use std::iter::Iterator;
 
-use super::utility::{assert_token, assert_token_raw};
-use super::{BoxedMpsOpFactory, MpsLanguageDictionary, MpsOp, MpsOpFactory};
-use super::{RuntimeError, SyntaxError};
+use crate::lang::utility::{assert_token, assert_token_raw, check_name, assert_name};
+use crate::lang::{BoxedMpsOpFactory, MpsLanguageDictionary, MpsOp, MpsOpFactory};
+use crate::lang::{RuntimeError, SyntaxError};
 use crate::tokens::MpsToken;
 use crate::MpsContext;
 use crate::MpsMusicItem;
@@ -35,7 +35,7 @@ impl SqlStatement {
                     Ok(item) => Some(Ok(item.clone())),
                     Err(e) => Some(Err(RuntimeError {
                         line: e.line,
-                        op: Box::new(self.clone()),
+                        op: (Box::new(self.clone()) as Box<dyn MpsOp>).into(),
                         msg: e.msg.clone(),
                     })),
                 }
@@ -43,7 +43,7 @@ impl SqlStatement {
         } else {
             Some(Err(RuntimeError {
                 line: 0,
-                op: Box::new(self.clone()),
+                op: (Box::new(self.clone()) as Box<dyn MpsOp>).into(),
                 msg: format!("Context error: rows is None").into(),
             }))
         }
@@ -84,7 +84,7 @@ impl Iterator for SqlStatement {
             // query has not been executed yet
             match ctx
                 .database
-                .raw(&self.query, &mut move || Box::new(self_clone.clone()))
+                .raw(&self.query, &mut move || (Box::new(self_clone.clone()) as Box<dyn MpsOp>).into())
             {
                 Err(e) => return Some(Err(e)),
                 Ok(rows) => {
@@ -108,7 +108,7 @@ impl MpsOpFactory<SqlStatement> for SqlStatementFactory {
     #[inline]
     fn is_op(&self, tokens: &VecDeque<MpsToken>) -> bool {
         tokens.len() > 3
-            && tokens[0].is_sql()
+            && check_name("sql", &tokens[0])
             && tokens[1].is_open_bracket()
             && tokens[2].is_literal()
             && tokens[3].is_close_bracket()
@@ -121,7 +121,8 @@ impl MpsOpFactory<SqlStatement> for SqlStatementFactory {
         _dict: &MpsLanguageDictionary,
     ) -> Result<SqlStatement, SyntaxError> {
         // sql ( `some query` )
-        assert_token_raw(MpsToken::Sql, tokens)?;
+        assert_name("sql", tokens)?;
+        //assert_token_raw(MpsToken::Sql, tokens)?;
         assert_token_raw(MpsToken::OpenBracket, tokens)?;
         let literal = assert_token(
             |t| match t {

mps-interpreter/src/lang/vocabulary/sql_simple_query.rs

@@ -2,9 +2,9 @@ use std::collections::VecDeque;
 use std::fmt::{Debug, Display, Error, Formatter};
 use std::iter::Iterator;
 
-use super::utility::{assert_token, assert_token_raw};
-use super::{BoxedMpsOpFactory, MpsLanguageDictionary, MpsOp, MpsOpFactory};
-use super::{RuntimeError, SyntaxError};
+use crate::lang::utility::{assert_token, assert_token_raw};
+use crate::lang::{BoxedMpsOpFactory, MpsLanguageDictionary, MpsOp, MpsOpFactory};
+use crate::lang::{RuntimeError, SyntaxError};
 use crate::tokens::MpsToken;
 use crate::MpsContext;
 use crate::MpsMusicItem;
@@ -83,7 +83,7 @@ impl SimpleSqlStatement {
                     Ok(item) => Some(Ok(item.clone())),
                     Err(e) => Some(Err(RuntimeError {
                         line: e.line,
-                        op: Box::new(self.clone()),
+                        op: (Box::new(self.clone()) as Box<dyn MpsOp>).into(),
                         msg: e.msg.clone(),
                     })),
                 }
@@ -91,7 +91,7 @@ impl SimpleSqlStatement {
         } else {
             Some(Err(RuntimeError {
                 line: 0,
-                op: Box::new(self.clone()),
+                op: (Box::new(self.clone()) as Box<dyn MpsOp>).into(),
                 msg: format!("Context error: rows is None").into(),
             }))
         }
@@ -134,16 +134,16 @@ impl Iterator for SimpleSqlStatement {
             let query_result = match self.mode {
                 QueryMode::Artist => ctx
                     .database
-                    .artist_like(&self.query, &mut move || Box::new(self_clone.clone())),
+                    .artist_like(&self.query, &mut move || (Box::new(self_clone.clone()) as Box<dyn MpsOp>).into()),
                 QueryMode::Album => ctx
                     .database
-                    .album_like(&self.query, &mut move || Box::new(self_clone.clone())),
+                    .album_like(&self.query, &mut move || (Box::new(self_clone.clone()) as Box<dyn MpsOp>).into()),
                 QueryMode::Song => ctx
                     .database
-                    .song_like(&self.query, &mut move || Box::new(self_clone.clone())),
+                    .song_like(&self.query, &mut move || (Box::new(self_clone.clone()) as Box<dyn MpsOp>).into()),
                 QueryMode::Genre => ctx
                     .database
-                    .genre_like(&self.query, &mut move || Box::new(self_clone.clone())),
+                    .genre_like(&self.query, &mut move || (Box::new(self_clone.clone()) as Box<dyn MpsOp>).into()),
             };
             match query_result {
                 Err(e) => return Some(Err(e)),

mps-interpreter/src/lang/vocabulary/variable_assign.rs

@@ -0,0 +1,198 @@
+use std::collections::VecDeque;
+use std::fmt::{Debug, Display, Error, Formatter};
+use std::iter::Iterator;
+
+use crate::MpsContext;
+use crate::MpsMusicItem;
+use crate::tokens::MpsToken;
+
+use crate::lang::{RuntimeError, SyntaxError};
+use crate::lang::{MpsOp, PseudoOp, MpsOpFactory, BoxedMpsOpFactory, MpsTypePrimitive};
+use crate::lang::MpsLanguageDictionary;
+use crate::lang::utility::{assert_token_raw, assert_token, check_is_type, assert_type};
+use crate::processing::general::MpsType;
+
+#[derive(Debug)]
+pub struct AssignStatement {
+    variable_name: String,
+    inner_statement: Option<PseudoOp>,
+    assign_type: Option<MpsTypePrimitive>,
+    context: Option<MpsContext>,
+    is_declaration: bool,
+    is_simple: bool,
+}
+
+impl Display for AssignStatement {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        if let Some(inner_statement) = &self.inner_statement {
+            write!(f, "{} = {}", self.variable_name, inner_statement)
+        } else {
+            write!(f, "{} = ???", self.variable_name)
+        }
+    }
+}
+
+impl std::clone::Clone for AssignStatement {
+    fn clone(&self) -> Self {
+        Self {
+            variable_name: self.variable_name.clone(),
+            inner_statement: self.inner_statement.clone(),
+            assign_type: self.assign_type.clone(),
+            context: None,
+            is_declaration: self.is_declaration,
+            is_simple: self.is_simple,
+        }
+    }
+}
+
+impl Iterator for AssignStatement {
+    type Item = Result<MpsMusicItem, RuntimeError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(inner_statement) = &mut self.inner_statement {
+            if inner_statement.is_fake() {
+                Some(Err(RuntimeError {
+                    line: 0,
+                    op: (Box::new(self.clone()) as Box<dyn MpsOp>).into(),
+                    msg: format!("Variable {} already assigned, cannot redo assignment", self.variable_name),
+                }))
+            } else {
+                let mut inner = inner_statement.clone();
+                std::mem::swap(inner_statement, &mut inner);
+                let real = match inner.unwrap_real() {
+                    Ok(real) => real,
+                    Err(e) => return Some(Err(e)),
+                };
+                let pseudo_clone = self.clone();
+                let result;
+                if self.is_declaration {
+                    result = self.context.as_mut().unwrap()
+                        .variables.declare(
+                            &self.variable_name,
+                            MpsType::Op(real),
+                            &mut move ||(Box::new(pseudo_clone.clone()) as Box<dyn MpsOp>).into()
+                    );
+                } else {
+                    result = self.context.as_mut().unwrap()
+                        .variables.assign(
+                            &self.variable_name,
+                            MpsType::Op(real),
+                            &mut move ||(Box::new(pseudo_clone.clone()) as Box<dyn MpsOp>).into()
+                    );
+                }
+                match result {
+                    Ok(_) => None,
+                    Err(e) => Some(Err(e))
+                }
+            }
+        } else if !self.is_simple {
+            panic!("Assignee statement for {} is None but assignment is not simple type", self.variable_name)
+            /*Some(Err(RuntimeError {
+                line: 0,
+                op: (Box::new(self.clone()) as Box<dyn MpsOp>).into(),
+                msg: format!("(BUG) Assignee statement for {} is None but assignment is not simple type", self.variable_name),
+            }))*/
+        } else {
+            let assign_type = self.assign_type.clone().unwrap();
+            let pseudo_clone = self.clone();
+            let result;
+            if self.is_declaration {
+                result = self.context.as_mut().unwrap()
+                    .variables.declare(
+                        &self.variable_name,
+                        MpsType::Primitive(assign_type),
+                        &mut move ||(Box::new(pseudo_clone.clone()) as Box<dyn MpsOp>).into()
+                );
+            } else {
+                result = self.context.as_mut().unwrap()
+                    .variables.assign(
+                        &self.variable_name,
+                        MpsType::Primitive(assign_type),
+                        &mut move ||(Box::new(pseudo_clone.clone()) as Box<dyn MpsOp>).into()
+                );
+            }
+            match result {
+                Ok(_) => None,
+                Err(e) => Some(Err(e))
+            }
+        }
+    }
+}
+
+
+impl MpsOp for AssignStatement {
+    fn enter(&mut self, ctx: MpsContext) {
+        self.context = Some(ctx)
+    }
+
+    fn escape(&mut self) -> MpsContext {
+        self.context.take().unwrap()
+    }
+}
+
+pub struct AssignStatementFactory;
+
+impl MpsOpFactory<AssignStatement> for AssignStatementFactory {
+    fn is_op(&self, tokens: &VecDeque<MpsToken>) -> bool {
+        (tokens.len() >= 3
+            &&tokens[0].is_name() // can be any (valid) variable name
+            && tokens[1].is_equals())
+        || (tokens.len() >= 4
+            && tokens[0].is_let()
+            && tokens[1].is_name() // any name
+            && tokens[2].is_equals())
+    }
+
+    fn build_op(
+        &self,
+        tokens: &mut VecDeque<MpsToken>,
+        dict: &MpsLanguageDictionary,
+    ) -> Result<AssignStatement, SyntaxError> {
+        // [let] variable_name = inner_statement
+        let is_decl = tokens[0].is_let();
+        if is_decl { // variable declarations start with let, assignments do not
+            assert_token_raw(MpsToken::Let, tokens)?;
+        }
+        let name = assert_token(|t| match t {
+            MpsToken::Name(s) => Some(s),
+            _ => None
+        }, MpsToken::Name("variable_name".into()), tokens)?;
+        assert_token_raw(MpsToken::Equals, tokens)?;
+        let is_simple_assign = check_is_type(&tokens[0]);
+        if is_simple_assign {
+            let simple_type = assert_type(tokens)?;
+            Ok(AssignStatement {
+                variable_name: name,
+                inner_statement: None,
+                assign_type: Some(simple_type),
+                context: None,
+                is_declaration: is_decl,
+                is_simple: true,
+            })
+        } else {
+            let inner_statement = dict.try_build_statement(tokens)?;
+            Ok(AssignStatement {
+                variable_name: name,
+                inner_statement: Some(inner_statement.into()),
+                assign_type: None,
+                context: None,
+                is_declaration: is_decl,
+                is_simple: false,
+            })
+        }
+    }
+}
+
+impl BoxedMpsOpFactory for AssignStatementFactory {
+    fn build_op_boxed(
+        &self,
+        tokens: &mut VecDeque<MpsToken>,
+        dict: &MpsLanguageDictionary,
+    ) -> Result<Box<dyn MpsOp>, SyntaxError> {
+        self.build_box(tokens, dict)
+    }
+
+    fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool {
+        self.is_op(tokens)
+    }
+}

mps-interpreter/src/music/build_library.rs

@@ -1,9 +1,39 @@
 use std::path::Path;
 
+use crate::lang::db::*;
 use super::MpsLibrary;
 
-pub fn build_library<P: AsRef<Path>>(path: P) -> std::io::Result<MpsLibrary> {
-    let mut result = MpsLibrary::new();
-    result.read_path(path, 10)?;
-    Ok(result)
+pub fn build_library_from_files<P: AsRef<Path>>(path: P, lib: &mut MpsLibrary) -> std::io::Result<()> {
+    //let mut result = MpsLibrary::new();
+    lib.read_path(path, 10)?;
+    Ok(())
+}
+
+pub fn build_library_from_sqlite(conn: &rusqlite::Connection, lib: &mut MpsLibrary) -> rusqlite::Result<()> {
+    // build songs
+    for song in conn.prepare("SELECT * from songs")?
+        .query_map([], DbMusicItem::map_row)? {
+        lib.add_song(song?);
+    }
+    // build metadata
+    for meta in conn.prepare("SELECT * from metadata")?
+        .query_map([], DbMetaItem::map_row)? {
+        lib.add_metadata(meta?);
+    }
+    // build artists
+    for artist in conn.prepare("SELECT * from artists")?
+        .query_map([], DbArtistItem::map_row)? {
+        lib.add_artist(artist?);
+    }
+    // build albums
+    for album in conn.prepare("SELECT * from albums")?
+        .query_map([], DbAlbumItem::map_row)? {
+        lib.add_album(album?);
+    }
+    // build genres
+    for genre in conn.prepare("SELECT * from genres")?
+        .query_map([], DbGenreItem::map_row)? {
+        lib.add_genre(genre?);
+    }
+    Ok(())
 }

mps-interpreter/src/music/library.rs

@@ -1,5 +1,6 @@
-use std::collections::HashMap;
-use std::path::Path;
+use std::collections::{HashMap, HashSet};
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
 
 use symphonia::core::io::MediaSourceStream;
 use symphonia::core::probe::Hint;
@@ -14,6 +15,8 @@ pub struct MpsLibrary {
     artists: HashMap<String, DbArtistItem>,
     albums: HashMap<String, DbAlbumItem>,
     genres: HashMap<String, DbGenreItem>,
+    files: HashSet<PathBuf>,
+    dirty: bool,
 }
 
 impl MpsLibrary {
@@ -24,6 +27,8 @@ impl MpsLibrary {
             artists: HashMap::new(),
             albums: HashMap::new(),
             genres: HashMap::new(),
+            files: HashSet::new(),
+            dirty: false,
         }
     }
 
@@ -31,28 +36,80 @@ impl MpsLibrary {
         self.songs.len()
     }
 
+    pub fn clear_modified(&mut self) {
+        self.dirty = false;
+    }
+
+    pub fn is_modified(&self) -> bool {
+        self.dirty
+    }
+
+    #[inline(always)]
+    fn modify(&mut self) {
+        self.dirty = true;
+    }
+
+    #[inline]
+    pub fn contains_path<P: AsRef<Path>>(&self, path: P) -> bool {
+        self.files.contains(&path.as_ref().to_path_buf())
+    }
+
     pub fn all_songs<'a>(&'a self) -> Vec<&'a DbMusicItem> {
         self.songs.values().collect()
     }
 
+    #[inline]
+    pub fn add_song(&mut self, song: DbMusicItem) {
+        self.modify();
+        if let Ok(path) = PathBuf::from_str(&song.filename) {
+            self.files.insert(path);
+        }
+        self.songs.insert(song.song_id, song);
+    }
+
     pub fn all_metadata<'a>(&'a self) -> Vec<&'a DbMetaItem> {
         self.metadata.values().collect()
     }
 
+    #[inline]
+    pub fn add_metadata(&mut self, meta: DbMetaItem) {
+        self.modify();
+        self.metadata.insert(meta.meta_id, meta);
+    }
+
     pub fn all_artists<'a>(&'a self) -> Vec<&'a DbArtistItem> {
         self.artists.values().collect()
     }
 
+    #[inline]
+    pub fn add_artist(&mut self, artist: DbArtistItem) {
+        self.modify();
+        self.artists.insert(Self::sanitise_key(&artist.name), artist);
+    }
+
     pub fn all_albums<'a>(&'a self) -> Vec<&'a DbAlbumItem> {
         self.albums.values().collect()
     }
 
+    #[inline]
+    pub fn add_album(&mut self, album: DbAlbumItem) {
+        self.modify();
+        self.albums.insert(Self::sanitise_key(&album.title), album);
+    }
+
     pub fn all_genres<'a>(&'a self) -> Vec<&'a DbGenreItem> {
         self.genres.values().collect()
     }
 
+    #[inline]
+    pub fn add_genre(&mut self, genre: DbGenreItem) {
+        self.modify();
+        self.genres.insert(Self::sanitise_key(&genre.title), genre);
+    }
+
     pub fn read_path<P: AsRef<Path>>(&mut self, path: P, depth: usize) -> std::io::Result<()> {
         let path = path.as_ref();
+        if self.contains_path(path) { return Ok(()); } // skip existing entries
         if path.is_dir() && depth != 0 {
             for entry in path.read_dir()? {
                 self.read_path(entry?.path(), depth - 1)?;
@@ -104,27 +161,24 @@ impl MpsLibrary {
         } // probably not a valid song, let's skip it
         let song_id = self.songs.len() as u64; // guaranteed to be created
         let meta_id = self.metadata.len() as u64; // guaranteed to be created
-        self.metadata.insert(meta_id, tags.meta(meta_id)); // definitely necessary
+        self.add_metadata(tags.meta(meta_id)); // definitely necessary
                                                            // genre has no links to others, so find that first
         let mut genre = tags.genre(0);
         genre.genre_id = Self::find_or_gen_id(&self.genres, &genre.title);
         if genre.genre_id == self.genres.len() as u64 {
-            self.genres
-                .insert(Self::sanitise_key(&genre.title), genre.clone());
+            self.add_genre(genre.clone());
         }
         // artist only links to genre, so that can be next
         let mut artist = tags.artist(0, genre.genre_id);
         artist.artist_id = Self::find_or_gen_id(&self.artists, &artist.name);
         if artist.artist_id == self.artists.len() as u64 {
-            self.artists
-                .insert(Self::sanitise_key(&artist.name), artist.clone());
+            self.add_artist(artist.clone());
         }
         // same with album artist
         let mut album_artist = tags.album_artist(0, genre.genre_id);
         album_artist.artist_id = Self::find_or_gen_id(&self.artists, &album_artist.name);
         if album_artist.artist_id == self.artists.len() as u64 {
-            self.artists
-                .insert(Self::sanitise_key(&album_artist.name), album_artist.clone());
+            self.add_artist(album_artist.clone());
         }
         // album now has all links ready
         let mut album = tags.album(0, 0, album_artist.artist_id, genre.genre_id);
@@ -132,14 +186,12 @@ impl MpsLibrary {
         if album.album_id == self.albums.len() as u64 {
             let album_meta = tags.album_meta(self.metadata.len() as u64);
             album.metadata = album_meta.meta_id;
-            self.albums
-                .insert(Self::sanitise_key(&album.title), album.clone());
-            self.metadata.insert(album_meta.meta_id, album_meta);
+            self.add_album(album.clone());
+            self.add_metadata(album_meta);
         }
         //let meta_album_id = self.metadata.len() as u64;
         //let album = tags.album(album_id, meta_album_id);
-        self.songs.insert(
-            song_id,
+        self.add_song(
             tags.song(
                 song_id,
                 artist.artist_id,

mps-interpreter/src/music/mod.rs

@@ -2,5 +2,5 @@ mod build_library;
 mod library;
 mod tag;
 
-pub use build_library::build_library;
+pub use build_library::{build_library_from_files, build_library_from_sqlite};
 pub use library::MpsLibrary;

mps-interpreter/src/music/tag.rs

@@ -45,7 +45,7 @@ impl Tags {
             self.filename
                 .file_name()
                 .and_then(|file| file.to_str())
-                .and_then(|file| Some(file.replacen(extension, "", 1)))
+                .and_then(|file| Some(file.replacen(&format!(".{}", extension), "", 1)))
                 .unwrap_or("Unknown Title".into())
         };
         DbMusicItem {

mps-interpreter/src/music_item.rs

@@ -1,9 +1,14 @@
-use super::lang::db::{DatabaseObj, DbMusicItem};
+use super::lang::db::{DatabaseObj, DbMusicItem, DbArtistItem, DbAlbumItem, DbMetaItem, DbGenreItem};
 
 #[derive(Clone, Debug)]
 pub struct MpsMusicItem {
     pub title: String,
+    pub artist: Option<String>,
+    pub album: Option<String>,
     pub filename: String,
+    pub genre: Option<String>,
+    pub track: Option<u64>,
+    pub year: Option<u64>,
 }
 
 impl MpsMusicItem {
@@ -11,7 +16,30 @@ impl MpsMusicItem {
         let item = DbMusicItem::map_row(row)?;
         Ok(Self {
             title: item.title,
+            artist: None,
+            album: None,
             filename: item.filename,
+            genre: None,
+            track: None,
+            year: None,
         })
     }
+
+    pub fn merge(
+        music: DbMusicItem,
+        artist: DbArtistItem,
+        album: DbAlbumItem,
+        meta: DbMetaItem,
+        genre: DbGenreItem
+    ) -> Self {
+        Self {
+            title: music.title,
+            artist: Some(artist.name),
+            album: Some(album.title),
+            filename: music.filename,
+            genre: Some(genre.title),
+            track: Some(meta.track),
+            year: Some(meta.date),
+        }
+    }
 }

mps-interpreter/src/processing/mod.rs

@@ -1,5 +1,12 @@
 mod sql;
+mod variables;
+
+pub type OpGetter = dyn FnMut() -> crate::lang::PseudoOp;
 
 pub mod database {
-    pub use super::sql::{MpsDatabaseQuerier, MpsSQLiteExecutor, QueryOp, QueryResult};
+    pub use super::sql::{MpsDatabaseQuerier, MpsSQLiteExecutor, QueryResult};
+}
+
+pub mod general {
+    pub use super::variables::{MpsVariableStorer, MpsOpStorage, MpsType};
 }

mps-interpreter/src/processing/sql.rs

@@ -1,22 +1,36 @@
 use core::fmt::Debug;
+use std::collections::{HashMap, HashSet};
 
 use crate::lang::db::*;
-use crate::lang::{MpsOp, RuntimeError};
+use crate::lang::RuntimeError;
 use crate::MpsMusicItem;
 
-pub type QueryResult = Result<Vec<Result<MpsMusicItem, RuntimeError>>, RuntimeError>;
-pub type QueryOp = dyn FnMut() -> Box<dyn MpsOp>;
+use super::OpGetter as QueryOp;
 
+pub type QueryResult = Result<Vec<Result<MpsMusicItem, RuntimeError>>, RuntimeError>;
+
+/// SQL querying functionality, loosely de-coupled from any specific SQL dialect (excluding raw call)
 pub trait MpsDatabaseQuerier: Debug {
+    /// raw SQL call, assumed (but not guaranteed) to retrieved music items
     fn raw(&mut self, query: &str, op: &mut QueryOp) -> QueryResult;
 
+    /// get music, searching by artist name like `query`
     fn artist_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult;
 
+    /// get music, searching by album title like `query`
     fn album_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult;
 
+    /// get music, searching by song title like `query`
     fn song_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult;
 
+    /// get music, searching by genre title like `query`
     fn genre_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult;
+
+    /// connect to the SQL database with (optional) settings such as:
+    /// `"folder" = "path"` - path to root music directory
+    /// `"database" = "uri"` - connection URI for database (for SQLite this is just a filepath)
+    /// `"generate" = "true"|"yes"|"false"|"no"` - whether to populate the database using the music directory
+    fn init_with_params(&mut self, params: &HashMap<String, String>, op: &mut QueryOp) -> Result<(), RuntimeError>;
 }
 
 #[derive(Default, Debug)]
@@ -135,6 +149,133 @@ impl MpsDatabaseQuerier for MpsSQLiteExecutor {
             WHERE genres.title like ? ORDER BY songs.album, metadata.track";
         self.music_query_single_param(query_stmt, param, op)
     }
+
+    fn init_with_params(&mut self, params: &HashMap<String, String>, op: &mut QueryOp) -> Result<(), RuntimeError> {
+        // must be executed before connection is created
+        if self.sqlite_connection.is_some() {
+            Err(RuntimeError {
+                line: 0,
+                op: op(),
+                msg: "Cannot init SQLite connection: Already connected".into(),
+            })
+        } else {
+            // process params
+            // init connection
+            let mut keys: HashSet<&String> = params.keys().collect();
+            let mut settings = SqliteSettings::default();
+            for (key, val) in params.iter() {
+                let mut match_found = false;
+                match key as &str {
+                    "folder" | "dir" => {
+                        match_found = true;
+                        settings.music_path = Some(val.clone());
+                    },
+                    "database" | "db" => {
+                        match_found = true;
+                        settings.db_path = Some(val.clone());
+                    },
+                    "generate" | "gen" => {
+                        match_found = true;
+                        settings.auto_generate = match val as &str {
+                            "true" | "yes" => Ok(true),
+                            "false" | "no" => Ok(false),
+                            x => Err(RuntimeError{
+                                line: 0,
+                                op: op(),
+                                msg: format!("Unrecognised right hand side of param \"{}\" = \"{}\"", key, x),
+                            })
+                        }?;
+                    }
+                    _ => {}
+                }
+                if match_found {
+                    keys.remove(key);
+                }
+            }
+            if !keys.is_empty() {
+                // build error msg
+                let mut concat_keys = "".to_string();
+                let mut first = true;
+                for key in keys.drain() {
+                    if first {
+                        first = false;
+                        concat_keys += key;
+                    } else {
+                        concat_keys += &format!("{}, ", key);
+                    }
+                }
+                return Err(RuntimeError {
+                    line: 0,
+                    op: op(),
+                    msg: format!("Unrecognised sql init parameter(s): {}", concat_keys)
+                })
+            }
+            self.sqlite_connection = Some(
+                settings.try_into()
+                    .map_err(|e|
+                        RuntimeError {
+                            line: 0,
+                            op: op(),
+                            msg: format!("SQL connection error: {}", e)
+                        }
+                    )?
+            );
+            Ok(())
+        }
+    }
+}
+
+struct SqliteSettings {
+    music_path: Option<String>,
+    db_path: Option<String>,
+    auto_generate: bool,
+}
+
+impl std::default::Default for SqliteSettings {
+    fn default() -> Self {
+        SqliteSettings {
+            music_path: None,
+            db_path: None,
+            auto_generate: true
+        }
+    }
+}
+
+impl std::convert::TryInto<rusqlite::Connection> for SqliteSettings {
+    type Error = rusqlite::Error;
+
+    fn try_into(self) -> Result<rusqlite::Connection, Self::Error> {
+        let music_path = self.music_path
+            .and_then(
+                |p| Some(std::path::PathBuf::from(p))
+            ).unwrap_or_else(
+                || crate::lang::utility::music_folder()
+            );
+        let sqlite_path = self.db_path.unwrap_or_else(|| crate::lang::db::DEFAULT_SQLITE_FILEPATH.to_string());
+        crate::lang::db::generate_db(music_path, sqlite_path, self.auto_generate)
+    }
+}
+
+#[inline(always)]
+fn build_mps_item(conn: &mut rusqlite::Connection, item: DbMusicItem) -> rusqlite::Result<MpsMusicItem> {
+    // query artist
+    let mut stmt = conn
+        .prepare_cached("SELECT * from artists WHERE artist_id = ?")?;
+    let artist = stmt.query_row([item.artist], DbArtistItem::map_row)?;
+    // query album
+    let mut stmt = conn
+        .prepare_cached("SELECT * from albums WHERE album_id = ?")?;
+    let album = stmt.query_row([item.album], DbAlbumItem::map_row)?;
+    // query metadata
+    let mut stmt = conn
+        .prepare_cached("SELECT * from metadata WHERE meta_id = ?")?;
+    let meta = stmt.query_row([item.metadata], DbMetaItem::map_row)?;
+    // query genre
+    let mut stmt = conn
+        .prepare_cached("SELECT * from genres WHERE genre_id = ?")?;
+    let genre = stmt.query_row([item.genre], DbGenreItem::map_row)?;
+
+    Ok(MpsMusicItem::merge(item, artist, album, meta, genre))
 }
 
 #[inline]
@@ -142,13 +283,22 @@ fn perform_query(
     conn: &mut rusqlite::Connection,
     query: &str,
 ) -> Result<Vec<rusqlite::Result<MpsMusicItem>>, String> {
+    let collection: Vec<rusqlite::Result<DbMusicItem>>;
+    {
         let mut stmt = conn
             .prepare(query)
             .map_err(|e| format!("SQLite query error: {}", e))?;
-    let iter = stmt
-        .query_map([], MpsMusicItem::map_row)
-        .map_err(|e| format!("SQLite item mapping error: {}", e))?;
-    Ok(iter.collect())
+        collection = stmt
+            .query_map([], DbMusicItem::map_row)
+            .map_err(|e| format!("SQLite item mapping error: {}", e))?
+            .collect();
+    }
+    let iter2 = collection.into_iter()
+        .map(|item| match item {
+            Ok(item) => build_mps_item(conn, item),
+            Err(e) => Err(e)
+        });
+    Ok(iter2.collect())
 }
 
 #[inline]
@@ -157,11 +307,20 @@ fn perform_single_param_query(
     query: &str,
     param: &str,
 ) -> Result<Vec<rusqlite::Result<MpsMusicItem>>, String> {
+    let collection: Vec<rusqlite::Result<DbMusicItem>>;
+    {
         let mut stmt = conn
             .prepare_cached(query)
             .map_err(|e| format!("SQLite query error: {}", e))?;
-    let iter = stmt
-        .query_map([param], MpsMusicItem::map_row)
-        .map_err(|e| format!("SQLite item mapping error: {}", e))?;
-    Ok(iter.collect())
+        collection = stmt
+            .query_map([param], DbMusicItem::map_row)
+            .map_err(|e| format!("SQLite item mapping error: {}", e))?
+            .collect();
+    }
+    let iter2 = collection.into_iter()
+        .map(|item| match item {
+            Ok(item) => build_mps_item(conn, item),
+            Err(e) => Err(e)
+        });
+    Ok(iter2.collect())
 }

mps-interpreter/src/processing/variables.rs

@@ -0,0 +1,103 @@
+use std::fmt::{Debug, Display, Error, Formatter};
+
+use std::collections::HashMap;
+
+use crate::lang::RuntimeError;
+use crate::lang::MpsOp;
+use crate::lang::MpsTypePrimitive;
+
+use super::OpGetter;
+
+#[derive(Debug)]
+pub enum MpsType {
+    Op(Box<dyn MpsOp>),
+    Primitive(MpsTypePrimitive)
+}
+
+impl Display for MpsType {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        match self {
+            Self::Op(op) => write!(f, "Op({})", op),
+            Self::Primitive(p) => write!(f, "{}", p)
+        }
+    }
+}
+
+pub trait MpsVariableStorer: Debug {
+    fn get(&self, name: &str, op: &mut OpGetter) -> Result<&MpsType, RuntimeError>;
+
+    fn get_mut(&mut self, name: &str, op: &mut OpGetter) -> Result<&mut MpsType, RuntimeError>;
+
+    fn assign(&mut self, name: &str, value: MpsType, op: &mut OpGetter) -> Result<(), RuntimeError>;
+
+    fn declare(&mut self, name: &str, value: MpsType, op: &mut OpGetter) -> Result<(), RuntimeError>;
+
+    fn remove(&mut self, name: &str, op: &mut OpGetter) -> Result<MpsType, RuntimeError>;
+}
+
+#[derive(Default, Debug)]
+pub struct MpsOpStorage {
+    storage: HashMap<String, MpsType>,
+}
+
+impl MpsVariableStorer for MpsOpStorage {
+    fn get(&self, key: &str, op: &mut OpGetter) -> Result<&MpsType, RuntimeError> {
+        match self.storage.get(key) {
+            Some(item) => Ok(item),
+            None => Err(RuntimeError {
+                line: 0,
+                op: op(),
+                msg: format!("Variable {} not found", key),
+            })
+        }
+    }
+
+    fn get_mut(&mut self, key: &str, op: &mut OpGetter) -> Result<&mut MpsType, RuntimeError> {
+        match self.storage.get_mut(key) {
+            Some(item) => Ok(item),
+            None => Err(RuntimeError {
+                line: 0,
+                op: op(),
+                msg: format!("Variable {} not found", key),
+            })
+        }
+    }
+
+    fn assign(&mut self, key: &str, item: MpsType, op: &mut OpGetter) -> Result<(), RuntimeError> {
+        if !self.storage.contains_key(key) {
+            Err(RuntimeError {
+                line: 0,
+                op: op(),
+                msg: format!("Cannot assign to non-existent variable {}", key),
+            })
+        } else {
+            self.storage.insert(key.to_string(), item);
+            Ok(())
+        }
+    }
+
+    fn declare(&mut self, key: &str, item: MpsType, op: &mut OpGetter) -> Result<(), RuntimeError> {
+        if self.storage.contains_key(key) {
+            Err(RuntimeError {
+                line: 0,
+                op: op(),
+                msg: format!("Cannot overwrite existing variable {}", key),
+            })
+        } else {
+            self.storage.insert(key.to_string(), item);
+            Ok(())
+        }
+    }
+
+    fn remove(&mut self, key: &str, op: &mut OpGetter) -> Result<MpsType, RuntimeError> {
+        if self.storage.contains_key(key) {
+            Ok(self.storage.remove(key).unwrap())
+        } else {
+            Err(RuntimeError {
+                line: 0,
+                op: op(),
+                msg: format!("Cannot remove non-existing variable {}", key),
+            })
+        }
+    }
+}

mps-interpreter/src/tokens/token_enum.rs

@@ -1,8 +1,8 @@
 use std::fmt::{Debug, Display, Error, Formatter};
 
-#[derive(Debug, Eq, PartialEq)]
+#[derive(Debug, Eq, PartialEq, Clone)]
 pub enum MpsToken {
-    Sql,
+    //Sql,
     OpenBracket,
     CloseBracket,
     Comma,
@@ -10,20 +10,30 @@ pub enum MpsToken {
     Name(String),
     //Octothorpe,
     Comment(String),
+    Equals,
+    Let,
+    OpenAngleBracket,
+    CloseAngleBracket,
+    Dot,
 }
 
 impl MpsToken {
     pub fn parse_from_string(s: String) -> Result<Self, String> {
         match &s as &str {
-            "sql" => Ok(Self::Sql),
+            //"sql" => Ok(Self::Sql),
             "(" => Ok(Self::OpenBracket),
             ")" => Ok(Self::CloseBracket),
             "," => Ok(Self::Comma),
             //"#" => Ok(Self::Octothorpe),
+            "=" => Ok(Self::Equals),
+            "let" => Ok(Self::Let),
+            "<" => Ok(Self::OpenAngleBracket),
+            ">" => Ok(Self::CloseAngleBracket),
+            "." => Ok(Self::Dot),
             _ => {
                 // name validation
                 let mut ok = true;
-                for invalid_c in ["-", "+", ",", " ", "/", "\n", "\r", "!", "?"] {
+                for invalid_c in ["-", "+", ",", " ", "/", "\n", "\r", "!", "?", "=", "."] {
                     if s.contains(invalid_c) {
                         ok = false;
                         break;
@@ -38,12 +48,12 @@ impl MpsToken {
         }
     }
 
-    pub fn is_sql(&self) -> bool {
+    /*pub fn is_sql(&self) -> bool {
         match self {
             Self::Sql => true,
             _ => false,
         }
-    }
+    }*/
 
     pub fn is_open_bracket(&self) -> bool {
         match self {
@@ -59,6 +69,13 @@ impl MpsToken {
         }
     }
 
+    pub fn is_comma(&self) -> bool {
+        match self {
+            Self::Comma => true,
+            _ => false,
+        }
+    }
+
     pub fn is_literal(&self) -> bool {
         match self {
             Self::Literal(_) => true,
@@ -86,19 +103,59 @@ impl MpsToken {
             _ => false,
         }
     }
+
+    pub fn is_equals(&self) -> bool {
+        match self {
+            Self::Equals => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_let(&self) -> bool {
+        match self {
+            Self::Let => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_open_angle_bracket(&self) -> bool {
+        match self {
+            Self::OpenAngleBracket => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_close_angle_bracket(&self) -> bool {
+        match self {
+            Self::CloseAngleBracket => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_dot(&self) -> bool {
+        match self {
+            Self::Dot => true,
+            _ => false,
+        }
+    }
 }
 
 impl Display for MpsToken {
     fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
         match self {
-            Self::Sql => write!(f, "sql"),
+            //Self::Sql => write!(f, "sql"),
             Self::OpenBracket => write!(f, "("),
             Self::CloseBracket => write!(f, ")"),
             Self::Comma => write!(f, ","),
             Self::Literal(s) => write!(f, "\"{}\"", s),
             Self::Name(s) => write!(f, "{}", s),
             //Self::Octothorpe => write!(f, "#"),
-            Self::Comment(s) => write!(f, "//{}", s),
+            Self::Comment(s) => write!(f, "{}", s),
+            Self::Equals => write!(f, "="),
+            Self::Let => write!(f, "let"),
+            Self::OpenAngleBracket => write!(f, "<"),
+            Self::CloseAngleBracket => write!(f, ">"),
+            Self::Dot => write!(f, "."),
         }
     }
 }

mps-interpreter/src/tokens/tokenizer.rs

@@ -234,6 +234,7 @@ impl ReaderStateMachine {
             | Self::EndToken {}
             | Self::EndComment {}
             | Self::EndStatement {}
+            | Self::EndOfFile {}
             | Self::Invalid {..} => match input_char {
                 '\\' => Self::Escaped { inside: '_' },
                 '/' => Self::Slash { out: input },
@@ -243,7 +244,7 @@ impl ReaderStateMachine {
                 ' ' => Self::EndToken {},
                 '\n' | '\r' | ';' => Self::EndStatement {},
                 '\0' => Self::EndOfFile {},
-                '(' | ')' | ',' => Self::SingleCharToken { out: input },
+                '(' | ')' | ',' | '=' | '<' | '>' | '.' => Self::SingleCharToken { out: input },
                 _ => Self::Regular { out: input },
             },
             Self::Escaped { inside } => match inside {
@@ -278,7 +279,7 @@ impl ReaderStateMachine {
                 '\n' | '\r' | '\0' => Self::EndComment {},
                 _ => Self::Comment { out: input },
             },
-            Self::EndOfFile {} => Self::EndOfFile {},
+            //Self::EndOfFile {} => Self::EndOfFile {}, // For REPL, the end of the file is not necessarily the end forever
         }
     }
 

mps-interpreter/tests/single_line.rs

@@ -8,7 +8,7 @@ use std::io::Cursor;
 fn parse_line() -> Result<(), ParseError> {
     let cursor = Cursor::new("sql(`SELECT * FROM songs;`)");
     let correct_tokens: Vec<MpsToken> = vec![
-        MpsToken::Sql,
+        MpsToken::Name("sql".into()),
         MpsToken::OpenBracket,
         MpsToken::Literal("SELECT * FROM songs;".into()),
         MpsToken::CloseBracket,
@@ -27,14 +27,16 @@ fn parse_line() -> Result<(), ParseError> {
     // validity tests
     assert_eq!(buf.len(), correct_tokens.len());
     for i in 0..buf.len() {
-        assert_eq!(buf[i], correct_tokens[i]);
+        assert_eq!(buf[i], correct_tokens[i], "Tokens at position {} do not match ()", i);
     }
 
     tokenizer.read_line(&mut buf)?; // this should immediately return
     Ok(())
 }
 
-fn execute_single_line(line: &str, should_be_emtpy: bool) -> Result<(), Box<dyn MpsLanguageError>> {
+#[inline(always)]
+fn execute_single_line(line: &str, should_be_emtpy: bool, should_complete: bool) -> Result<(), Box<dyn MpsLanguageError>> {
+    println!("--- Executing MPS code: '{}' ---", line);
     let cursor = Cursor::new(line);
 
     let tokenizer = MpsTokenizer::new(cursor);
@@ -45,34 +47,67 @@ fn execute_single_line(line: &str, should_be_emtpy: bool) -> Result<(), Box<dyn
         if let Ok(item) = result {
             count += 1;
             if count > 100 {
+                if should_complete {
                     continue;
+                } else {
+                    println!("Got 100 items, stopping to avoid infinite loop");
+                    break;
+                }
             } // no need to spam the rest of the songs
             println!("Got song `{}` (file: `{}`)", item.title, item.filename);
         } else {
-            println!("Got error while iterating (executing)");
+            println!("!!! Got error while iterating (executing) !!!");
             result?;
         }
     }
     if should_be_emtpy {
-        assert_eq!(count, 0);
+        assert_eq!(count, 0, "{} music items found while iterating over line which should be None", count);
     } else {
-        assert_ne!(count, 0); // database is populated
+        println!("Got {} items, execution complete (no songs were harmed in the making of this test)", count);
+        assert_ne!(count, 0, "0 music items found while iterating over line which should have Some results"); // assumption: database is populated
     }
     Ok(())
 }
 
 #[test]
 fn execute_sql_line() -> Result<(), Box<dyn MpsLanguageError>> {
-    execute_single_line("sql(`SELECT * FROM songs ORDER BY artist;`)", false)
+    execute_single_line("sql(`SELECT * FROM songs ORDER BY artist;`)", false, true)
 }
 
 #[test]
 fn execute_simple_sql_line() -> Result<(), Box<dyn MpsLanguageError>> {
-    execute_single_line("song(`lov`)", false)
+    execute_single_line("song(`lov`)", false, true)
 }
 
 #[test]
 fn execute_comment_line() -> Result<(), Box<dyn MpsLanguageError>> {
-    execute_single_line("// this is a comment", true)?;
-    execute_single_line("# this is a special comment", true)
+    execute_single_line("// this is a comment", true, true)?;
+    execute_single_line("# this is a special comment", true, true)
+}
+
+#[test]
+fn execute_repeat_line() -> Result<(), Box<dyn MpsLanguageError>> {
+    execute_single_line("repeat(song(`Christmas in L.A.`))", false, false)?;
+    execute_single_line("repeat(song(`Christmas in L.A.`), 4)", false, true)
+}
+
+#[test]
+fn execute_sql_init_line() -> Result<(), Box<dyn MpsLanguageError>> {
+    execute_single_line("sql_init(generate = no, folder = `/home/ngnius/Music`)", true, true)
+}
+
+#[test]
+fn execute_assign_line() -> Result<(), Box<dyn MpsLanguageError>> {
+    execute_single_line("let some_var = repeat(song(`Christmas in L.A.`))", true, true)?;
+    execute_single_line("let some_var2 = 1234", true, true)
+}
+
+#[test]
+fn execute_emptyfilter_line() -> Result<(), Box<dyn MpsLanguageError>> {
+    execute_single_line("song(`lov`).()", false, true)
+}
+
+#[test]
+fn execute_fieldfilter_line() -> Result<(), Box<dyn MpsLanguageError>> {
+    execute_single_line("song(`lov`).(year >= 2020)", false, true)
 }