diff --git a/mps-interpreter/Cargo.toml b/mps-interpreter/Cargo.toml index f5def9b..0af74ba 100644 --- a/mps-interpreter/Cargo.toml +++ b/mps-interpreter/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -rusqlite = { version = "0.26.1" } +rusqlite = { version = "0.26.3" } symphonia = { version = "0.4.0", optional = true, features = [ "aac", "flac", "mp3", "pcm", "vorbis", "isomp4", "ogg", "wav" ] } diff --git a/mps-interpreter/src/context.rs b/mps-interpreter/src/context.rs index 9b2f38b..5ecf834 100644 --- a/mps-interpreter/src/context.rs +++ b/mps-interpreter/src/context.rs @@ -1,14 +1,15 @@ -use std::fmt::{Debug, Display, Formatter, Error}; +use super::processing::database::{MpsDatabaseQuerier, MpsSQLiteExecutor}; +use std::fmt::{Debug, Display, Error, Formatter}; #[derive(Debug)] pub struct MpsContext { - pub sqlite_connection: Option, + pub database: Box, } impl Default for MpsContext { fn default() -> Self { Self { - sqlite_connection: None, // initialized by first SQL statement instead + database: Box::new(MpsSQLiteExecutor::default()), } } } @@ -16,7 +17,7 @@ impl Default for MpsContext { impl std::clone::Clone for MpsContext { fn clone(&self) -> Self { Self { - sqlite_connection: None, + database: Box::new(MpsSQLiteExecutor::default()), } } } diff --git a/mps-interpreter/src/interpretor.rs b/mps-interpreter/src/interpretor.rs index da9131e..2092aa9 100644 --- a/mps-interpreter/src/interpretor.rs +++ b/mps-interpreter/src/interpretor.rs @@ -1,14 +1,17 @@ -use std::iter::Iterator; use std::collections::VecDeque; -use std::path::Path; use std::fs::File; +use std::iter::Iterator; +use std::path::Path; -use super::MpsMusicItem; -use super::MpsContext; +use super::lang::{MpsLanguageDictionary, MpsLanguageError, MpsOp}; use super::tokens::MpsToken; -use super::lang::{MpsOp, MpsLanguageError, MpsLanguageDictionary}; +use super::MpsContext; +use super::MpsMusicItem; -pub struct MpsInterpretor where T: crate::tokens::MpsTokenReader { +pub struct MpsInterpretor +where + T: crate::tokens::MpsTokenReader, +{ tokenizer: T, buffer: VecDeque, current_stmt: Option>, @@ -22,7 +25,8 @@ pub fn interpretor(stream: R) -> MpsInterpretor MpsInterpretor - where T: crate::tokens::MpsTokenReader +where + T: crate::tokens::MpsTokenReader, { pub fn with_vocab(tokenizer: T, vocab: MpsLanguageDictionary) -> Self { Self { @@ -72,7 +76,8 @@ impl MpsInterpretor> { } impl Iterator for MpsInterpretor - where T: crate::tokens::MpsTokenReader +where + T: crate::tokens::MpsTokenReader, { type Item = Result>; @@ -84,21 +89,27 @@ impl Iterator for MpsInterpretor is_stmt_done = true; } match next_item { - Some(item) => Some(item.map_err(|e| box_error_with_ctx( - e, self.tokenizer.current_line() - ))), - None => None + Some(item) => { + Some(item.map_err(|e| box_error_with_ctx(e, self.tokenizer.current_line()))) + } + None => None, } } else { - if self.tokenizer.end_of_file() { return None; } + if self.tokenizer.end_of_file() { + return None; + } // build new statement - let token_result = self.tokenizer.next_statements(1, &mut self.buffer) + let token_result = self + .tokenizer + .next_statement(&mut self.buffer) .map_err(|e| box_error_with_ctx(e, self.tokenizer.current_line())); match token_result { - Ok(_) => {}, - Err(x) => return Some(Err(x)) + Ok(_) => {} + Err(x) => return Some(Err(x)), + } + if self.tokenizer.end_of_file() && self.buffer.len() == 0 { + return None; } - if self.tokenizer.end_of_file() && self.buffer.len() == 0 { return None; } let stmt = self.vocabulary.try_build_statement(&mut self.buffer); match stmt { Ok(mut stmt) => { @@ -109,17 +120,15 @@ impl Iterator for MpsInterpretor is_stmt_done = true; } match next_item { - Some(item) => Some(item.map_err(|e| box_error_with_ctx( - e, - self.tokenizer.current_line() - ))), - None => None + Some(item) => Some( + item.map_err(|e| box_error_with_ctx(e, self.tokenizer.current_line())), + ), + None => None, } - }, - Err(e) => Some(Err(e).map_err(|e| box_error_with_ctx( - e, - self.tokenizer.current_line() - ))) + } + Err(e) => { + Some(Err(e).map_err(|e| box_error_with_ctx(e, self.tokenizer.current_line()))) + } } }; if is_stmt_done { @@ -129,12 +138,17 @@ impl Iterator for MpsInterpretor } } -fn box_error_with_ctx(mut error: E, line: usize) -> Box { +fn box_error_with_ctx( + mut error: E, + line: usize, +) -> Box { error.set_line(line); Box::new(error) as Box } pub(crate) fn standard_vocab(vocabulary: &mut MpsLanguageDictionary) { vocabulary - .add(crate::lang::vocabulary::SqlStatementFactory); + .add(crate::lang::vocabulary::SqlStatementFactory) + .add(crate::lang::vocabulary::SimpleSqlStatementFactory) + .add(crate::lang::vocabulary::CommentStatementFactory); } diff --git a/mps-interpreter/src/lang/comment.rs b/mps-interpreter/src/lang/comment.rs new file mode 100644 index 0000000..58d8481 --- /dev/null +++ b/mps-interpreter/src/lang/comment.rs @@ -0,0 +1,91 @@ +use std::collections::VecDeque; +use std::fmt::{Debug, Display, Error, Formatter}; +use std::iter::Iterator; + +use crate::MpsContext; +use crate::MpsMusicItem; +use crate::tokens::MpsToken; + +use super::{RuntimeError, SyntaxError}; +use super::{MpsOp, SimpleMpsOpFactory, MpsOpFactory, BoxedMpsOpFactory}; +use super::MpsLanguageDictionary; +use super::utility::assert_token; + +#[derive(Debug, Clone)] +pub struct CommentStatement { + comment: String, + context: Option +} + +impl CommentStatement { + /*fn comment_text(&self) -> String { + let mut clone = self.comment.clone(); + if clone.starts_with("#") { + clone.replace_range(..1, ""); // remove "#" + } else { + clone.replace_range(..2, ""); // remove "//" + } + clone + }*/ +} + +impl Display for CommentStatement { + fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { + write!(f, "{}", self.comment) + } +} + +impl Iterator for CommentStatement { + type Item = Result; + + fn next(&mut self) -> Option { + None + } +} + +impl MpsOp for CommentStatement { + fn enter(&mut self, ctx: MpsContext) { + self.context = Some(ctx) + } + + fn escape(&mut self) -> MpsContext { + self.context.take().unwrap() + } +} + +pub struct CommentStatementFactory; + +impl SimpleMpsOpFactory for CommentStatementFactory { + fn is_op_simple(&self, tokens: &VecDeque) -> bool { + tokens.len() == 1 + && tokens[0].is_comment() + } + + fn build_op_simple( + &self, + tokens: &mut VecDeque, + ) -> Result { + let comment = assert_token(|t| match t { + MpsToken::Comment(c) => Some(c), + _ => None + }, MpsToken::Comment("comment".into()), tokens)?; + Ok(CommentStatement { + comment: comment, + context: None + }) + } +} + +impl BoxedMpsOpFactory for CommentStatementFactory { + fn build_op_boxed( + &self, + tokens: &mut VecDeque, + dict: &MpsLanguageDictionary, + ) -> Result, SyntaxError> { + self.build_box(tokens, dict) + } + + fn is_op_boxed(&self, tokens: &VecDeque) -> bool { + self.is_op(tokens) + } +} diff --git a/mps-interpreter/src/lang/db_items.rs b/mps-interpreter/src/lang/db_items.rs index c24b602..ed2fa64 100644 --- a/mps-interpreter/src/lang/db_items.rs +++ b/mps-interpreter/src/lang/db_items.rs @@ -13,7 +13,9 @@ pub fn generate_default_db() -> rusqlite::Result { let conn = rusqlite::Connection::open(DEFAULT_SQLITE_FILEPATH)?; // skip db building if SQLite file already exists // TODO do a more exhaustive db check to make sure it's actually the correct file - if db_exists {return Ok(conn);} + if db_exists { + return Ok(conn); + } // build db tables conn.execute_batch( "BEGIN; @@ -50,7 +52,7 @@ pub fn generate_default_db() -> rusqlite::Result { genre_id INTEGER NOT NULL PRIMARY KEY, title TEXT ); - COMMIT;" + COMMIT;", )?; // generate data and store in db #[cfg(feature = "music_library")] @@ -67,7 +69,7 @@ pub fn generate_default_db() -> rusqlite::Result { filename, metadata, genre - ) VALUES (?, ?, ?, ?, ?, ?, ?)" + ) VALUES (?, ?, ?, ?, ?, ?, ?)", )?; for song in lib.all_songs() { song_insert.execute(song.to_params().as_slice())?; @@ -81,7 +83,7 @@ pub fn generate_default_db() -> rusqlite::Result { disc, duration, date - ) VALUES (?, ?, ?, ?, ?, ?)" + ) VALUES (?, ?, ?, ?, ?, ?)", )?; for meta in lib.all_metadata() { metadata_insert.execute(meta.to_params().as_slice())?; @@ -92,7 +94,7 @@ pub fn generate_default_db() -> rusqlite::Result { artist_id, name, genre - ) VALUES (?, ?, ?)" + ) VALUES (?, ?, ?)", )?; for artist in lib.all_artists() { artist_insert.execute(artist.to_params().as_slice())?; @@ -105,7 +107,7 @@ pub fn generate_default_db() -> rusqlite::Result { metadata, artist, genre - ) VALUES (?, ?, ?, ?, ?)" + ) VALUES (?, ?, ?, ?, ?)", )?; for album in lib.all_albums() { album_insert.execute(album.to_params().as_slice())?; @@ -115,13 +117,13 @@ pub fn generate_default_db() -> rusqlite::Result { "INSERT OR REPLACE INTO genres ( genre_id, title - ) VALUES (?, ?)" + ) VALUES (?, ?)", )?; for genre in lib.all_genres() { genre_insert.execute(genre.to_params().as_slice())?; } - }, - Err(e) => println!("Unable to load music from {}: {}", music_path.display(), e) + } + Err(e) => println!("Unable to load music from {}: {}", music_path.display(), e), } } Ok(conn) @@ -140,7 +142,7 @@ pub struct DbMusicItem { impl DatabaseObj for DbMusicItem { fn map_row(row: &rusqlite::Row) -> rusqlite::Result { - Ok(Self{ + Ok(Self { song_id: row.get(0)?, title: row.get(1)?, artist: row.get(2)?, @@ -163,7 +165,9 @@ impl DatabaseObj for DbMusicItem { ] } - fn id(&self) -> u64 {self.song_id} + fn id(&self) -> u64 { + self.song_id + } } #[derive(Clone, Debug)] @@ -173,12 +177,12 @@ pub struct DbMetaItem { pub track: u64, pub disc: u64, pub duration: u64, // seconds - pub date: u64, // year + pub date: u64, // year } impl DatabaseObj for DbMetaItem { fn map_row(row: &rusqlite::Row) -> rusqlite::Result { - Ok(Self{ + Ok(Self { meta_id: row.get(0)?, plays: row.get(1)?, track: row.get(2)?, @@ -199,7 +203,9 @@ impl DatabaseObj for DbMetaItem { ] } - fn id(&self) -> u64 {self.meta_id} + fn id(&self) -> u64 { + self.meta_id + } } #[derive(Clone, Debug)] @@ -211,7 +217,7 @@ pub struct DbArtistItem { impl DatabaseObj for DbArtistItem { fn map_row(row: &rusqlite::Row) -> rusqlite::Result { - Ok(Self{ + Ok(Self { artist_id: row.get(0)?, name: row.get(1)?, genre: row.get(2)?, @@ -219,14 +225,12 @@ impl DatabaseObj for DbArtistItem { } fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> { - vec![ - &self.artist_id, - &self.name, - &self.genre, - ] + vec![&self.artist_id, &self.name, &self.genre] } - fn id(&self) -> u64 {self.artist_id} + fn id(&self) -> u64 { + self.artist_id + } } #[derive(Clone, Debug)] @@ -240,7 +244,7 @@ pub struct DbAlbumItem { impl DatabaseObj for DbAlbumItem { fn map_row(row: &rusqlite::Row) -> rusqlite::Result { - Ok(Self{ + Ok(Self { album_id: row.get(0)?, title: row.get(1)?, metadata: row.get(2)?, @@ -259,7 +263,9 @@ impl DatabaseObj for DbAlbumItem { ] } - fn id(&self) -> u64 {self.album_id} + fn id(&self) -> u64 { + self.album_id + } } #[derive(Clone, Debug)] @@ -270,18 +276,17 @@ pub struct DbGenreItem { impl DatabaseObj for DbGenreItem { fn map_row(row: &rusqlite::Row) -> rusqlite::Result { - Ok(Self{ + Ok(Self { genre_id: row.get(0)?, title: row.get(1)?, }) } fn to_params(&self) -> Vec<&dyn rusqlite::ToSql> { - vec![ - &self.genre_id, - &self.title, - ] + vec![&self.genre_id, &self.title] } - fn id(&self) -> u64 {self.genre_id} + fn id(&self) -> u64 { + self.genre_id + } } diff --git a/mps-interpreter/src/lang/dictionary.rs b/mps-interpreter/src/lang/dictionary.rs index 54314f1..40f5941 100644 --- a/mps-interpreter/src/lang/dictionary.rs +++ b/mps-interpreter/src/lang/dictionary.rs @@ -1,28 +1,32 @@ use std::collections::VecDeque; -use crate::tokens::MpsToken; -use super::{BoxedMpsOpFactory, MpsOp}; use super::SyntaxError; +use super::{BoxedMpsOpFactory, MpsOp}; +use crate::tokens::MpsToken; pub struct MpsLanguageDictionary { - vocabulary: Vec> + vocabulary: Vec>, } impl MpsLanguageDictionary { pub fn add(&mut self, factory: T) -> &mut Self { - self.vocabulary.push(Box::new(factory) as Box); + self.vocabulary + .push(Box::new(factory) as Box); self } - pub fn try_build_statement(&self, tokens: &mut VecDeque) -> Result, SyntaxError> { + pub fn try_build_statement( + &self, + tokens: &mut VecDeque, + ) -> Result, SyntaxError> { for factory in &self.vocabulary { if factory.is_op_boxed(tokens) { - return factory.build_op_boxed(tokens); + return factory.build_op_boxed(tokens, self); } } Err(SyntaxError { line: 0, - token: tokens.pop_front().unwrap() + token: tokens.pop_front().unwrap(), }) } diff --git a/mps-interpreter/src/lang/error.rs b/mps-interpreter/src/lang/error.rs index 474037f..3911d38 100644 --- a/mps-interpreter/src/lang/error.rs +++ b/mps-interpreter/src/lang/error.rs @@ -1,7 +1,7 @@ -use std::fmt::{Debug, Display, Formatter, Error}; +use std::fmt::{Debug, Display, Error, Formatter}; -use crate::tokens::MpsToken; use super::MpsOp; +use crate::tokens::MpsToken; #[derive(Debug)] pub struct SyntaxError { @@ -11,12 +11,18 @@ pub struct SyntaxError { impl Display for SyntaxError { fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { - write!(f, "SyntaxError (line {}): Unexpected {}", &self.line, &self.token) + write!( + f, + "SyntaxError (line {}): Unexpected {}", + &self.line, &self.token + ) } } impl MpsLanguageError for SyntaxError { - fn set_line(&mut self, line: usize) {self.line = line} + fn set_line(&mut self, line: usize) { + self.line = line + } } #[derive(Debug)] @@ -33,7 +39,9 @@ impl Display for RuntimeError { } impl MpsLanguageError for RuntimeError { - fn set_line(&mut self, line: usize) {self.line = line} + fn set_line(&mut self, line: usize) { + self.line = line + } } pub trait MpsLanguageError: Display + Debug { diff --git a/mps-interpreter/src/lang/mod.rs b/mps-interpreter/src/lang/mod.rs index ec88b0d..55d0894 100644 --- a/mps-interpreter/src/lang/mod.rs +++ b/mps-interpreter/src/lang/mod.rs @@ -1,22 +1,29 @@ +mod comment; mod db_items; mod dictionary; mod error; mod operation; mod sql_query; +mod sql_simple_query; //mod statement; pub(crate) mod utility; pub use dictionary::MpsLanguageDictionary; -pub use error::{SyntaxError, RuntimeError, MpsLanguageError}; -pub use operation::{MpsOp, MpsOpFactory, BoxedMpsOpFactory}; +pub use error::{MpsLanguageError, RuntimeError, SyntaxError}; +pub use operation::{BoxedMpsOpFactory, MpsOp, MpsOpFactory, SimpleMpsOpFactory}; //pub(crate) use statement::MpsStatement; pub mod vocabulary { pub use super::sql_query::{SqlStatement, SqlStatementFactory}; + pub use super::sql_simple_query::{SimpleSqlStatement, SimpleSqlStatementFactory}; + pub use super::comment::{CommentStatement, CommentStatementFactory}; } pub mod db { - pub use super::db_items::{DEFAULT_SQLITE_FILEPATH, generate_default_db, DatabaseObj, DbMusicItem, DbAlbumItem, DbArtistItem, DbMetaItem, DbGenreItem}; + pub use super::db_items::{ + generate_default_db, DatabaseObj, DbAlbumItem, DbArtistItem, DbGenreItem, DbMetaItem, + DbMusicItem, DEFAULT_SQLITE_FILEPATH, + }; } #[cfg(test)] diff --git a/mps-interpreter/src/lang/operation.rs b/mps-interpreter/src/lang/operation.rs index f6cb8f8..8e9a459 100644 --- a/mps-interpreter/src/lang/operation.rs +++ b/mps-interpreter/src/lang/operation.rs @@ -1,30 +1,66 @@ -use std::iter::Iterator; use std::collections::VecDeque; use std::fmt::{Debug, Display}; +use std::iter::Iterator; -use crate::MpsMusicItem; -use crate::MpsContext; +use super::MpsLanguageDictionary; +use super::{RuntimeError, SyntaxError}; use crate::tokens::MpsToken; -use super::{SyntaxError, RuntimeError}; +use crate::MpsContext; +use crate::MpsMusicItem; + +pub trait SimpleMpsOpFactory { + fn is_op_simple(&self, tokens: &VecDeque) -> bool; + + fn build_op_simple( + &self, + tokens: &mut VecDeque, + ) -> Result; +} + +impl + 'static> MpsOpFactory for X { + fn is_op(&self, tokens: &VecDeque) -> bool { + self.is_op_simple(tokens) + } + + fn build_op( + &self, + tokens: &mut VecDeque, + _dict: &MpsLanguageDictionary, + ) -> Result { + self.build_op_simple(tokens) + } +} pub trait MpsOpFactory { fn is_op(&self, tokens: &VecDeque) -> bool; - fn build_op(&self, tokens: &mut VecDeque) -> Result; + fn build_op( + &self, + tokens: &mut VecDeque, + dict: &MpsLanguageDictionary, + ) -> Result; #[inline] - fn build_box(&self, tokens: &mut VecDeque) -> Result, SyntaxError> { - Ok(Box::new(self.build_op(tokens)?)) + fn build_box( + &self, + tokens: &mut VecDeque, + dict: &MpsLanguageDictionary, + ) -> Result, SyntaxError> { + Ok(Box::new(self.build_op(tokens, dict)?)) } } pub trait BoxedMpsOpFactory { - fn build_op_boxed(&self, tokens: &mut VecDeque) -> Result, SyntaxError>; + fn build_op_boxed( + &self, + tokens: &mut VecDeque, + dict: &MpsLanguageDictionary, + ) -> Result, SyntaxError>; fn is_op_boxed(&self, tokens: &VecDeque) -> bool; } -pub trait MpsOp: Iterator> + Debug + Display { +pub trait MpsOp: Iterator> + Debug + Display { fn enter(&mut self, ctx: MpsContext); fn escape(&mut self) -> MpsContext; diff --git a/mps-interpreter/src/lang/sql_query.rs b/mps-interpreter/src/lang/sql_query.rs index 727b21d..8ddc54f 100644 --- a/mps-interpreter/src/lang/sql_query.rs +++ b/mps-interpreter/src/lang/sql_query.rs @@ -1,42 +1,43 @@ -use std::iter::Iterator; use std::collections::VecDeque; -use std::fmt::{Debug, Display, Formatter, Error}; +use std::fmt::{Debug, Display, Error, Formatter}; +use std::iter::Iterator; +use super::utility::{assert_token, assert_token_raw}; +use super::{BoxedMpsOpFactory, MpsLanguageDictionary, MpsOp, MpsOpFactory}; +use super::{RuntimeError, SyntaxError}; +use crate::tokens::MpsToken; use crate::MpsContext; use crate::MpsMusicItem; -use crate::tokens::MpsToken; -use super::{MpsOp, MpsOpFactory, BoxedMpsOpFactory}; -use super::{SyntaxError, RuntimeError}; -use super::utility::{assert_token, assert_token_raw}; -use super::db::*; +//use super::db::*; #[derive(Debug)] pub struct SqlStatement { query: String, context: Option, - rows: Option>>, + rows: Option>>, current: usize, } impl SqlStatement { - fn map_item(&mut self, increment: bool) -> Option> { + fn get_item(&mut self, increment: bool) -> Option> { if let Some(rows) = &self.rows { if increment { if self.current == rows.len() { - return None + return None; } self.current += 1; } if self.current >= rows.len() { None } else { + //Some(rows[self.current].clone()) match &rows[self.current] { Ok(item) => Some(Ok(item.clone())), Err(e) => Some(Err(RuntimeError { - line: 0, + line: e.line, op: Box::new(self.clone()), - msg: format!("SQL music item mapping error: {}", e).into(), - })) + msg: e.msg.clone(), + })), } } } else { @@ -46,12 +47,11 @@ impl SqlStatement { msg: format!("Context error: rows is None").into(), })) } - } } impl MpsOp for SqlStatement { - fn enter(&mut self, ctx: MpsContext){ + fn enter(&mut self, ctx: MpsContext) { self.context = Some(ctx) } @@ -64,8 +64,8 @@ impl std::clone::Clone for SqlStatement { fn clone(&self) -> Self { Self { query: self.query.clone(), - context: self.context.clone(), - rows: None, // TODO use different Result type so this is cloneable + context: None, // unecessary to include in clone (not used for displaying) + rows: None, // unecessary to include current: self.current, } } @@ -77,34 +77,21 @@ impl Iterator for SqlStatement { fn next(&mut self) -> Option { if self.rows.is_some() { // query has executed, return another result - self.map_item(true) + self.get_item(true) } else { + let self_clone = self.clone(); let ctx = self.context.as_mut().unwrap(); // query has not been executed yet - if let None = ctx.sqlite_connection { - // connection needs to be created - match generate_default_db() { - Ok(conn) => ctx.sqlite_connection = Some(conn), - Err(e) => return Some(Err(RuntimeError{ - line: 0, - op: Box::new(self.clone()), - msg: format!("SQL connection error: {}", e).into() - })) + match ctx + .database + .raw(&self.query, &mut move || Box::new(self_clone.clone())) + { + Err(e) => return Some(Err(e)), + Ok(rows) => { + self.rows = Some(rows); + self.get_item(false) } } - let conn = ctx.sqlite_connection.as_mut().unwrap(); - // execute query - match perform_query(conn, &self.query) { - Ok(items) => { - self.rows = Some(items); - self.map_item(false) - } - Err(e) => Some(Err(RuntimeError{ - line: 0, - op: Box::new(self.clone()), - msg: e - })) - } } } } @@ -120,20 +107,30 @@ pub struct SqlStatementFactory; impl MpsOpFactory for SqlStatementFactory { #[inline] fn is_op(&self, tokens: &VecDeque) -> bool { - tokens.len() > 3 && tokens[0].is_sql() + tokens.len() > 3 + && tokens[0].is_sql() + && tokens[1].is_open_bracket() + && tokens[2].is_literal() + && tokens[3].is_close_bracket() } #[inline] - fn build_op(&self, tokens: &mut VecDeque) -> Result { + fn build_op( + &self, + tokens: &mut VecDeque, + _dict: &MpsLanguageDictionary, + ) -> Result { // sql ( `some query` ) assert_token_raw(MpsToken::Sql, tokens)?; assert_token_raw(MpsToken::OpenBracket, tokens)?; - let literal = assert_token(|t| { - match t { + let literal = assert_token( + |t| match t { MpsToken::Literal(query) => Some(query), - _ => None - } - }, MpsToken::Literal("".into()), tokens)?; + _ => None, + }, + MpsToken::Literal("".into()), + tokens, + )?; assert_token_raw(MpsToken::CloseBracket, tokens)?; Ok(SqlStatement { query: literal, @@ -145,22 +142,15 @@ impl MpsOpFactory for SqlStatementFactory { } impl BoxedMpsOpFactory for SqlStatementFactory { - fn build_op_boxed(&self, tokens: &mut VecDeque) -> Result, SyntaxError> { - self.build_box(tokens) + fn build_op_boxed( + &self, + tokens: &mut VecDeque, + dict: &MpsLanguageDictionary, + ) -> Result, SyntaxError> { + self.build_box(tokens, dict) } fn is_op_boxed(&self, tokens: &VecDeque) -> bool { self.is_op(tokens) } } - -fn perform_query( - conn: &mut rusqlite::Connection, - query: &str -) -> Result>, String> { - let mut stmt = conn.prepare(query) - .map_err(|e| format!("SQLite query error: {}", e))?; - let iter = stmt.query_map([], MpsMusicItem::map_row) - .map_err(|e| format!("SQLite item mapping error: {}", e))?; - Ok(iter.collect()) -} diff --git a/mps-interpreter/src/lang/sql_simple_query.rs b/mps-interpreter/src/lang/sql_simple_query.rs new file mode 100644 index 0000000..1a85b0b --- /dev/null +++ b/mps-interpreter/src/lang/sql_simple_query.rs @@ -0,0 +1,233 @@ +use std::collections::VecDeque; +use std::fmt::{Debug, Display, Error, Formatter}; +use std::iter::Iterator; + +use super::utility::{assert_token, assert_token_raw}; +use super::{BoxedMpsOpFactory, MpsLanguageDictionary, MpsOp, MpsOpFactory}; +use super::{RuntimeError, SyntaxError}; +use crate::tokens::MpsToken; +use crate::MpsContext; +use crate::MpsMusicItem; + +#[derive(Debug, Clone)] +enum QueryMode { + Artist, + Album, + Song, + Genre, +} + +impl QueryMode { + fn from_name(name: String) -> Result { + match &name as &str { + "artist" => Ok(QueryMode::Artist), + "album" => Ok(QueryMode::Album), + "song" => Ok(QueryMode::Song), + "genre" => Ok(QueryMode::Genre), + _ => Err(SyntaxError { + line: 0, + token: Self::tokenify(name), + }), + } + } + + fn is_valid_name(name: &str) -> bool { + match name { + "artist" | "album" | "song" | "genre" => true, + _ => false, + } + } + + #[inline] + fn tokenify(name: String) -> MpsToken { + MpsToken::Name(name) + } + + #[inline] + fn tokenify_self(&self) -> MpsToken { + MpsToken::Name( + match self { + Self::Artist => "artist", + Self::Album => "album", + Self::Song => "song", + Self::Genre => "genre", + } + .into(), + ) + } +} + +#[derive(Debug)] +pub struct SimpleSqlStatement { + query: String, + mode: QueryMode, + context: Option, + rows: Option>>, + current: usize, +} + +impl SimpleSqlStatement { + fn get_item(&mut self, increment: bool) -> Option> { + if let Some(rows) = &self.rows { + if increment { + if self.current == rows.len() { + return None; + } + self.current += 1; + } + if self.current >= rows.len() { + None + } else { + //Some(rows[self.current].clone()) + match &rows[self.current] { + Ok(item) => Some(Ok(item.clone())), + Err(e) => Some(Err(RuntimeError { + line: e.line, + op: Box::new(self.clone()), + msg: e.msg.clone(), + })), + } + } + } else { + Some(Err(RuntimeError { + line: 0, + op: Box::new(self.clone()), + msg: format!("Context error: rows is None").into(), + })) + } + } +} + +impl MpsOp for SimpleSqlStatement { + fn enter(&mut self, ctx: MpsContext) { + self.context = Some(ctx) + } + + fn escape(&mut self) -> MpsContext { + self.context.take().unwrap() + } +} + +impl std::clone::Clone for SimpleSqlStatement { + fn clone(&self) -> Self { + Self { + query: self.query.clone(), + mode: self.mode.clone(), + context: None, // unecessary to include in clone (not used for displaying) + rows: None, // unecessary to include + current: self.current, + } + } +} + +impl Iterator for SimpleSqlStatement { + type Item = Result; + + fn next(&mut self) -> Option { + if self.rows.is_some() { + // query has executed, return another result + self.get_item(true) + } else { + let self_clone = self.clone(); + let ctx = self.context.as_mut().unwrap(); + // query has not been executed yet + let query_result = match self.mode { + QueryMode::Artist => ctx + .database + .artist_like(&self.query, &mut move || Box::new(self_clone.clone())), + QueryMode::Album => ctx + .database + .album_like(&self.query, &mut move || Box::new(self_clone.clone())), + QueryMode::Song => ctx + .database + .song_like(&self.query, &mut move || Box::new(self_clone.clone())), + QueryMode::Genre => ctx + .database + .genre_like(&self.query, &mut move || Box::new(self_clone.clone())), + }; + match query_result { + Err(e) => return Some(Err(e)), + Ok(rows) => { + self.rows = Some(rows); + self.get_item(false) + } + } + } + } +} + +impl Display for SimpleSqlStatement { + fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { + write!(f, "{}(`{}`)", self.mode.tokenify_self(), &self.query) + } +} + +pub struct SimpleSqlStatementFactory; + +impl MpsOpFactory for SimpleSqlStatementFactory { + #[inline] + fn is_op(&self, tokens: &VecDeque) -> bool { + tokens.len() > 3 + && match &tokens[0] { + MpsToken::Name(name) => QueryMode::is_valid_name(name), + _ => false, + } + && tokens[1].is_open_bracket() + && tokens[2].is_literal() + && tokens[3].is_close_bracket() + } + + #[inline] + fn build_op( + &self, + tokens: &mut VecDeque, + _dict: &MpsLanguageDictionary, + ) -> Result { + // artist|album|song|genre ( `like` ) + let mode_name = assert_token( + |t| match t { + MpsToken::Name(name) => { + if QueryMode::is_valid_name(&name) { + Some(name) + } else { + None + } + } + _ => None, + }, + MpsToken::Name("artist|album|song|genre".into()), + tokens, + )?; + assert_token_raw(MpsToken::OpenBracket, tokens)?; + let literal = assert_token( + |t| match t { + MpsToken::Literal(query) => Some(query), + _ => None, + }, + MpsToken::Literal("literal".into()), + tokens, + )?; + assert_token_raw(MpsToken::CloseBracket, tokens)?; + Ok(SimpleSqlStatement { + query: literal, + mode: QueryMode::from_name(mode_name)?, + context: None, + current: 0, + rows: None, + }) + } +} + +impl BoxedMpsOpFactory for SimpleSqlStatementFactory { + fn build_op_boxed( + &self, + tokens: &mut VecDeque, + dict: &MpsLanguageDictionary, + ) -> Result, SyntaxError> { + self.build_box(tokens, dict) + } + + fn is_op_boxed(&self, tokens: &VecDeque) -> bool { + self.is_op(tokens) + } +} diff --git a/mps-interpreter/src/lib.rs b/mps-interpreter/src/lib.rs index dbd65ad..66e0d79 100644 --- a/mps-interpreter/src/lib.rs +++ b/mps-interpreter/src/lib.rs @@ -1,16 +1,17 @@ mod context; mod interpretor; -mod runner; -mod music_item; pub mod lang; #[cfg(feature = "music_library")] pub mod music; +mod music_item; +pub mod processing; +mod runner; pub mod tokens; pub use context::MpsContext; -pub use interpretor::{MpsInterpretor, interpretor}; -pub use runner::MpsRunner; +pub use interpretor::{interpretor, MpsInterpretor}; pub use music_item::MpsMusicItem; +pub use runner::MpsRunner; #[cfg(test)] mod tests {} diff --git a/mps-interpreter/src/music/library.rs b/mps-interpreter/src/music/library.rs index ca427c9..64d979e 100644 --- a/mps-interpreter/src/music/library.rs +++ b/mps-interpreter/src/music/library.rs @@ -4,8 +4,8 @@ use std::path::Path; use symphonia::core::io::MediaSourceStream; use symphonia::core::probe::Hint; -use crate::lang::db::*; use super::tag::Tags; +use crate::lang::db::*; #[derive(Clone, Default)] pub struct MpsLibrary { @@ -55,7 +55,7 @@ impl MpsLibrary { let path = path.as_ref(); if path.is_dir() && depth != 0 { for entry in path.read_dir()? { - self.read_path(entry?.path(), depth-1)?; + self.read_path(entry?.path(), depth - 1)?; } } else if path.is_file() { self.read_file(path)?; @@ -72,7 +72,7 @@ impl MpsLibrary { &Hint::new(), mss, &Default::default(), - &Default::default() + &Default::default(), ); // process audio file, ignoring any processing errors (skip file on error) if let Ok(mut probed) = probed { @@ -99,27 +99,32 @@ impl MpsLibrary { /// generate data structures and links fn generate_entries(&mut self, tags: &Tags) { - if tags.len() == 0 { return; } // probably not a valid song, let's skip it + if tags.len() == 0 { + return; + } // probably not a valid song, let's skip it let song_id = self.songs.len() as u64; // guaranteed to be created let meta_id = self.metadata.len() as u64; // guaranteed to be created self.metadata.insert(meta_id, tags.meta(meta_id)); // definitely necessary - // genre has no links to others, so find that first + // genre has no links to others, so find that first let mut genre = tags.genre(0); genre.genre_id = Self::find_or_gen_id(&self.genres, &genre.title); if genre.genre_id == self.genres.len() as u64 { - self.genres.insert(Self::sanitise_key(&genre.title), genre.clone()); + self.genres + .insert(Self::sanitise_key(&genre.title), genre.clone()); } // artist only links to genre, so that can be next let mut artist = tags.artist(0, genre.genre_id); artist.artist_id = Self::find_or_gen_id(&self.artists, &artist.name); if artist.artist_id == self.artists.len() as u64 { - self.artists.insert(Self::sanitise_key(&artist.name), artist.clone()); + self.artists + .insert(Self::sanitise_key(&artist.name), artist.clone()); } // same with album artist let mut album_artist = tags.album_artist(0, genre.genre_id); album_artist.artist_id = Self::find_or_gen_id(&self.artists, &album_artist.name); if album_artist.artist_id == self.artists.len() as u64 { - self.artists.insert(Self::sanitise_key(&album_artist.name), album_artist.clone()); + self.artists + .insert(Self::sanitise_key(&album_artist.name), album_artist.clone()); } // album now has all links ready let mut album = tags.album(0, 0, album_artist.artist_id, genre.genre_id); @@ -127,12 +132,22 @@ impl MpsLibrary { if album.album_id == self.albums.len() as u64 { let album_meta = tags.album_meta(self.metadata.len() as u64); album.metadata = album_meta.meta_id; - self.albums.insert(Self::sanitise_key(&album.title), album.clone()); + self.albums + .insert(Self::sanitise_key(&album.title), album.clone()); self.metadata.insert(album_meta.meta_id, album_meta); } //let meta_album_id = self.metadata.len() as u64; //let album = tags.album(album_id, meta_album_id); - self.songs.insert(song_id, tags.song(song_id, artist.artist_id, Some(album.album_id), meta_id, genre.genre_id)); + self.songs.insert( + song_id, + tags.song( + song_id, + artist.artist_id, + Some(album.album_id), + meta_id, + genre.genre_id, + ), + ); } #[inline] diff --git a/mps-interpreter/src/music/tag.rs b/mps-interpreter/src/music/tag.rs index 8da66b0..25e88bd 100644 --- a/mps-interpreter/src/music/tag.rs +++ b/mps-interpreter/src/music/tag.rs @@ -24,20 +24,37 @@ impl Tags { } } - pub fn len(&self) -> usize {self.data.len()} + pub fn len(&self) -> usize { + self.data.len() + } - pub fn song(&self, id: u64, artist_id: u64, album_id: Option, meta_id: u64, genre_id: u64) -> DbMusicItem { + pub fn song( + &self, + id: u64, + artist_id: u64, + album_id: Option, + meta_id: u64, + genre_id: u64, + ) -> DbMusicItem { let default_title = || { - let extension = self.filename.extension().and_then(|ext| ext.to_str()).unwrap_or(""); - self.filename.file_name() + let extension = self + .filename + .extension() + .and_then(|ext| ext.to_str()) + .unwrap_or(""); + self.filename + .file_name() .and_then(|file| file.to_str()) .and_then(|file| Some(file.replacen(extension, "", 1))) .unwrap_or("Unknown Title".into()) }; DbMusicItem { song_id: id, - title: self.data.get("TITLE") - .unwrap_or(&TagType::Unknown).str() + title: self + .data + .get("TITLE") + .unwrap_or(&TagType::Unknown) + .str() .and_then(|s| Some(s.to_string())) .unwrap_or_else(default_title), artist: artist_id, @@ -51,18 +68,49 @@ impl Tags { pub fn meta(&self, id: u64) -> DbMetaItem { DbMetaItem { meta_id: id, - plays: self.data.get("PLAYS").unwrap_or(&TagType::Unknown).uint().unwrap_or(0), - track: self.data.get("TRACKNUMBER").unwrap_or(&TagType::Unknown).uint().unwrap_or(id), - disc: self.data.get("DISCNUMBER").unwrap_or(&TagType::Unknown).uint().unwrap_or(1), - duration: self.data.get("DURATION").unwrap_or(&TagType::Unknown).uint().unwrap_or(0), - date: self.data.get("DATE").unwrap_or(&TagType::Unknown).uint().unwrap_or(0), + plays: self + .data + .get("PLAYS") + .unwrap_or(&TagType::Unknown) + .uint() + .unwrap_or(0), + track: self + .data + .get("TRACKNUMBER") + .unwrap_or(&TagType::Unknown) + .uint() + .unwrap_or(id), + disc: self + .data + .get("DISCNUMBER") + .unwrap_or(&TagType::Unknown) + .uint() + .unwrap_or(1), + duration: self + .data + .get("DURATION") + .unwrap_or(&TagType::Unknown) + .uint() + .unwrap_or(0), + date: self + .data + .get("DATE") + .unwrap_or(&TagType::Unknown) + .uint() + .unwrap_or(0), } } pub fn artist(&self, id: u64, genre_id: u64) -> DbArtistItem { DbArtistItem { artist_id: id, - name: self.data.get("ARTIST").unwrap_or(&TagType::Unknown).str().unwrap_or("Unknown Artist").into(), + name: self + .data + .get("ARTIST") + .unwrap_or(&TagType::Unknown) + .str() + .unwrap_or("Unknown Artist") + .into(), genre: genre_id, } } @@ -70,7 +118,13 @@ impl Tags { pub fn album_artist(&self, id: u64, genre_id: u64) -> DbArtistItem { DbArtistItem { artist_id: id, - name: self.data.get("ALBUMARTIST").unwrap_or(&TagType::Unknown).str().unwrap_or("Unknown Artist").into(), + name: self + .data + .get("ALBUMARTIST") + .unwrap_or(&TagType::Unknown) + .str() + .unwrap_or("Unknown Artist") + .into(), genre: genre_id, } } @@ -78,7 +132,13 @@ impl Tags { pub fn album(&self, id: u64, meta_id: u64, artist_id: u64, genre_id: u64) -> DbAlbumItem { DbAlbumItem { album_id: id, - title: self.data.get("ALBUM").unwrap_or(&TagType::Unknown).str().unwrap_or("Unknown Album").into(), + title: self + .data + .get("ALBUM") + .unwrap_or(&TagType::Unknown) + .str() + .unwrap_or("Unknown Album") + .into(), metadata: meta_id, artist: artist_id, genre: genre_id, @@ -88,18 +148,44 @@ impl Tags { pub fn album_meta(&self, id: u64) -> DbMetaItem { DbMetaItem { meta_id: id, - plays: self.data.get("PLAYS").unwrap_or(&TagType::Unknown).uint().unwrap_or(0), - track: self.data.get("TRACKTOTAL").unwrap_or(&TagType::Unknown).uint().unwrap_or(0), - disc: self.data.get("DISCTOTAL").unwrap_or(&TagType::Unknown).uint().unwrap_or(1), + plays: self + .data + .get("PLAYS") + .unwrap_or(&TagType::Unknown) + .uint() + .unwrap_or(0), + track: self + .data + .get("TRACKTOTAL") + .unwrap_or(&TagType::Unknown) + .uint() + .unwrap_or(0), + disc: self + .data + .get("DISCTOTAL") + .unwrap_or(&TagType::Unknown) + .uint() + .unwrap_or(1), duration: 0, - date: self.data.get("DATE").unwrap_or(&TagType::Unknown).uint().unwrap_or(0), + date: self + .data + .get("DATE") + .unwrap_or(&TagType::Unknown) + .uint() + .unwrap_or(0), } } pub fn genre(&self, id: u64) -> DbGenreItem { DbGenreItem { genre_id: id, - title: self.data.get("GENRE").unwrap_or(&TagType::Unknown).str().unwrap_or("Unknown Genre").into(), + title: self + .data + .get("GENRE") + .unwrap_or(&TagType::Unknown) + .str() + .unwrap_or("Unknown Genre") + .into(), } } } @@ -111,7 +197,7 @@ enum TagType { I64(i64), U64(u64), Str(String), - Unknown + Unknown, } impl TagType { @@ -130,7 +216,7 @@ impl TagType { fn str(&self) -> Option<&str> { match self { Self::Str(s) => Some(&s), - _ => None + _ => None, } } @@ -139,7 +225,7 @@ impl TagType { Self::I64(i) => (*i).try_into().ok(), Self::U64(u) => Some(*u), Self::Str(s) => s.parse::().ok(), - _ => None + _ => None, } } } diff --git a/mps-interpreter/src/music_item.rs b/mps-interpreter/src/music_item.rs index 0fb8f86..f6772bd 100644 --- a/mps-interpreter/src/music_item.rs +++ b/mps-interpreter/src/music_item.rs @@ -9,7 +9,7 @@ pub struct MpsMusicItem { impl MpsMusicItem { pub fn map_row(row: &rusqlite::Row) -> rusqlite::Result { let item = DbMusicItem::map_row(row)?; - Ok(Self{ + Ok(Self { title: item.title, filename: item.filename, }) diff --git a/mps-interpreter/src/processing/mod.rs b/mps-interpreter/src/processing/mod.rs new file mode 100644 index 0000000..18db84c --- /dev/null +++ b/mps-interpreter/src/processing/mod.rs @@ -0,0 +1,5 @@ +mod sql; + +pub mod database { + pub use super::sql::{MpsDatabaseQuerier, MpsSQLiteExecutor, QueryOp, QueryResult}; +} diff --git a/mps-interpreter/src/processing/sql.rs b/mps-interpreter/src/processing/sql.rs new file mode 100644 index 0000000..ab1294c --- /dev/null +++ b/mps-interpreter/src/processing/sql.rs @@ -0,0 +1,167 @@ +use core::fmt::Debug; + +use crate::lang::db::*; +use crate::lang::{MpsOp, RuntimeError}; +use crate::MpsMusicItem; + +pub type QueryResult = Result>, RuntimeError>; +pub type QueryOp = dyn FnMut() -> Box; + +pub trait MpsDatabaseQuerier: Debug { + fn raw(&mut self, query: &str, op: &mut QueryOp) -> QueryResult; + + fn artist_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult; + + fn album_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult; + + fn song_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult; + + fn genre_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult; +} + +#[derive(Default, Debug)] +pub struct MpsSQLiteExecutor { + sqlite_connection: Option, // initialized by first SQL statement +} + +impl MpsSQLiteExecutor { + #[inline] + fn gen_db_maybe(&mut self, op: &mut QueryOp) -> Result<(), RuntimeError> { + if let None = self.sqlite_connection { + // connection needs to be created + match generate_default_db() { + Ok(conn) => { + self.sqlite_connection = Some(conn); + } + Err(e) => { + return Err(RuntimeError { + line: 0, + op: op(), + msg: format!("SQL connection error: {}", e).into(), + }) + } + } + } + Ok(()) + } + + fn music_query_single_param( + &mut self, + query: &str, + param: &str, + op: &mut QueryOp, + ) -> QueryResult { + self.gen_db_maybe(op)?; + let conn = self.sqlite_connection.as_mut().unwrap(); + match perform_single_param_query(conn, query, param) { + Ok(items) => Ok(items + .into_iter() + .map(|item| { + item.map_err(|e| RuntimeError { + line: 0, + op: op(), + msg: format!("SQL item mapping error: {}", e).into(), + }) + }) + .collect()), + Err(e) => Err(RuntimeError { + line: 0, + op: op(), + msg: e, + }), + } + } +} + +impl MpsDatabaseQuerier for MpsSQLiteExecutor { + fn raw(&mut self, query: &str, op: &mut QueryOp) -> QueryResult { + self.gen_db_maybe(op)?; + let conn = self.sqlite_connection.as_mut().unwrap(); + // execute query + match perform_query(conn, query) { + Ok(items) => Ok(items + .into_iter() + .map(|item| { + item.map_err(|e| RuntimeError { + line: 0, + op: op(), + msg: format!("SQL item mapping error: {}", e).into(), + }) + }) + .collect()), + Err(e) => Err(RuntimeError { + line: 0, + op: op(), + msg: e, + }), + } + } + + fn artist_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult { + let param = &format!("%{}%", query); + let query_stmt = + "SELECT songs.* FROM songs + JOIN artists ON songs.artist = artists.artist_id + JOIN metadata ON songs.metadata = metadata.meta_id + WHERE artists.name like ? ORDER BY songs.album, metadata.track"; + self.music_query_single_param(query_stmt, param, op) + } + + fn album_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult { + let param = &format!("%{}%", query); + let query_stmt = + "SELECT songs.* FROM songs + JOIN albums ON songs.album = artists.album_id + JOIN metadata ON songs.metadata = metadata.meta_id + WHERE albums.title like ? ORDER BY songs.album, metadata.track"; + self.music_query_single_param(query_stmt, param, op) + } + + fn song_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult { + let param = &format!("%{}%", query); + let query_stmt = + "SELECT songs.* FROM songs + JOIN metadata ON songs.metadata = metadata.meta_id + WHERE songs.title like ? ORDER BY songs.album, metadata.track"; + self.music_query_single_param(query_stmt, param, op) + } + + fn genre_like(&mut self, query: &str, op: &mut QueryOp) -> QueryResult { + let param = &format!("%{}%", query); + let query_stmt = + "SELECT songs.* FROM songs + JOIN genres ON songs.genre = genres.genre_id + JOIN metadata ON songs.metadata = metadata.meta_id + WHERE genres.title like ? ORDER BY songs.album, metadata.track"; + self.music_query_single_param(query_stmt, param, op) + } +} + +#[inline] +fn perform_query( + conn: &mut rusqlite::Connection, + query: &str, +) -> Result>, String> { + let mut stmt = conn + .prepare(query) + .map_err(|e| format!("SQLite query error: {}", e))?; + let iter = stmt + .query_map([], MpsMusicItem::map_row) + .map_err(|e| format!("SQLite item mapping error: {}", e))?; + Ok(iter.collect()) +} + +#[inline] +fn perform_single_param_query( + conn: &mut rusqlite::Connection, + query: &str, + param: &str, +) -> Result>, String> { + let mut stmt = conn + .prepare_cached(query) + .map_err(|e| format!("SQLite query error: {}", e))?; + let iter = stmt + .query_map([param], MpsMusicItem::map_row) + .map_err(|e| format!("SQLite item mapping error: {}", e))?; + Ok(iter.collect()) +} diff --git a/mps-interpreter/src/runner.rs b/mps-interpreter/src/runner.rs index 00d0d16..94cb5aa 100644 --- a/mps-interpreter/src/runner.rs +++ b/mps-interpreter/src/runner.rs @@ -1,9 +1,9 @@ -use std::iter::Iterator; use std::io::Read; +use std::iter::Iterator; -use super::{MpsInterpretor, MpsContext, MpsMusicItem}; -use super::tokens::{MpsTokenReader, MpsTokenizer}; use super::lang::{MpsLanguageDictionary, MpsLanguageError}; +use super::tokens::{MpsTokenReader, MpsTokenizer}; +use super::{MpsContext, MpsInterpretor, MpsMusicItem}; pub struct MpsRunnerSettings { pub vocabulary: MpsLanguageDictionary, diff --git a/mps-interpreter/src/tokens/error.rs b/mps-interpreter/src/tokens/error.rs index b3c9af3..b94caad 100644 --- a/mps-interpreter/src/tokens/error.rs +++ b/mps-interpreter/src/tokens/error.rs @@ -1,4 +1,4 @@ -use std::fmt::{Debug, Display, Formatter, Error}; +use std::fmt::{Debug, Display, Error, Formatter}; use crate::lang::MpsLanguageError; @@ -11,14 +11,22 @@ pub struct ParseError { impl Display for ParseError { fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { - write!(f, "ParseError (line {}, column {}): Unexpected {}", &self.line, &self.column, &self.item) + write!( + f, + "ParseError (line {}, column {}): Unexpected {}", + &self.line, &self.column, &self.item + ) } } impl MpsTokenError for ParseError { - fn set_line(&mut self, line: usize) {self.line = line} + fn set_line(&mut self, line: usize) { + self.line = line + } - fn set_column(&mut self, column: usize) {self.column = column} + fn set_column(&mut self, column: usize) { + self.column = column + } } pub trait MpsTokenError: Display + Debug { diff --git a/mps-interpreter/src/tokens/mod.rs b/mps-interpreter/src/tokens/mod.rs index 8fc6cdf..e226a24 100644 --- a/mps-interpreter/src/tokens/mod.rs +++ b/mps-interpreter/src/tokens/mod.rs @@ -2,6 +2,6 @@ mod error; mod token_enum; mod tokenizer; -pub use error::{ParseError, MpsTokenError}; +pub use error::{MpsTokenError, ParseError}; pub use token_enum::MpsToken; -pub use tokenizer::{MpsTokenizer, MpsTokenReader}; +pub use tokenizer::{MpsTokenReader, MpsTokenizer}; diff --git a/mps-interpreter/src/tokens/token_enum.rs b/mps-interpreter/src/tokens/token_enum.rs index fdb6efd..530f142 100644 --- a/mps-interpreter/src/tokens/token_enum.rs +++ b/mps-interpreter/src/tokens/token_enum.rs @@ -1,4 +1,4 @@ -use std::fmt::{Debug, Display, Formatter, Error}; +use std::fmt::{Debug, Display, Error, Formatter}; #[derive(Debug, Eq, PartialEq)] pub enum MpsToken { @@ -8,6 +8,8 @@ pub enum MpsToken { Comma, Literal(String), Name(String), + //Octothorpe, + Comment(String), } impl MpsToken { @@ -17,10 +19,11 @@ impl MpsToken { "(" => Ok(Self::OpenBracket), ")" => Ok(Self::CloseBracket), "," => Ok(Self::Comma), + //"#" => Ok(Self::Octothorpe), _ => { // name validation let mut ok = true; - for invalid_c in ["-", "+", ","] { + for invalid_c in ["-", "+", ",", " ", "/", "\n", "\r", "!", "?"] { if s.contains(invalid_c) { ok = false; break; @@ -31,42 +34,56 @@ impl MpsToken { } else { Err(s) } - }, + } } } pub fn is_sql(&self) -> bool { match self { Self::Sql => true, - _ => false + _ => false, } } pub fn is_open_bracket(&self) -> bool { match self { Self::OpenBracket => true, - _ => false + _ => false, } } pub fn is_close_bracket(&self) -> bool { match self { Self::CloseBracket => true, - _ => false + _ => false, } } pub fn is_literal(&self) -> bool { match self { Self::Literal(_) => true, - _ => false + _ => false, } } pub fn is_name(&self) -> bool { match self { Self::Name(_) => true, - _ => false + _ => false, + } + } + + /*pub fn is_octothorpe(&self) -> bool { + match self { + Self::Octothorpe => true, + _ => false, + } + }*/ + + pub fn is_comment(&self) -> bool { + match self { + Self::Comment(_) => true, + _ => false, } } } @@ -80,6 +97,8 @@ impl Display for MpsToken { Self::Comma => write!(f, ","), Self::Literal(s) => write!(f, "\"{}\"", s), Self::Name(s) => write!(f, "{}", s), + //Self::Octothorpe => write!(f, "#"), + Self::Comment(s) => write!(f, "//{}", s), } } } diff --git a/mps-interpreter/src/tokens/tokenizer.rs b/mps-interpreter/src/tokens/tokenizer.rs index 0bd8ae5..62ebce5 100644 --- a/mps-interpreter/src/tokens/tokenizer.rs +++ b/mps-interpreter/src/tokens/tokenizer.rs @@ -1,30 +1,39 @@ use std::collections::VecDeque; -use super::ParseError; use super::MpsToken; +use super::ParseError; pub trait MpsTokenReader { fn current_line(&self) -> usize; fn current_column(&self) -> usize; - fn next_statements(&mut self, count: usize, token_buffer: &mut VecDeque) -> Result<(), ParseError>; + fn next_statement( + &mut self, + token_buffer: &mut VecDeque, + ) -> Result<(), ParseError>; fn end_of_file(&self) -> bool; } -pub struct MpsTokenizer where R: std::io::Read { +pub struct MpsTokenizer +where + R: std::io::Read, +{ reader: R, fsm: ReaderStateMachine, line: usize, column: usize, } -impl MpsTokenizer where R: std::io::Read { +impl MpsTokenizer +where + R: std::io::Read, +{ pub fn new(reader: R) -> Self { Self { reader: reader, - fsm: ReaderStateMachine::Start{}, + fsm: ReaderStateMachine::Start {}, line: 0, column: 0, } @@ -35,7 +44,12 @@ impl MpsTokenizer where R: std::io::Read { // first read special case // always read before checking if end of statement // since FSM could be from previous (already ended) statement - if self.reader.read(&mut byte_buf).map_err(|e| self.error(format!("IO read error: {}", e)))? == 0 { + if self + .reader + .read(&mut byte_buf) + .map_err(|e| self.error(format!("IO read error: {}", e)))? + == 0 + { byte_buf[0] = 0; // clear to null char (nothing read is assumed to mean end of file) } self.do_tracking(byte_buf[0]); @@ -48,29 +62,38 @@ impl MpsTokenizer where R: std::io::Read { } // handle parse endings match self.fsm { - ReaderStateMachine::EndLiteral{} => { + ReaderStateMachine::EndLiteral {} => { let literal = String::from_utf8(bigger_buf.clone()) .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?; buf.push_back(MpsToken::Literal(literal)); bigger_buf.clear(); }, - ReaderStateMachine::EndToken{} => { - let token = String::from_utf8(bigger_buf.clone()) + ReaderStateMachine::EndComment {} => { + let comment = String::from_utf8(bigger_buf.clone()) .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?; - buf.push_back( - MpsToken::parse_from_string(token) - .map_err(|e| self.error(format!("Invalid token {}", e)))? - ); + buf.push_back(MpsToken::Comment(comment)); bigger_buf.clear(); }, - ReaderStateMachine::SingleCharToken{..} => { - let out = bigger_buf.pop().unwrap(); // bracket or comma token - if bigger_buf.len() != 0 { // bracket tokens can be beside other tokens, without separator + ReaderStateMachine::EndToken {} => { + if bigger_buf.len() != 0 { // ignore consecutive end tokens let token = String::from_utf8(bigger_buf.clone()) .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?; buf.push_back( MpsToken::parse_from_string(token) - .map_err(|e| self.error(format!("Invalid token {}", e)))? + .map_err(|e| self.error(format!("Invalid token {}", e)))?, + ); + bigger_buf.clear(); + } + }, + ReaderStateMachine::SingleCharToken { .. } => { + let out = bigger_buf.pop().unwrap(); // bracket or comma token + if bigger_buf.len() != 0 { + // bracket tokens can be beside other tokens, without separator + let token = String::from_utf8(bigger_buf.clone()) + .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?; + buf.push_back( + MpsToken::parse_from_string(token) + .map_err(|e| self.error(format!("Invalid token {}", e)))?, ); bigger_buf.clear(); } @@ -80,32 +103,42 @@ impl MpsTokenizer where R: std::io::Read { .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?; buf.push_back( MpsToken::parse_from_string(token) - .map_err(|e| self.error(format!("Invalid token {}", e)))? + .map_err(|e| self.error(format!("Invalid token {}", e)))?, ); bigger_buf.clear(); }, - ReaderStateMachine::EndStatement{} => { + ReaderStateMachine::EndStatement {} => { // unnecessary; loop will have already exited }, - ReaderStateMachine::EndOfFile{} => { + ReaderStateMachine::EndOfFile {} => { // unnecessary; loop will have already exited }, - _ => {}, + ReaderStateMachine::Invalid { .. } => { + let invalid_char = bigger_buf.pop().unwrap(); // invalid single char + Err(self.error(format!("Unexpected character {}", invalid_char)))?; + }, + _ => {} } - if self.reader.read(&mut byte_buf).map_err(|e| self.error(format!("IO read error: {}", e)))? == 0 { + if self + .reader + .read(&mut byte_buf) + .map_err(|e| self.error(format!("IO read error: {}", e)))? + == 0 + { byte_buf[0] = 0; // clear to null char (nothing read is assumed to mean end of file) } self.do_tracking(byte_buf[0]); self.fsm = self.fsm.next_state(byte_buf[0]); } // handle end statement - if bigger_buf.len() != 0 { // also end of token + if bigger_buf.len() != 0 { + // also end of token // note: never also end of literal, since those have explicit closing characters let token = String::from_utf8(bigger_buf.clone()) .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?; buf.push_back( MpsToken::parse_from_string(token) - .map_err(|e| self.error(format!("Invalid token {}", e)))? + .map_err(|e| self.error(format!("Invalid token {}", e)))?, ); bigger_buf.clear(); } @@ -132,7 +165,7 @@ impl MpsTokenizer where R: std::io::Read { impl MpsTokenReader for MpsTokenizer where - R: std::io::Read + R: std::io::Read, { fn current_line(&self) -> usize { self.line @@ -142,8 +175,13 @@ where self.column } - fn next_statements(&mut self, count: usize, buf: &mut VecDeque) -> Result<(), ParseError> { - for _ in 0..count { + fn next_statement( + &mut self, + buf: &mut VecDeque, + ) -> Result<(), ParseError> { + // read until buffer gets some tokens, in case multiple end of line tokens are at start of stream + let original_size = buf.len(); + while original_size == buf.len() && !self.end_of_file() { self.read_line(buf)?; } Ok(()) @@ -156,94 +194,119 @@ where #[derive(Copy, Clone)] enum ReaderStateMachine { - Start{}, // beginning of machine, no parsing has occured - Regular{ + Start {}, // beginning of machine, no parsing has occured + Regular { out: u8, }, // standard - Escaped{ + Escaped { inside: char, // literal }, // escape character; applied to next character - StartTickLiteral{}, - StartQuoteLiteral{}, - InsideTickLiteral{ + StartTickLiteral {}, + StartQuoteLiteral {}, + InsideTickLiteral { out: u8, }, - InsideQuoteLiteral{ + InsideQuoteLiteral { out: u8, }, SingleCharToken { out: u8, }, - EndLiteral{}, - EndToken{}, - EndStatement{}, - EndOfFile{}, + Slash {out: u8}, + Octothorpe {out: u8}, + Comment {out: u8}, + EndLiteral {}, + EndToken {}, + EndComment {}, + EndStatement {}, + EndOfFile {}, + Invalid { out: u8 }, } impl ReaderStateMachine { pub fn next_state(self, input: u8) -> Self { let input_char = input as char; match self { - Self::Start{} - | Self::Regular{..} - | Self::SingleCharToken{..} - | Self::EndLiteral{} - | Self::EndToken{} - | Self::EndStatement{} => - match input_char { - '\\' => Self::Escaped{inside: '_'}, - '`' => Self::StartTickLiteral{}, - '"' => Self::StartQuoteLiteral{}, - ' ' => Self::EndToken{}, - '\n' | '\r' | ';' => Self::EndStatement{}, - '\0' => Self::EndOfFile{}, - '(' | ')' | ',' => Self::SingleCharToken{out: input}, - _ => Self::Regular{out: input}, - }, - Self::Escaped{inside} => match inside { - '`' => Self::InsideTickLiteral{out: input}, - '"' => Self::InsideQuoteLiteral{out: input}, - '_' | _ => Self::Regular{out: input} + Self::Start {} + | Self::Regular { .. } + | Self::SingleCharToken { .. } + | Self::EndLiteral {} + | Self::EndToken {} + | Self::EndComment {} + | Self::EndStatement {} + | Self::Invalid {..} => match input_char { + '\\' => Self::Escaped { inside: '_' }, + '/' => Self::Slash { out: input }, + '#' => Self::Octothorpe { out: input }, + '`' => Self::StartTickLiteral {}, + '"' => Self::StartQuoteLiteral {}, + ' ' => Self::EndToken {}, + '\n' | '\r' | ';' => Self::EndStatement {}, + '\0' => Self::EndOfFile {}, + '(' | ')' | ',' => Self::SingleCharToken { out: input }, + _ => Self::Regular { out: input }, }, - Self::StartTickLiteral{} - | Self::InsideTickLiteral{..} => - match input_char { - '\\' => Self::Escaped{inside: '`'}, - '`' => Self::EndLiteral{}, - _ => Self::InsideTickLiteral{out: input}, - }, - Self::StartQuoteLiteral{} - | Self::InsideQuoteLiteral{..} => - match input_char { - '\\' => Self::Escaped{inside: '"'}, - '"' => Self::EndLiteral{}, - _ => Self::InsideQuoteLiteral{out: input}, - }, - Self::EndOfFile{} => Self::EndOfFile{}, + Self::Escaped { inside } => match inside { + '`' => Self::InsideTickLiteral { out: input }, + '"' => Self::InsideQuoteLiteral { out: input }, + '_' | _ => Self::Regular { out: input }, + }, + Self::StartTickLiteral {} | Self::InsideTickLiteral { .. } => match input_char { + '\\' => Self::Escaped { inside: '`' }, + '`' => Self::EndLiteral {}, + '\0' => Self::Invalid { out: input }, + _ => Self::InsideTickLiteral { out: input }, + }, + Self::StartQuoteLiteral {} | Self::InsideQuoteLiteral { .. } => match input_char { + '\\' => Self::Escaped { inside: '"' }, + '"' => Self::EndLiteral {}, + '\0' => Self::Invalid { out: input }, + _ => Self::InsideQuoteLiteral { out: input }, + }, + Self::Slash {..} => match input_char { + '/' => Self::Comment { out: input }, + ' ' => Self::EndToken {}, + '\0' => Self::EndOfFile {}, + '\n' | '\r' | ';' => Self::EndStatement {}, + _ => Self::Regular { out: input }, + }, + Self::Octothorpe {..} => match input_char { + '\n' | '\r' | '\0' => Self::EndComment {}, + _ => Self::Comment { out: input } + }, + Self::Comment {..} => match input_char { + '\n' | '\r' | '\0' => Self::EndComment {}, + _ => Self::Comment { out: input }, + }, + Self::EndOfFile {} => Self::EndOfFile {}, } } pub fn is_end_statement(&self) -> bool { match self { - Self::EndStatement{} => true, - _ => false + Self::EndStatement {} => true, + _ => false, } } pub fn is_end_of_file(&self) -> bool { match self { - Self::EndOfFile{} => true, - _ => false + Self::EndOfFile {} => true, + _ => false, } } pub fn output(&self) -> Option { match self { - Self::Regular{ out, ..} - | Self::SingleCharToken{ out, ..} - | Self::InsideTickLiteral{ out, ..} - | Self::InsideQuoteLiteral{ out, ..} => Some(*out), - _ => None + Self::Regular { out, .. } + | Self::SingleCharToken { out, .. } + | Self::InsideTickLiteral { out, .. } + | Self::InsideQuoteLiteral { out, .. } + | Self::Slash { out, .. } + | Self::Octothorpe { out, ..} + | Self::Comment { out, .. } + | Self::Invalid { out, .. } => Some(*out), + _ => None, } } } diff --git a/mps-interpreter/tests/single_line.rs b/mps-interpreter/tests/single_line.rs index 354d3b6..216b83f 100644 --- a/mps-interpreter/tests/single_line.rs +++ b/mps-interpreter/tests/single_line.rs @@ -1,8 +1,8 @@ -use std::io::Cursor; -use std::collections::VecDeque; -use mps_interpreter::*; use mps_interpreter::lang::MpsLanguageError; -use mps_interpreter::tokens::{ParseError, MpsToken, MpsTokenizer}; +use mps_interpreter::tokens::{MpsToken, MpsTokenizer, ParseError}; +use mps_interpreter::*; +use std::collections::VecDeque; +use std::io::Cursor; #[test] fn parse_line() -> Result<(), ParseError> { @@ -34,9 +34,8 @@ fn parse_line() -> Result<(), ParseError> { Ok(()) } -#[test] -fn execute_line() -> Result<(), Box> { - let cursor = Cursor::new("sql(`SELECT * FROM songs ORDER BY artist;`)"); +fn execute_single_line(line: &str, should_be_emtpy: bool) -> Result<(), Box> { + let cursor = Cursor::new(line); let tokenizer = MpsTokenizer::new(cursor); let interpreter = MpsInterpretor::with_standard_vocab(tokenizer); @@ -44,14 +43,36 @@ fn execute_line() -> Result<(), Box> { let mut count = 0; for result in interpreter { if let Ok(item) = result { - count +=1; - if count > 100 {continue;} // no need to spam the rest of the songs + count += 1; + if count > 100 { + continue; + } // no need to spam the rest of the songs println!("Got song `{}` (file: `{}`)", item.title, item.filename); } else { println!("Got error while iterating (executing)"); result?; } } - assert_ne!(count, 0); // database is populated + if should_be_emtpy { + assert_eq!(count, 0); + } else { + assert_ne!(count, 0); // database is populated + } Ok(()) } + +#[test] +fn execute_sql_line() -> Result<(), Box> { + execute_single_line("sql(`SELECT * FROM songs ORDER BY artist;`)", false) +} + +#[test] +fn execute_simple_sql_line() -> Result<(), Box> { + execute_single_line("song(`lov`)", false) +} + +#[test] +fn execute_comment_line() -> Result<(), Box> { + execute_single_line("// this is a comment", true)?; + execute_single_line("# this is a special comment", true) +}