Refactor functions to use standard function vocab parser

This commit is contained in:
NGnius (Graham) 2021-12-29 19:46:53 -05:00
parent 5fab151451
commit e937a7221f
13 changed files with 236 additions and 145 deletions

View file

@ -150,14 +150,12 @@ fn box_error_with_ctx<E: MpsLanguageError + 'static>(
pub(crate) fn standard_vocab(vocabulary: &mut MpsLanguageDictionary) {
vocabulary
// high-priority vocabulary (low-priority may accept this, but will not execute as expected)
.add(crate::lang::vocabulary::filters::empty_filter())
.add(crate::lang::vocabulary::filters::field_filter())
// low-priority (more forgiving statements which may not parse complete statement)
.add(crate::lang::vocabulary::SqlStatementFactory)
.add(crate::lang::vocabulary::SimpleSqlStatementFactory)
.add(crate::lang::vocabulary::sql_function_factory())
.add(crate::lang::vocabulary::simple_sql_function_factory())
.add(crate::lang::vocabulary::CommentStatementFactory)
.add(crate::lang::vocabulary::RepeatStatementFactory)
.add(crate::lang::vocabulary::repeat_function_factory())
.add(crate::lang::vocabulary::AssignStatementFactory)
.add(crate::lang::vocabulary::SqlInitStatementFactory);
.add(crate::lang::vocabulary::sql_init_function_factory());
}

View file

@ -25,9 +25,18 @@ impl MpsLanguageDictionary {
return factory.build_op_boxed(tokens, self);
}
}
let result = match tokens.pop_front() {
Some(x) => Ok(x),
None => Err(SyntaxError {
line: 0,
token: MpsToken::Name("???".into()),
got: None,
})
}?;
Err(SyntaxError {
line: 0,
token: tokens.pop_front().unwrap(),
token: MpsToken::Name("???".into()),
got: Some(result),
})
}

View file

@ -7,15 +7,23 @@ use crate::tokens::MpsToken;
pub struct SyntaxError {
pub line: usize,
pub token: MpsToken,
pub got: Option<MpsToken>
}
impl Display for SyntaxError {
fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
write!(
match &self.got {
Some(t) => write!(
f,
"SyntaxError (line {}): Unexpected {}",
"SyntaxError (line {}): Expected {}, got {}",
&self.line, &self.token, t
),
None => write!(
f,
"SyntaxError (line {}): Expected {}, got nothing",
&self.line, &self.token
)
),
}
}
}

View file

@ -0,0 +1,68 @@
use std::collections::VecDeque;
//use std::fmt::{Debug, Display, Error, Formatter};
use std::marker::PhantomData;
use crate::tokens::MpsToken;
use crate::lang::{MpsOp, BoxedMpsOpFactory};
use crate::lang::SyntaxError;
use crate::lang::MpsLanguageDictionary;
use crate::lang::utility::{assert_token_raw, assert_token_raw_back, assert_token};
pub trait MpsFunctionFactory<Op: MpsOp + 'static> {
fn is_function(&self, name: &str) -> bool;
fn build_function_params(
&self,
name: String,
tokens: &mut VecDeque<MpsToken>,
dict: &MpsLanguageDictionary,
) -> Result<Op, SyntaxError>;
}
pub struct MpsFunctionStatementFactory<Op: MpsOp + 'static, F: MpsFunctionFactory<Op> + 'static> {
op_factory: F,
idc: PhantomData<Op>,
}
impl<Op: MpsOp + 'static, F: MpsFunctionFactory<Op> + 'static> MpsFunctionStatementFactory<Op, F> {
pub fn new(factory: F) -> Self {
Self {
op_factory: factory,
idc: PhantomData::default(),
}
}
}
impl<Op: MpsOp + 'static, F: MpsFunctionFactory<Op> + 'static>
BoxedMpsOpFactory
for
MpsFunctionStatementFactory<Op, F>
{
fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool {
let tokens_len = tokens.len();
if tokens_len < 3 {
false
} else {
match &tokens[0] {
MpsToken::Name(n) => self.op_factory.is_function(n)
&& tokens[1].is_open_bracket()
&& tokens[tokens_len - 1].is_close_bracket(),
_ => false
}
}
}
fn build_op_boxed(
&self,
tokens: &mut VecDeque<MpsToken>,
dict: &MpsLanguageDictionary,
) -> Result<Box<dyn MpsOp>, SyntaxError> {
let name = assert_token(|t| match t {
MpsToken::Name(n) => Some(n),
_ => None
}, MpsToken::Name("function_name".into()), tokens)?;
assert_token_raw(MpsToken::OpenBracket, tokens)?;
assert_token_raw_back(MpsToken::CloseBracket, tokens)?;
Ok(Box::new(self.op_factory.build_function_params(name, tokens, dict)?))
}
}

View file

@ -2,6 +2,7 @@ mod db_items;
mod dictionary;
mod error;
mod filter;
mod function;
mod operation;
mod pseudo_op;
mod repeated_meme;
@ -12,6 +13,7 @@ pub(crate) mod utility;
pub use dictionary::MpsLanguageDictionary;
pub use error::{MpsLanguageError, RuntimeError, SyntaxError};
pub use filter::{MpsFilterFactory, MpsFilterPredicate, MpsFilterStatement, MpsFilterStatementFactory};
pub use function::{MpsFunctionFactory, MpsFunctionStatementFactory};
pub use operation::{BoxedMpsOpFactory, MpsOp, MpsOpFactory, SimpleMpsOpFactory};
pub use pseudo_op::PseudoOp;
pub use repeated_meme::{RepeatedTokens, repeated_tokens};

View file

@ -11,12 +11,21 @@ pub fn assert_token<T, F: FnOnce(MpsToken) -> Option<T>>(
token: MpsToken,
tokens: &mut VecDeque<MpsToken>,
) -> Result<T, SyntaxError> {
if let Some(out) = caster(tokens.pop_front().unwrap()) {
let result = match tokens.pop_front() {
Some(x) => Ok(x),
None => Err(SyntaxError {
line: 0,
token: token.clone(),
got: None,
})
}?;
if let Some(out) = caster(result.clone()) {
Ok(out)
} else {
Err(SyntaxError {
line: 0,
token: token,
got: Some(result),
})
}
}
@ -25,13 +34,44 @@ pub fn assert_token_raw(
token: MpsToken,
tokens: &mut VecDeque<MpsToken>,
) -> Result<MpsToken, SyntaxError> {
let result = tokens.pop_front().unwrap();
let result = match tokens.pop_front() {
Some(x) => Ok(x),
None => Err(SyntaxError {
line: 0,
token: token.clone(),
got: None,
})
}?;
if std::mem::discriminant(&token) == std::mem::discriminant(&result) {
Ok(result)
} else {
Err(SyntaxError {
line: 0,
token: token,
got: Some(result),
})
}
}
pub fn assert_token_raw_back(
token: MpsToken,
tokens: &mut VecDeque<MpsToken>,
) -> Result<MpsToken, SyntaxError> {
let result = match tokens.pop_back() {
Some(x) => Ok(x),
None => Err(SyntaxError {
line: 0,
token: token.clone(),
got: None,
})
}?;
if std::mem::discriminant(&token) == std::mem::discriminant(&result) {
Ok(result)
} else {
Err(SyntaxError {
line: 0,
token: token,
got: Some(result),
})
}
}
@ -43,8 +83,17 @@ pub fn check_token_raw(
std::mem::discriminant(&token) == std::mem::discriminant(token_target)
}
#[allow(dead_code)]
pub fn assert_name(name: &str, tokens: &mut VecDeque<MpsToken>) -> Result<String, SyntaxError> {
match tokens.pop_front().unwrap() {
let result = match tokens.pop_front() {
Some(x) => Ok(x),
None => Err(SyntaxError {
line: 0,
token: MpsToken::Name(name.to_owned()),
got: None,
})
}?;
match result {
MpsToken::Name(n) => {
if n == name {
Ok(n)
@ -53,17 +102,20 @@ pub fn assert_name(name: &str, tokens: &mut VecDeque<MpsToken>) -> Result<String
SyntaxError {
line: 0,
token: MpsToken::Name(name.to_owned()),
got: Some(MpsToken::Name(n)),
}
)
}
},
_token => Err(SyntaxError {
token => Err(SyntaxError {
line: 0,
token: MpsToken::Name(name.to_owned()),
got: Some(token),
})
}
}
#[allow(dead_code)]
pub fn check_name(name: &str, token: &MpsToken) -> bool {
match token {
MpsToken::Name(n) => n == name,
@ -85,7 +137,15 @@ pub fn check_is_type(token: &MpsToken) -> bool {
}
pub fn assert_type(tokens: &mut VecDeque<MpsToken>) -> Result<MpsTypePrimitive, SyntaxError> {
match tokens.pop_front().unwrap() {
let token = match tokens.pop_front() {
Some(x) => Ok(x),
None => Err(SyntaxError {
line: 0,
token: MpsToken::Name("Float | UInt | Int | Bool".into()),
got: None,
})
}?;
match token {
MpsToken::Literal(s) => Ok(MpsTypePrimitive::String(s)),
MpsToken::Name(s) => {
if let Ok(f) = s.parse::<f64>() {
@ -102,12 +162,14 @@ pub fn assert_type(tokens: &mut VecDeque<MpsToken>) -> Result<MpsTypePrimitive,
Err(SyntaxError {
line: 0,
token: MpsToken::Name("Float | UInt | Int | Bool".into()),
got: Some(MpsToken::Name(s))
})
}
},
_token => Err(SyntaxError {
token => Err(SyntaxError {
line: 0,
token: MpsToken::Name("Float | UInt | Int | Bool | \"String\"".into()),
got: Some(token)
})
}
}

View file

@ -30,6 +30,7 @@ pub fn assert_comparison_operator(tokens: &mut VecDeque<MpsToken>) -> Result<[i8
Err(SyntaxError {
line: 0,
token: MpsToken::Equals,
got: if tokens.len() != 0 {Some(tokens[0].clone())} else {None},
})
}
},
@ -52,6 +53,7 @@ pub fn assert_comparison_operator(tokens: &mut VecDeque<MpsToken>) -> Result<[i8
_ => Err(SyntaxError {
line: 0,
token: MpsToken::Equals, // TODO this can be < > or =
got: Some(token1),
})
}
}

View file

@ -5,10 +5,10 @@ mod sql_query;
mod sql_simple_query;
mod variable_assign;
pub use sql_query::{SqlStatement, SqlStatementFactory};
pub use sql_simple_query::{SimpleSqlStatement, SimpleSqlStatementFactory};
pub use sql_query::{SqlStatementFactory, sql_function_factory};
pub use sql_simple_query::{SimpleSqlStatementFactory, simple_sql_function_factory};
pub use comment::{CommentStatement, CommentStatementFactory};
pub use repeat::{RepeatStatement, RepeatStatementFactory};
pub use repeat::{RepeatStatementFactory, repeat_function_factory};
pub use variable_assign::{AssignStatement, AssignStatementFactory};
pub use sql_init::{SqlInitStatement, SqlInitStatementFactory};
pub use sql_init::{SqlInitStatementFactory, sql_init_function_factory};
pub mod filters;

View file

@ -7,9 +7,9 @@ use crate::MpsMusicItem;
use crate::tokens::MpsToken;
use crate::lang::{RuntimeError, SyntaxError};
use crate::lang::{MpsOp, PseudoOp, MpsOpFactory, BoxedMpsOpFactory};
use crate::lang::{MpsOp, PseudoOp, MpsFunctionFactory, MpsFunctionStatementFactory};
use crate::lang::MpsLanguageDictionary;
use crate::lang::utility::{assert_name, assert_token_raw, assert_token, check_name};
use crate::lang::utility::{assert_token_raw, assert_token};
#[derive(Debug)]
pub struct RepeatStatement {
@ -113,34 +113,30 @@ impl MpsOp for RepeatStatement {
}
}
pub struct RepeatStatementFactory;
pub struct RepeatFunctionFactory;
impl MpsOpFactory<RepeatStatement> for RepeatStatementFactory {
fn is_op(&self, tokens: &VecDeque<MpsToken>) -> bool {
tokens.len() >= 3
&& check_name("repeat", &tokens[0])
&& tokens[1].is_open_bracket()
impl MpsFunctionFactory<RepeatStatement> for RepeatFunctionFactory {
fn is_function(&self, name: &str) -> bool {
name == "repeat"
}
fn build_op(
fn build_function_params(
&self,
_name: String,
tokens: &mut VecDeque<MpsToken>,
dict: &MpsLanguageDictionary,
) -> Result<RepeatStatement, SyntaxError> {
// repeat(query) or repeat(query, repetitions)
assert_name("repeat", tokens)?;
assert_token_raw(MpsToken::OpenBracket, tokens)?;
let end_tokens = tokens.split_off(next_comma(tokens));
let inner_statement = dict.try_build_statement(tokens)?;
tokens.extend(end_tokens);
let mut count: Option<usize> = None;
if tokens[0].is_close_bracket() { // no repititions
assert_token_raw(MpsToken::CloseBracket, tokens)?;
} else if tokens[0].is_comma() { // repetitions specified
if tokens.len() != 0 { // repititions specified
assert_token_raw(MpsToken::Comma, tokens)?;
count = Some(assert_token(|t| match t {
MpsToken::Name(n) => n.parse::<usize>().map(|d| d - 1).ok(),
_ => None
}, MpsToken::Name("usize".into()), tokens)?);
assert_token_raw(MpsToken::CloseBracket, tokens)?;
}
Ok(RepeatStatement {
inner_statement: inner_statement.into(),
@ -154,16 +150,18 @@ impl MpsOpFactory<RepeatStatement> for RepeatStatementFactory {
}
}
impl BoxedMpsOpFactory for RepeatStatementFactory {
fn build_op_boxed(
&self,
tokens: &mut VecDeque<MpsToken>,
dict: &MpsLanguageDictionary,
) -> Result<Box<dyn MpsOp>, SyntaxError> {
self.build_box(tokens, dict)
fn next_comma(tokens: &VecDeque<MpsToken>) -> usize {
for i in 0..tokens.len() {
if tokens[i].is_comma() {
return i;
}
fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool {
self.is_op(tokens)
}
tokens.len()
}
pub type RepeatStatementFactory = MpsFunctionStatementFactory<RepeatStatement, RepeatFunctionFactory>;
#[inline(always)]
pub fn repeat_function_factory() -> RepeatStatementFactory {
RepeatStatementFactory::new(RepeatFunctionFactory)
}

View file

@ -8,9 +8,9 @@ use crate::MpsMusicItem;
use crate::tokens::MpsToken;
use crate::lang::{RuntimeError, SyntaxError};
use crate::lang::{MpsOp, SimpleMpsOpFactory, MpsOpFactory, BoxedMpsOpFactory};
use crate::lang::{MpsOp, MpsFunctionFactory, MpsFunctionStatementFactory};
use crate::lang::MpsLanguageDictionary;
use crate::lang::utility::{assert_token_raw, check_name, assert_name, assert_token};
use crate::lang::utility::{assert_token_raw, assert_token};
use crate::lang::repeated_tokens;
#[derive(Debug)]
@ -62,21 +62,19 @@ impl MpsOp for SqlInitStatement {
}
}
pub struct SqlInitStatementFactory;
pub struct SqlInitFunctionFactory;
impl SimpleMpsOpFactory<SqlInitStatement> for SqlInitStatementFactory {
fn is_op_simple(&self, tokens: &VecDeque<MpsToken>) -> bool {
tokens.len() >= 3
&& check_name("sql_init", &tokens[0])
&& tokens[1].is_open_bracket()
impl MpsFunctionFactory<SqlInitStatement> for SqlInitFunctionFactory {
fn is_function(&self, name: &str) -> bool {
name == "sql_init"
}
fn build_op_simple(
fn build_function_params(
&self,
_name: String,
tokens: &mut VecDeque<MpsToken>,
_dict: &MpsLanguageDictionary,
) -> Result<SqlInitStatement, SyntaxError> {
assert_name("sql_init", tokens)?;
assert_token_raw(MpsToken::OpenBracket, tokens)?;
let ingest = |tokens2: &mut VecDeque<MpsToken>| {
if tokens2.len() < 3 {return Ok(None);} // nothing wrong, nothing left to ingest
let param_name = assert_token(|t| match t {
@ -92,7 +90,6 @@ impl SimpleMpsOpFactory<SqlInitStatement> for SqlInitStatementFactory {
Ok(Some((param_name, param_val))) // successfully ingested one phrase
};
let params = repeated_tokens(ingest, MpsToken::Comma).ingest_all(tokens)?;
assert_token_raw(MpsToken::CloseBracket, tokens)?;
Ok(SqlInitStatement {
context: None,
params: HashMap::from_iter(params),
@ -100,16 +97,9 @@ impl SimpleMpsOpFactory<SqlInitStatement> for SqlInitStatementFactory {
}
}
impl BoxedMpsOpFactory for SqlInitStatementFactory {
fn build_op_boxed(
&self,
tokens: &mut VecDeque<MpsToken>,
dict: &MpsLanguageDictionary,
) -> Result<Box<dyn MpsOp>, SyntaxError> {
self.build_box(tokens, dict)
}
pub type SqlInitStatementFactory = MpsFunctionStatementFactory<SqlInitStatement, SqlInitFunctionFactory>;
fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool {
self.is_op(tokens)
}
#[inline(always)]
pub fn sql_init_function_factory() -> SqlInitStatementFactory {
SqlInitStatementFactory::new(SqlInitFunctionFactory)
}

View file

@ -2,8 +2,8 @@ use std::collections::VecDeque;
use std::fmt::{Debug, Display, Error, Formatter};
use std::iter::Iterator;
use crate::lang::utility::{assert_token, assert_token_raw, check_name, assert_name};
use crate::lang::{BoxedMpsOpFactory, MpsLanguageDictionary, MpsOp, MpsOpFactory};
use crate::lang::utility::assert_token;
use crate::lang::{MpsLanguageDictionary, MpsOp, MpsFunctionFactory, MpsFunctionStatementFactory};
use crate::lang::{RuntimeError, SyntaxError};
use crate::tokens::MpsToken;
use crate::MpsContext;
@ -102,28 +102,20 @@ impl Display for SqlStatement {
}
}
pub struct SqlStatementFactory;
pub struct SqlFunctionFactory;
impl MpsOpFactory<SqlStatement> for SqlStatementFactory {
#[inline]
fn is_op(&self, tokens: &VecDeque<MpsToken>) -> bool {
tokens.len() > 3
&& check_name("sql", &tokens[0])
&& tokens[1].is_open_bracket()
&& tokens[2].is_literal()
&& tokens[3].is_close_bracket()
impl MpsFunctionFactory<SqlStatement> for SqlFunctionFactory {
fn is_function(&self, name: &str) -> bool {
name == "sql"
}
#[inline]
fn build_op(
fn build_function_params(
&self,
_name: String,
tokens: &mut VecDeque<MpsToken>,
_dict: &MpsLanguageDictionary,
) -> Result<SqlStatement, SyntaxError> {
// sql ( `some query` )
assert_name("sql", tokens)?;
//assert_token_raw(MpsToken::Sql, tokens)?;
assert_token_raw(MpsToken::OpenBracket, tokens)?;
let literal = assert_token(
|t| match t {
MpsToken::Literal(query) => Some(query),
@ -132,7 +124,6 @@ impl MpsOpFactory<SqlStatement> for SqlStatementFactory {
MpsToken::Literal("".into()),
tokens,
)?;
assert_token_raw(MpsToken::CloseBracket, tokens)?;
Ok(SqlStatement {
query: literal,
context: None,
@ -142,16 +133,9 @@ impl MpsOpFactory<SqlStatement> for SqlStatementFactory {
}
}
impl BoxedMpsOpFactory for SqlStatementFactory {
fn build_op_boxed(
&self,
tokens: &mut VecDeque<MpsToken>,
dict: &MpsLanguageDictionary,
) -> Result<Box<dyn MpsOp>, SyntaxError> {
self.build_box(tokens, dict)
}
pub type SqlStatementFactory = MpsFunctionStatementFactory<SqlStatement, SqlFunctionFactory>;
fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool {
self.is_op(tokens)
}
#[inline(always)]
pub fn sql_function_factory() -> SqlStatementFactory {
SqlStatementFactory::new(SqlFunctionFactory)
}

View file

@ -2,8 +2,8 @@ use std::collections::VecDeque;
use std::fmt::{Debug, Display, Error, Formatter};
use std::iter::Iterator;
use crate::lang::utility::{assert_token, assert_token_raw};
use crate::lang::{BoxedMpsOpFactory, MpsLanguageDictionary, MpsOp, MpsOpFactory};
use crate::lang::utility::assert_token;
use crate::lang::{MpsLanguageDictionary, MpsOp, MpsFunctionFactory, MpsFunctionStatementFactory};
use crate::lang::{RuntimeError, SyntaxError};
use crate::tokens::MpsToken;
use crate::MpsContext;
@ -26,7 +26,8 @@ impl QueryMode {
"genre" => Ok(QueryMode::Genre),
_ => Err(SyntaxError {
line: 0,
token: Self::tokenify(name),
token: MpsToken::Name("artist|album|song|genre".into()),
got: Some(Self::tokenify(name)),
}),
}
}
@ -162,43 +163,20 @@ impl Display for SimpleSqlStatement {
}
}
pub struct SimpleSqlStatementFactory;
pub struct SimpleSqlFunctionFactory;
impl MpsOpFactory<SimpleSqlStatement> for SimpleSqlStatementFactory {
#[inline]
fn is_op(&self, tokens: &VecDeque<MpsToken>) -> bool {
tokens.len() > 3
&& match &tokens[0] {
MpsToken::Name(name) => QueryMode::is_valid_name(name),
_ => false,
}
&& tokens[1].is_open_bracket()
&& tokens[2].is_literal()
&& tokens[3].is_close_bracket()
impl MpsFunctionFactory<SimpleSqlStatement> for SimpleSqlFunctionFactory {
fn is_function(&self, name: &str) -> bool {
QueryMode::is_valid_name(name)
}
#[inline]
fn build_op(
fn build_function_params(
&self,
mode_name: String,
tokens: &mut VecDeque<MpsToken>,
_dict: &MpsLanguageDictionary,
) -> Result<SimpleSqlStatement, SyntaxError> {
// artist|album|song|genre ( `like` )
let mode_name = assert_token(
|t| match t {
MpsToken::Name(name) => {
if QueryMode::is_valid_name(&name) {
Some(name)
} else {
None
}
}
_ => None,
},
MpsToken::Name("artist|album|song|genre".into()),
tokens,
)?;
assert_token_raw(MpsToken::OpenBracket, tokens)?;
// artist|album|song|genre ( `title_like` )
let literal = assert_token(
|t| match t {
MpsToken::Literal(query) => Some(query),
@ -207,7 +185,6 @@ impl MpsOpFactory<SimpleSqlStatement> for SimpleSqlStatementFactory {
MpsToken::Literal("literal".into()),
tokens,
)?;
assert_token_raw(MpsToken::CloseBracket, tokens)?;
Ok(SimpleSqlStatement {
query: literal,
mode: QueryMode::from_name(mode_name)?,
@ -218,16 +195,9 @@ impl MpsOpFactory<SimpleSqlStatement> for SimpleSqlStatementFactory {
}
}
impl BoxedMpsOpFactory for SimpleSqlStatementFactory {
fn build_op_boxed(
&self,
tokens: &mut VecDeque<MpsToken>,
dict: &MpsLanguageDictionary,
) -> Result<Box<dyn MpsOp>, SyntaxError> {
self.build_box(tokens, dict)
}
pub type SimpleSqlStatementFactory = MpsFunctionStatementFactory<SimpleSqlStatement, SimpleSqlFunctionFactory>;
fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool {
self.is_op(tokens)
}
#[inline(always)]
pub fn simple_sql_function_factory() -> SimpleSqlStatementFactory {
SimpleSqlStatementFactory::new(SimpleSqlFunctionFactory)
}

View file

@ -48,7 +48,7 @@ fn execute_single_line(line: &str, should_be_emtpy: bool, should_complete: bool)
count += 1;
if count > 100 {
if should_complete {
continue;
continue; // skip println, but still check for errors
} else {
println!("Got 100 items, stopping to avoid infinite loop");
break;