NGnius/muss
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity

Refactor operation emitter to distinguish filters/sorters/itemops from standalone functions

Browse source
This commit is contained in:
NGnius (Graham) 2023-07-10 18:21:51 -04:00
parent 7dbd896348
commit e6e58047f6
43 changed files with 932 additions and 1024 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

592
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

1
interpreter/src/faye.rs
View file

@ -134,6 +134,7 @@ where
Ok(stmt) => stmt,
Err(e) => return Some(Err(error_with_ctx(e, self.tokenizer.current_line()))),
};
//println!("Final parsed op: {}", stmt);
#[cfg(debug_assertions)]
if !self.buffer.is_empty() {
panic!("Token buffer was not emptied! (rem: {:?})", self.buffer)

44
interpreter/src/interpretor.rs
View file

@ -4,10 +4,9 @@ use super::lang::LanguageDictionary;
pub(crate) fn standard_vocab(vocabulary: &mut LanguageDictionary) {
vocabulary
// filters
.add(crate::lang::vocabulary::filters::empty_filter())
.add(crate::lang::vocabulary::filters::unique_filter()) // accepts .(unique)
.add(crate::lang::vocabulary::filters::range_filter())
.add( // accepts any .(.something)
.add_transform(crate::lang::vocabulary::filters::empty_filter())
.add_transform(crate::lang::vocabulary::filters::range_filter())
.add_transform( // accepts any .(.something)
crate::lang::vocabulary::filters::field::FieldFilterBlockFactory::new()
.push(crate::lang::vocabulary::filters::field::FieldFilterComparisonFactory)
.push(crate::lang::vocabulary::filters::field::FieldFilterMaybeFactory)
@ -15,39 +14,41 @@ pub(crate) fn standard_vocab(vocabulary: &mut LanguageDictionary) {
.push(crate::lang::vocabulary::filters::field::FieldRegexFilterFactory)
.to_statement_factory()
)
.add(crate::lang::vocabulary::filters::index_filter())
.add(crate::lang::vocabulary::filters::unique_field_filter())
.add(crate::lang::vocabulary::filters::nonempty_filter())
.add_transform(crate::lang::vocabulary::filters::unique_field_filter())
.add_transform(crate::lang::vocabulary::filters::unique_filter())
.add_transform(crate::lang::vocabulary::filters::nonempty_filter())
.add_transform(crate::lang::vocabulary::filters::index_filter())
// sorters
.add(crate::lang::vocabulary::sorters::empty_sort())
.add(crate::lang::vocabulary::sorters::shuffle_sort()) // accepts ~(~shuffle)
.add(crate::lang::vocabulary::sorters::bliss_sort())
.add(crate::lang::vocabulary::sorters::bliss_next_sort())
.add(crate::lang::vocabulary::sorters::radio_sort())
.add(crate::lang::vocabulary::sorters::field_sort()) // accepts any ~(.name)
.add_transform(crate::lang::vocabulary::sorters::empty_sort())
.add_transform(crate::lang::vocabulary::sorters::shuffle_sort()) // accepts ~(~shuffle)
.add_transform(crate::lang::vocabulary::sorters::bliss_sort())
.add_transform(crate::lang::vocabulary::sorters::bliss_next_sort())
.add_transform(crate::lang::vocabulary::sorters::radio_sort())
.add_transform(crate::lang::vocabulary::sorters::field_sort()) // accepts any ~(.name)
// iter blocks
.add(
.add_transform(
crate::lang::ItemBlockFactory::new()
.push(crate::lang::vocabulary::item_ops::ConstantItemOpFactory)
.push(crate::lang::vocabulary::item_ops::VariableAssignItemOpFactory)
.push(crate::lang::vocabulary::item_ops::FieldAssignItemOpFactory)
.push(crate::lang::vocabulary::item_ops::FileItemOpFactory)
.push(crate::lang::vocabulary::item_ops::VariableAssignItemOpFactory)
.push(crate::lang::vocabulary::item_ops::VariableDeclareItemOpFactory)
.push(crate::lang::vocabulary::item_ops::FileItemOpFactory)
.push(crate::lang::vocabulary::item_ops::InterpolateStringItemOpFactory)
.push(crate::lang::vocabulary::item_ops::BranchItemOpFactory)
.push(crate::lang::vocabulary::item_ops::IterItemOpFactory)
.push(crate::lang::vocabulary::item_ops::ConstructorItemOpFactory)
.push(crate::lang::vocabulary::item_ops::EmptyItemOpFactory)
.push(crate::lang::vocabulary::item_ops::RemoveItemOpFactory)
.push(crate::lang::vocabulary::item_ops::VariableRetrieveItemOpFactory)
.push(crate::lang::vocabulary::item_ops::NegateItemOpFactory)
.push(crate::lang::vocabulary::item_ops::NotItemOpFactory)
.push(crate::lang::vocabulary::item_ops::CompareItemOpFactory)
.push(crate::lang::vocabulary::item_ops::NegateItemOpFactory)
.push(crate::lang::vocabulary::item_ops::AddItemOpFactory)
.push(crate::lang::vocabulary::item_ops::SubtractItemOpFactory)
.push(crate::lang::vocabulary::item_ops::OrItemOpFactory)
.push(crate::lang::vocabulary::item_ops::AndItemOpFactory)
.push(crate::lang::vocabulary::item_ops::BracketsItemOpFactory),
.push(crate::lang::vocabulary::item_ops::BracketsItemOpFactory)
.push(crate::lang::vocabulary::item_ops::FieldRetrieveItemOpFactory)
.push(crate::lang::vocabulary::item_ops::ConstantItemOpFactory)
.push(crate::lang::vocabulary::item_ops::VariableRetrieveItemOpFactory)
)
// functions and misc
// functions don't enforce bracket coherence
@ -64,5 +65,6 @@ pub(crate) fn standard_vocab(vocabulary: &mut LanguageDictionary) {
.add(crate::lang::vocabulary::empties_function_factory())
.add(crate::lang::vocabulary::reset_function_factory())
.add(crate::lang::vocabulary::union_function_factory())
.add(crate::lang::vocabulary::intersection_function_factory());
.add(crate::lang::vocabulary::intersection_function_factory())
.add(crate::lang::vocabulary::VariableRetrieveStatementFactory);
}

62
interpreter/src/lang/dictionary.rs
View file

@ -1,26 +1,33 @@
use std::collections::VecDeque;
use super::SyntaxError;
use super::{BoxedOpFactory, Op};
use super::{BoxedOpFactory, Op, BoxedTransformOpFactory};
use crate::tokens::Token;
pub struct LanguageDictionary {
vocabulary: Vec<Box<dyn BoxedOpFactory>>,
root_vocabulary: Vec<Box<dyn BoxedOpFactory>>,
transform_vocabulary: Vec<Box<dyn BoxedTransformOpFactory>>,
}
impl LanguageDictionary {
pub fn add<T: BoxedOpFactory + 'static>(&mut self, factory: T) -> &mut Self {
self.vocabulary
self.root_vocabulary
.push(Box::new(factory) as Box<dyn BoxedOpFactory>);
self
}
pub fn try_build_statement(
pub fn add_transform<T: BoxedTransformOpFactory + 'static>(&mut self, factory: T) -> &mut Self {
self.transform_vocabulary
.push(Box::new(factory) as Box<dyn BoxedTransformOpFactory>);
self
}
fn try_build_root_statement(
&self,
tokens: &mut VecDeque<Token>,
) -> Result<Box<dyn Op>, SyntaxError> {
//println!("try_build_statement with tokens {:?}", tokens);
for factory in &self.vocabulary {
//println!("building root op with tokens {:?}", tokens);
for factory in &self.root_vocabulary {
if factory.is_op_boxed(tokens) {
return factory.build_op_boxed(tokens, self);
}
@ -40,9 +47,47 @@ impl LanguageDictionary {
})
}
fn try_build_transformed_statement(
&self,
mut op: Box<dyn Op>,
tokens: &mut VecDeque<Token>,
) -> Result<Box<dyn Op>, SyntaxError> {
//println!("building transformer for op {} with tokens {:?}", op, tokens);
let mut op_found = true;
while op_found && !tokens.is_empty() {
(op, op_found) = self.try_build_one_transform(op, tokens)?;
}
//println!("built transformed op {}, remaining tokens {:?}", op, tokens);
Ok(op)
}
pub fn try_build_one_transform(
&self,
mut op: Box<dyn Op>,
tokens: &mut VecDeque<Token>,
) -> Result<(Box<dyn Op>, bool), SyntaxError> {
for factory in &self.transform_vocabulary {
if factory.is_transform_op(tokens) {
op = factory.build_transform_op(tokens, self, op)?;
return Ok((op, true))
}
}
Ok((op, false))
}
pub fn try_build_statement(
&self,
tokens: &mut VecDeque<Token>,
) -> Result<Box<dyn Op>, SyntaxError> {
let root = self.try_build_root_statement(tokens)?;
//println!("built root op {}, remaining tokens {:?}", root, tokens);
self.try_build_transformed_statement(root, tokens)
}
pub fn new() -> Self {
Self {
vocabulary: Vec::new(),
root_vocabulary: Vec::new(),
transform_vocabulary: Vec::new(),
}
}
@ -57,7 +102,8 @@ impl LanguageDictionary {
impl Default for LanguageDictionary {
fn default() -> Self {
Self {
vocabulary: Vec::new(),
root_vocabulary: Vec::new(),
transform_vocabulary: Vec::new(),
}
}
}

410
interpreter/src/lang/filter.rs
View file

@ -3,11 +3,11 @@ use std::fmt::{Debug, Display, Error, Formatter};
use std::iter::Iterator;
use std::marker::PhantomData;
use crate::lang::utility::{assert_name, assert_token, assert_token_raw, check_name};
use crate::lang::utility::{assert_name, assert_token_raw, check_name};
use crate::lang::FilterReplaceStatement;
use crate::lang::LanguageDictionary;
use crate::lang::SingleItem;
use crate::lang::{BoxedOpFactory, IteratorItem, Op, PseudoOp};
use crate::lang::{BoxedTransformOpFactory, IteratorItem, Op, PseudoOp};
use crate::lang::{RuntimeError, RuntimeMsg, RuntimeOp, SyntaxError};
use crate::processing::general::Type;
use crate::tokens::Token;
@ -34,25 +34,10 @@ pub trait FilterFactory<P: FilterPredicate + 'static>: Send + Sync {
) -> Result<P, SyntaxError>;
}
#[derive(Debug, Clone)]
pub(super) enum VariableOrOp {
Variable(String),
Op(PseudoOp),
}
impl Display for VariableOrOp {
fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
match self {
Self::Variable(s) => write!(f, "{}", s),
Self::Op(op) => write!(f, "{}", op),
}
}
}
#[derive(Debug)]
pub struct FilterStatement<P: FilterPredicate + 'static> {
predicate: P,
iterable: VariableOrOp,
iterable: PseudoOp,
context: Option<Context>,
other_filters: Option<PseudoOp>,
is_failing: bool,
@ -75,7 +60,7 @@ impl<P: FilterPredicate + 'static> Display for FilterStatement<P> {
if let Some(other_filters) = &self.other_filters {
write!(
f,
"{}.({} || (like) {})",
"{}.({} || (retconned) {})",
self.iterable, self.predicate, other_filters
)
} else {
@ -95,20 +80,8 @@ impl<P: FilterPredicate + 'static> Op for FilterStatement<P> {
fn is_resetable(&self) -> bool {
let is_iterable_resetable = match &self.iterable {
VariableOrOp::Variable(s) => {
if self.context.is_some() {
let var = self.context.as_ref().unwrap().variables.get_opt(s);
if let Some(Type::Op(var)) = var {
var.is_resetable()
} else {
false
}
} else {
true
} // ASSUMPTION
}
VariableOrOp::Op(PseudoOp::Real(op)) => op.is_resetable(),
VariableOrOp::Op(PseudoOp::Fake(_)) => false,
PseudoOp::Real(op) => op.is_resetable(),
PseudoOp::Fake(_) => false,
};
let is_other_filter_resetable =
if let Some(PseudoOp::Real(other_filter)) = &self.other_filters {
@ -126,45 +99,13 @@ impl<P: FilterPredicate + 'static> Op for FilterStatement<P> {
.reset()
.map_err(|x| x.with(RuntimeOp(fake.clone())))?;
match &mut self.iterable {
VariableOrOp::Variable(s) => {
if self.context.as_mut().unwrap().variables.exists(s) {
let mut var = self
.context
.as_mut()
.unwrap()
.variables
.remove(s)
.map_err(|e| e.with(RuntimeOp(fake.clone())))?;
let result = if let Type::Op(var) = &mut var {
var.enter(self.context.take().unwrap());
let result = var.reset();
self.context = Some(var.escape());
result
} else {
Err(RuntimeError {
line: 0,
op: fake.clone(),
msg: "Cannot reset non-iterable filter variable".to_string(),
})
};
self.context
.as_mut()
.unwrap()
.variables
.declare(s, var)
.map_err(|e| e.with(RuntimeOp(fake)))?;
result
} else {
Ok(())
}
}
VariableOrOp::Op(PseudoOp::Real(op)) => {
PseudoOp::Real(op) => {
op.enter(self.context.take().unwrap());
let result = op.reset();
self.context = Some(op.escape());
result
}
VariableOrOp::Op(PseudoOp::Fake(_)) => Err(RuntimeError {
PseudoOp::Fake(_) => Err(RuntimeError {
line: 0,
op: fake,
msg: "Cannot reset PseudoOp::Fake filter".to_string(),
@ -183,10 +124,7 @@ impl<P: FilterPredicate + 'static> Op for FilterStatement<P> {
fn dup(&self) -> Box<dyn Op> {
Box::new(Self {
predicate: self.predicate.clone(),
iterable: match &self.iterable {
VariableOrOp::Variable(s) => VariableOrOp::Variable(s.clone()),
VariableOrOp::Op(op) => VariableOrOp::Op(op.try_real_ref().unwrap().dup().into()),
},
iterable: self.iterable.try_real_ref().unwrap().dup().into(),
context: None,
other_filters: self
.other_filters
@ -199,57 +137,15 @@ impl<P: FilterPredicate + 'static> Op for FilterStatement<P> {
impl <P: FilterPredicate + 'static> FilterStatement<P> {
fn next_item(&mut self) -> Option<IteratorItem> {
let fake_op = PseudoOp::from_printable(self);
match &mut self.iterable {
VariableOrOp::Op(op) => match op.try_real() {
Ok(real_op) => {
let ctx = self.context.take().unwrap();
real_op.enter(ctx);
let item = real_op.next();
self.context = Some(real_op.escape());
item
}
Err(e) => return Some(Err(e)),
},
VariableOrOp::Variable(variable_name) => {
let mut variable = match self
.context
.as_mut()
.unwrap()
.variables
.remove(variable_name)
{
Ok(Type::Op(op)) => op,
Ok(x) => {
return Some(Err(RuntimeError {
line: 0,
op: fake_op,
msg: format!(
"Expected operation/iterable type in variable {}, got {}",
&variable_name, x
),
}))
}
Err(e) => {
self.is_failing = true;
return Some(Err(e.with(RuntimeOp(PseudoOp::from_printable(self)))));
},
};
variable.enter(self.context.take().unwrap());
let item = variable.next();
self.context = Some(variable.escape());
if let Err(e) = self
.context
.as_mut()
.unwrap()
.variables
.declare(variable_name, Type::Op(variable))
{
return Some(Err(e.with(RuntimeOp(PseudoOp::from_printable(self)))));
}
match self.iterable.try_real() {
Ok(real_op) => {
let ctx = self.context.take().unwrap();
real_op.enter(ctx);
let item = real_op.next();
self.context = Some(real_op.escape());
item
}
Err(e) => return Some(Err(e)),
}
}
}
@ -258,6 +154,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterStatement<P> {
type Item = IteratorItem;
fn next(&mut self) -> Option<Self::Item> {
//println!("In FilterStatement {}", self.predicate);
if (self.predicate.is_complete() && self.other_filters.is_none()) || self.is_failing {
return None;
}
@ -278,6 +175,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterStatement<P> {
if let Some(inner) = &mut self.other_filters {
// handle other filters
// make fake inner item
//println!("Making fake inner variable `{}`", INNER_VARIABLE_NAME);
let single_op = SingleItem::new_ok(item.clone());
let preexisting_var = self.context.as_mut().unwrap()
.variables
@ -310,18 +208,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterStatement<P> {
}
fn size_hint(&self) -> (usize, Option<usize>) {
match &self.iterable {
VariableOrOp::Variable(s) => self
.context
.as_ref()
.and_then(|x| x.variables.get_opt(s))
.and_then(|x| match x {
Type::Op(op) => Some(op.size_hint()),
_ => None,
}),
VariableOrOp::Op(op) => op.try_real_ref().map(|x| x.size_hint()).ok(),
}
.unwrap_or((0, None))
self.iterable.try_real_ref().map(|x| x.size_hint()).ok().unwrap_or((0, None))
}
}
@ -339,130 +226,64 @@ impl<P: FilterPredicate + 'static, F: FilterFactory<P> + 'static> FilterStatemen
}
}
impl<P: FilterPredicate + 'static, F: FilterFactory<P> + 'static> BoxedOpFactory
for FilterStatementFactory<P, F>
{
#[allow(clippy::unnecessary_unwrap)]
fn is_op_boxed(&self, tokens: &VecDeque<Token>) -> bool {
let tokens_len = tokens.len();
if is_correct_format(tokens) {
let start_of_predicate = last_dot_before_open_bracket(tokens) + 2; // .(predicate)
if start_of_predicate > tokens_len - 1 {
false
} else {
let pipe_location_opt = first_double_pipe(tokens, 1);
if pipe_location_opt.is_some() && pipe_location_opt.unwrap() > start_of_predicate {
let pipe_location = pipe_location_opt.unwrap();
// filters combined by OR operations
let tokens2: VecDeque<&Token> =
VecDeque::from_iter(tokens.range(start_of_predicate..pipe_location));
self.filter_factory.is_filter(&tokens2)
} else {
// single filter
let tokens2: VecDeque<&Token> =
VecDeque::from_iter(tokens.range(start_of_predicate..tokens_len - 1));
if !tokens2.is_empty() && check_name("if", tokens2[0]) {
// replacement filter
if let Some(colon_location) = first_colon2(&tokens2) {
let tokens3 = VecDeque::from_iter(tokens.range(
start_of_predicate + 1..start_of_predicate + colon_location,
));
self.filter_factory.is_filter(&tokens3)
} else {
false
}
} else {
// regular filter
self.filter_factory.is_filter(&tokens2)
}
}
}
} else {
false
}
}
fn build_op_boxed(
impl<P: FilterPredicate + 'static, F: FilterFactory<P> + 'static> BoxedTransformOpFactory
for FilterStatementFactory<P, F> {
fn build_transform_op(
&self,
tokens: &mut VecDeque<Token>,
dict: &LanguageDictionary,
op: Box<dyn Op>,
) -> Result<Box<dyn Op>, SyntaxError> {
let start_of_op = last_dot_before_open_bracket(tokens);
let op = if start_of_op == 1 && tokens[0].is_name() {
// variable_name.(predicate)
let variable_name = assert_token(
|t| match t {
Token::Name(s) => Some(s),
_ => None,
},
Token::Name("variable_name".into()),
tokens,
)?;
VariableOrOp::Variable(variable_name)
} else {
// <some other op>.(predicate)
//let mut new_tokens = tokens.range(0..start_of_op).map(|x| x.to_owned()).collect();
let end_tokens = tokens.split_off(start_of_op); // don't parse filter in inner statement
let inner_op = dict.try_build_statement(tokens)?;
tokens.extend(end_tokens);
VariableOrOp::Op(inner_op.into())
};
assert_token_raw(Token::Dot, tokens)?;
assert_token_raw(Token::OpenBracket, tokens)?;
if !tokens.is_empty() && check_name("if", &tokens[0]) {
return {
// replacement filter
//println!("Building replacement filter from tokens {:?}", tokens);
assert_name("if", tokens)?;
if let Some(colon_location) = first_colon(tokens) {
let end_tokens = tokens.split_off(colon_location);
let filter = self.filter_factory.build_filter(tokens, dict)?;
tokens.extend(end_tokens);
assert_token_raw(Token::Colon, tokens)?;
let mut else_op: Option<PseudoOp> = None;
let if_op: PseudoOp;
if let Some(else_location) = first_else_not_in_bracket(tokens) {
let end_tokens = tokens.split_off(else_location);
// build replacement system
if_op = dict.try_build_statement(tokens)?.into();
tokens.extend(end_tokens);
assert_name("else", tokens)?;
let end_tokens = tokens.split_off(tokens.len() - 1); // up to ending close bracket
// build replacement system
else_op = Some(dict.try_build_statement(tokens)?.into());
tokens.extend(end_tokens);
} else {
let end_tokens = tokens.split_off(tokens.len() - 1);
// build replacement system
if_op = dict.try_build_statement(tokens)?.into();
tokens.extend(end_tokens);
}
assert_token_raw(Token::CloseBracket, tokens)?;
Ok(Box::new(FilterReplaceStatement {
predicate: filter,
iterable: op,
context: None,
op_if: if_op,
op_else: else_op,
item_cache: super::filter_replace::item_cache_deque(),
}))
} else {
Err(SyntaxError {
line: 0,
token: Token::Colon,
got: None,
})
}
};
// replacement filter
assert_name("if", tokens)?;
let filter = self.filter_factory.build_filter(tokens, dict)?;
assert_token_raw(Token::Colon, tokens)?;
let mut else_op: Option<PseudoOp> = None;
let if_op: PseudoOp = dict.try_build_statement(tokens)?.into();
if check_name("else", &tokens[0]) {
assert_name("else", tokens)?;
else_op = Some(dict.try_build_statement(tokens)?.into());
}
/*if let Some(else_location) = first_else_not_in_bracket(tokens) {
//println!("First else found at {}; {:?}", else_location, tokens);
//let end_tokens = tokens.split_off(else_location);
// build replacement system
if_op = dict.try_build_statement(tokens)?.into();
//tokens.extend(end_tokens);
assert_name("else", tokens)?;
//let end_tokens = tokens.split_off(tokens.len() - 1); // up to ending close bracket
// build replacement system
else_op = Some(dict.try_build_statement(tokens)?.into());
//tokens.extend(end_tokens);
} else {
//let end_tokens = tokens.split_off(tokens.len() - 1);
// build replacement system
if_op = dict.try_build_statement(tokens)?.into();
//tokens.extend(end_tokens);
}*/
assert_token_raw(Token::CloseBracket, tokens)?;
Ok(Box::new(FilterReplaceStatement {
predicate: filter,
iterable: op.into(),
context: None,
op_if: if_op,
op_else: else_op,
item_cache: super::filter_replace::item_cache_deque(),
}))
} else {
// regular filter
let mut another_filter = None;
let (has_or, end_tokens) = if let Some(pipe_location) = first_double_pipe(tokens, 0) {
let has_or = first_double_pipe(tokens, 0).is_some();
/*let (has_or, end_tokens) = if let Some(pipe_location) = first_double_pipe(tokens, 0) {
(true, tokens.split_off(pipe_location)) // parse up to OR operator
} else {
(false, tokens.split_off(tokens.len() - 1)) // don't parse closing bracket in filter
};
};*/
let filter = self.filter_factory.build_filter(tokens, dict)?;
tokens.extend(end_tokens);
//tokens.extend(end_tokens);
if has_or {
// recursively build other filters for OR operation
assert_token_raw(Token::Pipe, tokens)?;
@ -470,66 +291,57 @@ impl<P: FilterPredicate + 'static, F: FilterFactory<P> + 'static> BoxedOpFactory
// emit fake filter syntax
tokens.push_front(Token::OpenBracket);
tokens.push_front(Token::Dot);
tokens.push_front(Token::Name(INNER_VARIABLE_NAME.into())); // impossible to obtain through parsing on purpose
another_filter = Some(dict.try_build_statement(tokens)?.into());
//tokens.push_front(Token::Name(INNER_VARIABLE_NAME.into())); // impossible to obtain through parsing on purpose
/*let inner_op = Box::new(crate::lang::vocabulary::VariableRetrieveStatement {
variable_name: INNER_VARIABLE_NAME.into(),
context: None,
is_tried: false,
});*/
let mut inner_tokens = VecDeque::with_capacity(1);
inner_tokens.push_front(Token::Name(INNER_VARIABLE_NAME.into()));
let inner_op = dict.try_build_statement(&mut inner_tokens)?;
let (inner_op, op_transformed) = dict.try_build_one_transform(inner_op, tokens)?;
if !op_transformed {
return Err(
SyntaxError {
token: Token::Name(INNER_VARIABLE_NAME.into()),
got: Some(Token::Name(INNER_VARIABLE_NAME.into())),
line: 0,
}
)
}
//println!("Built 2nd filter: {}", inner_op);
another_filter = Some(inner_op.into());
} else {
//println!("filter tokens before final bracket: {:?}", tokens);
assert_token_raw(Token::CloseBracket, tokens)?; // remove closing bracket
}
Ok(Box::new(FilterStatement {
predicate: filter,
iterable: op,
iterable: op.into(),
context: None,
other_filters: another_filter,
is_failing: false,
}))
}
}
}
fn is_correct_format(tokens: &VecDeque<Token>) -> bool {
let mut inside_brackets = 0;
let mut open_bracket_found = false;
let mut close_bracket = 0;
for i in (0..tokens.len()).rev() {
if tokens[i].is_close_bracket() {
if inside_brackets == 0 {
close_bracket = i;
}
inside_brackets += 1;
} else if tokens[i].is_open_bracket() {
if inside_brackets == 1 {
open_bracket_found = true;
} else if inside_brackets != 0 {
inside_brackets -= 1;
}
} else if open_bracket_found {
return tokens[i].is_dot() && (close_bracket == 0 || close_bracket == tokens.len() - 1);
}
}
false
}
fn last_dot_before_open_bracket(tokens: &VecDeque<Token>) -> usize {
let mut inside_brackets = 0;
let mut open_bracket_found = false;
for i in (0..tokens.len()).rev() {
if tokens[i].is_close_bracket() {
inside_brackets += 1;
} else if tokens[i].is_open_bracket() {
if inside_brackets == 1 {
open_bracket_found = true;
} else if inside_brackets != 0 {
inside_brackets -= 1;
}
} else if open_bracket_found {
if tokens[i].is_dot() {
return i;
fn is_transform_op(&self, tokens: &VecDeque<Token>) -> bool {
let result = if tokens.len() > 2 && tokens[0].is_dot() && tokens[1].is_open_bracket() {
if check_name("if", &tokens[2]) {
let tokens2: VecDeque<&Token> =
VecDeque::from_iter(tokens.range(3..));
self.filter_factory.is_filter(&tokens2)
} else {
return 0;
let tokens2: VecDeque<&Token> =
VecDeque::from_iter(tokens.range(2..));
self.filter_factory.is_filter(&tokens2)
}
}
} else {
false
};
result
}
0
}
fn first_double_pipe(tokens: &VecDeque<Token>, in_brackets: usize) -> Option<usize> {
@ -554,25 +366,7 @@ fn first_double_pipe(tokens: &VecDeque<Token>, in_brackets: usize) -> Option<usi
None
}
fn first_colon(tokens: &VecDeque<Token>) -> Option<usize> {
for i in 0..tokens.len() {
if tokens[i].is_colon() {
return Some(i);
}
}
None
}
fn first_colon2(tokens: &VecDeque<&Token>) -> Option<usize> {
for i in 0..tokens.len() {
if tokens[i].is_colon() {
return Some(i);
}
}
None
}
fn first_else_not_in_bracket(tokens: &VecDeque<Token>) -> Option<usize> {
/*fn first_else_not_in_bracket(tokens: &VecDeque<Token>) -> Option<usize> {
let mut inside_brackets = 0;
for i in 0..tokens.len() {
if check_name("else", &tokens[i]) && inside_brackets == 0 {
@ -584,4 +378,4 @@ fn first_else_not_in_bracket(tokens: &VecDeque<Token>) -> Option<usize> {
}
}
None
}
}*/

127
interpreter/src/lang/filter_replace.rs
View file

@ -3,7 +3,7 @@ use std::fmt::{Debug, Display, Error, Formatter};
use std::iter::Iterator;
use crate::lang::SingleItem;
use crate::lang::{filter::VariableOrOp, FilterPredicate};
use crate::lang::FilterPredicate;
use crate::lang::{IteratorItem, Op, PseudoOp};
use crate::lang::{RuntimeError, RuntimeMsg, RuntimeOp};
use crate::processing::general::Type;
@ -21,7 +21,7 @@ pub(super) fn item_cache_deque() -> VecDeque<Result<Item, RuntimeError>> {
#[derive(Debug)]
pub struct FilterReplaceStatement<P: FilterPredicate + 'static> {
pub(super) predicate: P,
pub(super) iterable: VariableOrOp,
pub(super) iterable: PseudoOp,
pub(super) context: Option<Context>,
pub(super) op_if: PseudoOp,
pub(super) op_else: Option<PseudoOp>,
@ -70,20 +70,8 @@ impl<P: FilterPredicate + 'static> Op for FilterReplaceStatement<P> {
fn is_resetable(&self) -> bool {
match &self.iterable {
VariableOrOp::Variable(s) => {
if self.context.is_some() {
let var = self.context.as_ref().unwrap().variables.get_opt(s);
if let Some(Type::Op(var)) = var {
var.is_resetable()
} else {
false
}
} else {
true
} // ASSUMPTION
}
VariableOrOp::Op(PseudoOp::Real(op)) => op.is_resetable(),
VariableOrOp::Op(PseudoOp::Fake(_)) => false,
PseudoOp::Real(op) => op.is_resetable(),
PseudoOp::Fake(_) => false,
}
}
@ -94,45 +82,13 @@ impl<P: FilterPredicate + 'static> Op for FilterReplaceStatement<P> {
.reset()
.map_err(|x| x.with(RuntimeOp(fake.clone())))?;
match &mut self.iterable {
VariableOrOp::Variable(s) => {
if self.context.as_mut().unwrap().variables.exists(s) {
let mut var = self
.context
.as_mut()
.unwrap()
.variables
.remove(s)
.map_err(|e| e.with(RuntimeOp(fake.clone())))?;
let result = if let Type::Op(var) = &mut var {
var.enter(self.context.take().unwrap());
let result = var.reset();
self.context = Some(var.escape());
result
} else {
Err(RuntimeError {
line: 0,
op: fake.clone(),
msg: "Cannot reset non-iterable filter variable".to_string(),
})
};
self.context
.as_mut()
.unwrap()
.variables
.declare(s, var)
.map_err(|e| e.with(RuntimeOp(fake)))?;
result
} else {
Ok(())
}
}
VariableOrOp::Op(PseudoOp::Real(op)) => {
PseudoOp::Real(op) => {
op.enter(self.context.take().unwrap());
let result = op.reset();
self.context = Some(op.escape());
result
}
VariableOrOp::Op(PseudoOp::Fake(_)) => Err(RuntimeError {
PseudoOp::Fake(_) => Err(RuntimeError {
line: 0,
op: fake,
msg: "Cannot reset PseudoOp::Fake filter".to_string(),
@ -143,10 +99,7 @@ impl<P: FilterPredicate + 'static> Op for FilterReplaceStatement<P> {
fn dup(&self) -> Box<dyn Op> {
Box::new(Self {
predicate: self.predicate.clone(),
iterable: match &self.iterable {
VariableOrOp::Variable(s) => VariableOrOp::Variable(s.clone()),
VariableOrOp::Op(op) => VariableOrOp::Op(op.try_real_ref().unwrap().dup().into()),
},
iterable: self.iterable.try_real_ref().unwrap().dup().into(),
context: None,
op_if: PseudoOp::from(self.op_if.try_real_ref().unwrap().dup()),
op_else: self
@ -167,53 +120,15 @@ impl<P: FilterPredicate + 'static> Iterator for FilterReplaceStatement<P> {
}
let fake = PseudoOp::Fake(format!("{}", self));
// get next item in iterator
let next_item = match &mut self.iterable {
VariableOrOp::Op(op) => match op.try_real() {
Ok(real_op) => {
let ctx = self.context.take().unwrap();
real_op.enter(ctx);
let item = real_op.next();
self.context = Some(real_op.escape());
item
}
Err(e) => return Some(Err(e)),
},
VariableOrOp::Variable(variable_name) => {
let mut variable = match self
.context
.as_mut()
.unwrap()
.variables
.remove(variable_name)
{
Ok(Type::Op(op)) => op,
Ok(x) => {
return Some(Err(RuntimeError {
line: 0,
op: fake,
msg: format!(
"Expected operation/iterable type in variable {}, got {}",
&variable_name, x
),
}))
}
Err(e) => return Some(Err(e.with(RuntimeOp(fake)))),
};
let next_item = match self.iterable.try_real() {
Ok(real_op) => {
let ctx = self.context.take().unwrap();
variable.enter(ctx);
let item = variable.next();
self.context = Some(variable.escape());
if let Err(e) = self
.context
.as_mut()
.unwrap()
.variables
.declare(variable_name, Type::Op(variable))
{
return Some(Err(e.with(RuntimeOp(fake))));
}
real_op.enter(ctx);
let item = real_op.next();
self.context = Some(real_op.escape());
item
}
Err(e) => return Some(Err(e)),
};
// process item
match next_item {
@ -242,6 +157,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterReplaceStatement<P> {
real_op.enter(self.context.take().unwrap());
if real_op.is_resetable() {
if let Err(e) = real_op.reset() {
self.context = Some(real_op.escape());
return Some(Err(e));
}
}
@ -269,6 +185,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterReplaceStatement<P> {
replacement
}
} else if let Some(op_else) = &mut self.op_else {
println!("op_else {}\n{:?}", op_else, op_else);
// unwrap inner operation
match op_else.try_real() {
Ok(real_op) => {
@ -286,6 +203,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterReplaceStatement<P> {
real_op.enter(self.context.take().unwrap());
if real_op.is_resetable() {
if let Err(e) = real_op.reset() {
self.context = Some(real_op.escape());
return Some(Err(e));
}
}
@ -325,18 +243,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterReplaceStatement<P> {
}
fn size_hint(&self) -> (usize, Option<usize>) {
match &self.iterable {
VariableOrOp::Variable(s) => self
.context
.as_ref()
.and_then(|x| x.variables.get_opt(s))
.and_then(|x| match x {
Type::Op(op) => Some(op.size_hint()),
_ => None,
}),
VariableOrOp::Op(op) => op.try_real_ref().map(|x| x.size_hint()).ok(),
}
.unwrap_or((0, None))
self.iterable.try_real_ref().map(|x| x.size_hint()).ok().unwrap_or((0, None))
}
}

5
interpreter/src/lang/function.rs
View file

@ -45,7 +45,6 @@ impl<O: Op + 'static, F: FunctionFactory<O> + 'static> BoxedOpFactory
Token::Name(n) => {
self.op_factory.is_function(n)
&& tokens[1].is_open_bracket()
&& tokens[tokens_len - 1].is_close_bracket()
}
_ => false,
}
@ -66,9 +65,9 @@ impl<O: Op + 'static, F: FunctionFactory<O> + 'static> BoxedOpFactory
tokens,
)?;
assert_token_raw(Token::OpenBracket, tokens)?;
let end_tokens = tokens.split_off(tokens.len() - 1);
//let end_tokens = tokens.split_off(tokens.len() - 1);
let func = self.op_factory.build_function_params(name, tokens, dict)?;
tokens.extend(end_tokens);
//tokens.extend(end_tokens);
assert_token_raw(Token::CloseBracket, tokens)?;
Ok(Box::new(func))
}

95
interpreter/src/lang/iter_block.rs
View file

@ -6,9 +6,9 @@ use std::iter::Iterator;
use std::marker::PhantomData;
use std::sync::Arc;
use crate::lang::utility::{assert_token_raw, assert_token_raw_back};
use crate::lang::utility::assert_token_raw;
use crate::lang::LanguageDictionary;
use crate::lang::{BoxedOpFactory, IteratorItem, Op, PseudoOp};
use crate::lang::{BoxedTransformOpFactory, IteratorItem, Op, PseudoOp};
use crate::lang::{RuntimeError, RuntimeMsg, RuntimeOp, SyntaxError};
use crate::processing::general::Type;
use crate::tokens::Token;
@ -72,7 +72,7 @@ impl Display for ItemBlockStatement {
writeln!(f)?;
}
for statement in self.statements.iter() {
writeln!(f, "{}", statement)?;
writeln!(f, "{},", statement)?;
}
write!(f, "}}")
}
@ -227,7 +227,7 @@ impl ItemBlockFactory {
tokens: &mut VecDeque<Token>,
dict: &LanguageDictionary,
) -> Result<Box<dyn ItemOp>, SyntaxError> {
for factory in &self.vocabulary {
for (i, factory) in self.vocabulary.iter().enumerate() {
if factory.is_item_op(tokens) {
return factory.build_item_op(tokens, self, dict);
}
@ -253,48 +253,44 @@ impl ItemBlockFactory {
}
}
impl BoxedOpFactory for ItemBlockFactory {
fn is_op_boxed(&self, tokens: &VecDeque<Token>) -> bool {
find_last_open_curly(tokens).is_some()
}
fn build_op_boxed(
impl BoxedTransformOpFactory for ItemBlockFactory {
fn build_transform_op(
&self,
tokens: &mut VecDeque<Token>,
dict: &LanguageDictionary,
op: Box<dyn Op>,
) -> Result<Box<dyn Op>, SyntaxError> {
let open_curly_pos = if let Some(pos) = find_last_open_curly(tokens) {
Ok(pos)
} else {
Err(SyntaxError {
line: 0,
token: Token::OpenCurly,
got: tokens.pop_front(),
})
}?;
let block_tokens = tokens.split_off(open_curly_pos - 1); // . always before {
let inner_op = dict.try_build_statement(tokens)?;
tokens.extend(block_tokens);
assert_token_raw(Token::Dot, tokens)?;
assert_token_raw(Token::OpenCurly, tokens)?;
assert_token_raw_back(Token::CloseCurly, tokens)?;
let mut item_ops = Vec::with_capacity(tokens.len() / 8);
while !tokens.is_empty() {
if let Some(next_comma) = find_next_comma(tokens) {
let end_tokens = tokens.split_off(next_comma);
item_ops.push(Arc::new(self.try_build_item_statement(tokens, dict)?));
tokens.extend(end_tokens);
assert_token_raw(Token::Comma, tokens)?;
if tokens[0].is_close_curly() {
break;
}
item_ops.push(Arc::new(self.try_build_item_statement(tokens, dict)?));
if !tokens.is_empty() {
if tokens[0].is_comma() {
assert_token_raw(Token::Comma, tokens)?;
}
} else {
item_ops.push(Arc::new(self.try_build_item_statement(tokens, dict)?));
return Err(SyntaxError {
got: tokens.pop_front(),
token: Token::Literal(", or }".into()),
line: 0,
})
}
}
assert_token_raw(Token::CloseCurly, tokens)?;
Ok(Box::new(ItemBlockStatement {
statements: item_ops,
iterable: inner_op.into(),
iterable: op.into(),
last_item: None,
}))
}
fn is_transform_op(&self, tokens: &VecDeque<Token>) -> bool {
tokens.len() > 1 && tokens[0].is_dot() && tokens[1].is_open_curly()
}
}
fn replace_item_var(ctx: &mut Context, item: Type) -> Option<Type> {
@ -320,42 +316,7 @@ fn restore_item_var(ctx: &mut Context, old_var: Option<Type>) -> Result<Option<T
Ok(new_var)
}
fn find_last_open_curly(tokens: &VecDeque<Token>) -> Option<usize> {
let mut bracket_depth = 0;
let mut curly_found = false;
for i in (0..tokens.len()).rev() {
let token = &tokens[i];
match token {
Token::OpenCurly => {
if bracket_depth != 0 {
bracket_depth -= 1;
}
}
Token::CloseCurly => {
bracket_depth += 1;
}
Token::Dot => {
if bracket_depth == 0 && curly_found {
return Some(i + 1);
}
}
Token::OpenBracket | Token::CloseBracket => {
if bracket_depth == 0 {
return None;
}
}
_ => {}
}
if token.is_open_curly() {
curly_found = true;
} else {
curly_found = false;
}
}
None
}
fn find_next_comma(tokens: &VecDeque<Token>) -> Option<usize> {
/*fn find_next_comma(tokens: &VecDeque<Token>) -> Option<usize> {
let mut bracket_depth = 0;
let mut curly_depth = 0;
for i in 0..tokens.len() {
@ -373,4 +334,4 @@ fn find_next_comma(tokens: &VecDeque<Token>) -> Option<usize> {
}
}
None
}
}*/

2
interpreter/src/lang/mod.rs
View file

@ -29,7 +29,7 @@ pub use function::{FunctionFactory, FunctionStatementFactory};
pub use generator_op::GeneratorOp;
pub use iter_block::{ItemBlockFactory, ItemOp, ItemOpFactory};
pub use lookup::Lookup;
pub use operation::{BoxedOpFactory, IteratorItem, Op, OpFactory, SimpleOpFactory};
pub use operation::{BoxedOpFactory, IteratorItem, Op, OpFactory, SimpleOpFactory, BoxedTransformOpFactory};
pub use pseudo_op::PseudoOp;
pub use repeated_meme::{repeated_tokens, RepeatedTokens};
pub use single_op::SingleItem;

33
interpreter/src/lang/operation.rs
View file

@ -12,20 +12,22 @@ use crate::Item;
// TODO change API to allow for accumulating modifiers
// e.g. build_op(&self, Option<Box<dyn Op>>, tokens) -> ...
pub trait SimpleOpFactory<T: Op + 'static> {
fn is_op_simple(&self, tokens: &VecDeque<Token>) -> bool;
type TokenList = VecDeque<Token>;
fn build_op_simple(&self, tokens: &mut VecDeque<Token>) -> Result<T, SyntaxError>;
pub trait SimpleOpFactory<T: Op + 'static> {
fn is_op_simple(&self, tokens: &TokenList) -> bool;
fn build_op_simple(&self, tokens: &mut TokenList) -> Result<T, SyntaxError>;
}
impl<T: Op + 'static, X: SimpleOpFactory<T> + 'static> OpFactory<T> for X {
fn is_op(&self, tokens: &VecDeque<Token>) -> bool {
fn is_op(&self, tokens: &TokenList) -> bool {
self.is_op_simple(tokens)
}
fn build_op(
&self,
tokens: &mut VecDeque<Token>,
tokens: &mut TokenList,
_dict: &LanguageDictionary,
) -> Result<T, SyntaxError> {
self.build_op_simple(tokens)
@ -33,18 +35,18 @@ impl<T: Op + 'static, X: SimpleOpFactory<T> + 'static> OpFactory<T> for X {
}
pub trait OpFactory<T: Op + 'static> {
fn is_op(&self, tokens: &VecDeque<Token>) -> bool;
fn is_op(&self, tokens: &TokenList) -> bool;
fn build_op(
&self,
tokens: &mut VecDeque<Token>,
tokens: &mut TokenList,
dict: &LanguageDictionary,
) -> Result<T, SyntaxError>;
#[inline]
fn build_box(
&self,
tokens: &mut VecDeque<Token>,
tokens: &mut TokenList,
dict: &LanguageDictionary,
) -> Result<Box<dyn Op>, SyntaxError> {
Ok(Box::new(self.build_op(tokens, dict)?))
@ -54,11 +56,22 @@ pub trait OpFactory<T: Op + 'static> {
pub trait BoxedOpFactory: Send {
fn build_op_boxed(
&self,
tokens: &mut VecDeque<Token>,
tokens: &mut TokenList,
dict: &LanguageDictionary,
) -> Result<Box<dyn Op>, SyntaxError>;
fn is_op_boxed(&self, tokens: &VecDeque<Token>) -> bool;
fn is_op_boxed(&self, tokens: &TokenList) -> bool;
}
pub trait BoxedTransformOpFactory: Send {
fn build_transform_op(
&self,
tokens: &mut TokenList,
dict: &LanguageDictionary,
op: Box<dyn Op>,
) -> Result<Box<dyn Op>, SyntaxError>;
fn is_transform_op(&self, tokens: &TokenList) -> bool;
}
pub type IteratorItem = Result<Item, RuntimeError>;

109
interpreter/src/lang/sorter.rs
View file

@ -3,9 +3,9 @@ use std::fmt::{Debug, Display, Error, Formatter};
use std::iter::Iterator;
use std::marker::PhantomData;
use crate::lang::utility::{assert_name, assert_token_raw, check_name};
use crate::lang::utility::assert_token_raw;
use crate::lang::LanguageDictionary;
use crate::lang::{BoxedOpFactory, IteratorItem, Op, PseudoOp};
use crate::lang::{IteratorItem, Op, PseudoOp, BoxedTransformOpFactory};
use crate::lang::{RuntimeError, RuntimeMsg, RuntimeOp, SyntaxError};
use crate::tokens::Token;
use crate::Context;
@ -125,58 +125,16 @@ impl<S: Sorter + 'static, F: SorterFactory<S> + 'static> SortStatementFactory<S,
}
}
impl<S: Sorter + 'static, F: SorterFactory<S> + 'static> BoxedOpFactory
impl<S: Sorter + 'static, F: SorterFactory<S> + 'static> BoxedTransformOpFactory
for SortStatementFactory<S, F>
{
fn is_op_boxed(&self, tokens: &VecDeque<Token>) -> bool {
let tokens_len = tokens.len();
if let Some(tilde_location) = last_tilde(tokens, 0) {
// iterable~(sorter)
if tokens_len > tilde_location + 2 {
let tokens2: VecDeque<&Token> =
VecDeque::from_iter(tokens.range(tilde_location + 2..tokens_len - 1));
tokens[tokens_len - 1].is_close_bracket() && self.sort_factory.is_sorter(&tokens2)
} else {
false
}
} else if let Some(dot_location) = last_dot_sort(tokens, 1) {
// iterable.sort(sorter)
if tokens_len > dot_location + 3 {
let tokens2: VecDeque<&Token> =
VecDeque::from_iter(tokens.range(dot_location + 3..tokens_len - 1));
tokens[tokens_len - 1].is_close_bracket() && self.sort_factory.is_sorter(&tokens2)
} else {
false
}
} else {
false
}
}
fn build_op_boxed(
fn build_transform_op(
&self,
tokens: &mut VecDeque<Token>,
dict: &LanguageDictionary,
op: Box<dyn Op>,
) -> Result<Box<dyn Op>, SyntaxError> {
let inner_op;
if let Some(tilde_location) = last_tilde(tokens, 0) {
let end_tokens = tokens.split_off(tilde_location);
inner_op = dict.try_build_statement(tokens)?;
tokens.extend(end_tokens);
assert_token_raw(Token::Tilde, tokens)?;
} else if let Some(dot_location) = last_dot_sort(tokens, 1) {
let end_tokens = tokens.split_off(dot_location);
inner_op = dict.try_build_statement(tokens)?;
tokens.extend(end_tokens);
assert_token_raw(Token::Dot, tokens)?;
assert_name("sort", tokens)?;
} else {
return Err(SyntaxError {
line: 0,
token: Token::Name(".|~".into()),
got: tokens.pop_front(),
});
}
assert_token_raw(Token::Tilde, tokens)?;
assert_token_raw(Token::OpenBracket, tokens)?;
let end_tokens = tokens.split_off(tokens.len() - 1);
let sorter = self.sort_factory.build_sorter(tokens, dict)?;
@ -184,58 +142,19 @@ impl<S: Sorter + 'static, F: SorterFactory<S> + 'static> BoxedOpFactory
assert_token_raw(Token::CloseBracket, tokens)?;
Ok(Box::new(SortStatement {
orderer: sorter,
iterable: inner_op.into(),
iterable: op.into(),
item_cache: VecDeque::with_capacity(SORTER_ITEM_CACHE_SIZE),
}))
}
}
fn last_tilde(tokens: &VecDeque<Token>, target_depth: usize) -> Option<usize> {
let mut bracket_depth = 0;
for i in (0..tokens.len()).rev() {
let current_token = &tokens[i];
if current_token.is_close_bracket() {
bracket_depth += 1;
} else if current_token.is_open_bracket() && bracket_depth != 0 {
bracket_depth -= 1;
} else if current_token.is_tilde() && bracket_depth == target_depth {
return Some(i);
fn is_transform_op(&self, tokens: &VecDeque<Token>) -> bool {
if tokens.len() > 2 {
let tokens2: VecDeque<&Token> =
VecDeque::from_iter(tokens.range(2..));
tokens[0].is_tilde() && self.sort_factory.is_sorter(&tokens2)
} else {
false
}
}
None
}
fn last_dot_sort(tokens: &VecDeque<Token>, target_depth: usize) -> Option<usize> {
let mut bracket_depth = 0;
let mut sort_found = false;
let mut bracket_found = false;
for i in (0..tokens.len()).rev() {
let current_token = &tokens[i];
if sort_found {
return {
if current_token.is_dot() {
Some(i)
} else {
None
}
};
} else if bracket_found {
if check_name("sort", current_token) {
sort_found = true;
} else {
bracket_found = false;
}
}
if current_token.is_close_bracket() {
bracket_depth += 1;
} else if current_token.is_open_bracket() {
if target_depth == bracket_depth {
bracket_found = true;
}
if bracket_depth != 0 {
bracket_depth -= 1;
}
}
}
None
}

23
interpreter/src/lang/utility.rs
View file

@ -49,29 +49,6 @@ pub fn assert_token_raw(token: Token, tokens: &mut VecDeque<Token>) -> Result<To
}
}
pub fn assert_token_raw_back(
token: Token,
tokens: &mut VecDeque<Token>,
) -> Result<Token, SyntaxError> {
let result = match tokens.pop_back() {
Some(x) => Ok(x),
None => Err(SyntaxError {
line: 0,
token: token.clone(),
got: None,
}),
}?;
if std::mem::discriminant(&token) == std::mem::discriminant(&result) {
Ok(result)
} else {
Err(SyntaxError {
line: 0,
token,
got: Some(result),
})
}
}
pub fn check_token_raw(token: Token, token_target: &Token) -> bool {
std::mem::discriminant(&token) == std::mem::discriminant(token_target)
}

6
interpreter/src/lang/vocabulary/files.rs
View file

@ -141,11 +141,11 @@ impl FunctionFactory<FilesStatement> for FilesFunctionFactory {
tokens: &mut VecDeque<Token>,
_dict: &LanguageDictionary,
) -> Result<FilesStatement, SyntaxError> {
// files([folder|dir=]"path", [regex|re = "pattern"], [recursive = true|false])
// files([folder|dir=]"path", [regex|re = "pattern",] [recursive = true|false,])
let mut root_path = None;
let mut pattern = None;
let mut recursive = None;
if !tokens.is_empty() {
if !tokens.is_empty() && !tokens[0].is_close_bracket() {
if tokens[0].is_literal() {
// folder is specified without keyword
root_path = Some(assert_token(
@ -162,7 +162,7 @@ impl FunctionFactory<FilesStatement> for FilesFunctionFactory {
}
// parse keyword function parameters
let ingest = |tokens2: &mut VecDeque<Token>| {
if tokens2.len() < 3 {
if tokens2[0].is_close_bracket() {
return Ok(None);
} // nothing wrong, nothing left to ingest
let param_name = assert_token(

2
interpreter/src/lang/vocabulary/filters/empty_filter.rs
View file

@ -35,7 +35,7 @@ pub struct EmptyFilterFactory;
impl FilterFactory<EmptyFilter> for EmptyFilterFactory {
fn is_filter(&self, tokens: &VecDeque<&Token>) -> bool {
tokens.is_empty()
!tokens.is_empty() && tokens[0].is_close_bracket()
}
fn build_filter(

2
interpreter/src/lang/vocabulary/filters/field/field_match_filter.rs
View file

@ -167,7 +167,7 @@ impl FieldFilterFactory<FieldRegexFilter> for FieldRegexFilterFactory {
fn regex_flags(tokens: &mut VecDeque<Token>) -> Result<u8, SyntaxError> {
// syntax: , "flags"
let mut result = 0_u8;
if tokens.is_empty() {
if tokens.is_empty() || !tokens[0].is_comma() {
Ok(result)
} else {
assert_token_raw(Token::Comma, tokens)?;

4
interpreter/src/lang/vocabulary/filters/index_filter.rs
View file

@ -64,8 +64,8 @@ pub struct IndexFilterFactory;
impl FilterFactory<IndexFilter> for IndexFilterFactory {
fn is_filter(&self, tokens: &VecDeque<&Token>) -> bool {
(tokens.len() == 1 && Lookup::check_is(tokens[0]))
|| (tokens.len() == 2 && tokens[0].is_exclamation() && Lookup::check_is(tokens[1]))
(tokens.len() >= 1 && Lookup::check_is(tokens[0]))
|| (tokens.len() >= 2 && tokens[0].is_exclamation() && Lookup::check_is(tokens[1]))
}
fn build_filter(

8
interpreter/src/lang/vocabulary/filters/range_filter.rs
View file

@ -101,9 +101,9 @@ pub struct RangeFilterFactory;
impl FilterFactory<RangeFilter> for RangeFilterFactory {
fn is_filter(&self, tokens: &VecDeque<&Token>) -> bool {
tokens.len() >= 2
&& ((tokens.len() >= 2 && tokens[0].is_dot() && tokens[1].is_dot())
|| (tokens.len() >= 3
tokens.len() > 1
&& ((tokens[0].is_dot() && tokens[1].is_dot())
|| (tokens.len() > 2
&& Lookup::check_is(tokens[0])
&& tokens[1].is_dot()
&& tokens[2].is_dot()))
@ -132,7 +132,7 @@ impl FilterFactory<RangeFilter> for RangeFilterFactory {
false
};
// end index
let end = if !tokens.is_empty() {
let end = if !tokens.is_empty() && Lookup::check_is(&tokens[0]) {
Some(Lookup::parse(tokens)?)
} else {
None

6
interpreter/src/lang/vocabulary/filters/unique.rs
View file

@ -90,7 +90,7 @@ pub struct UniqueFilterFactory;
impl FilterFactory<UniqueFieldFilter> for UniqueFilterFactory {
fn is_filter(&self, tokens: &VecDeque<&Token>) -> bool {
tokens.len() >= 2 && check_name("unique", tokens[0]) && tokens[1].is_dot()
tokens.len() > 1 && check_name("unique", tokens[0]) && tokens[1].is_dot()
}
fn build_filter(
@ -108,7 +108,7 @@ impl FilterFactory<UniqueFieldFilter> for UniqueFilterFactory {
Token::Name("field_name".into()),
tokens,
)?;
let error_handling = if !tokens.is_empty() {
let error_handling = if !tokens.is_empty() && (tokens[0].is_exclamation() || tokens[0].is_interrogation()) {
if tokens[0].is_exclamation() {
assert_token_raw(Token::Exclamation, tokens)?;
FieldFilterErrorHandling::Ignore
@ -129,7 +129,7 @@ impl FilterFactory<UniqueFieldFilter> for UniqueFilterFactory {
impl FilterFactory<UniqueFilter> for UniqueFilterFactory {
fn is_filter(&self, tokens: &VecDeque<&Token>) -> bool {
tokens.len() == 1 && check_name("unique", tokens[0])
tokens.len() > 1 && check_name("unique", tokens[0])
}
fn build_filter(

9
interpreter/src/lang/vocabulary/item_ops/add.rs
View file

@ -69,10 +69,11 @@ impl ItemOpFactory<AddItemOp> for AddItemOpFactory {
dict: &LanguageDictionary,
) -> Result<AddItemOp, SyntaxError> {
let plus_location = first_plus(tokens).unwrap();
let mut end_tokens = tokens.split_off(plus_location);
let end_tokens = tokens.split_off(plus_location);
let lhs_op = factory.try_build_item_statement(tokens, dict)?;
assert_token_raw(Token::Plus, &mut end_tokens)?;
let rhs_op = factory.try_build_item_statement(&mut end_tokens, dict)?;
tokens.extend(end_tokens);
assert_token_raw(Token::Plus, tokens)?;
let rhs_op = factory.try_build_item_statement(tokens, dict)?;
Ok(AddItemOp {
lhs: lhs_op,
rhs: rhs_op,
@ -90,6 +91,8 @@ fn first_plus(tokens: &VecDeque<Token>) -> Option<usize> {
bracket_depth += 1;
} else if token.is_close_bracket() && bracket_depth != 0 {
bracket_depth -= 1;
} else if token.is_comma() && bracket_depth == 0 {
return None;
}
}
None

10
interpreter/src/lang/vocabulary/item_ops/brackets.rs
View file

@ -1,7 +1,7 @@
use std::collections::VecDeque;
use std::convert::AsRef;
use crate::lang::utility::{assert_token_raw, assert_token_raw_back};
use crate::lang::utility::assert_token_raw;
use crate::lang::LanguageDictionary;
use crate::lang::{ItemBlockFactory, ItemOp, ItemOpFactory};
use crate::lang::{RuntimeMsg, SyntaxError};
@ -13,9 +13,8 @@ pub struct BracketsItemOpFactory;
impl ItemOpFactory<Box<dyn ItemOp>> for BracketsItemOpFactory {
fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
tokens.len() >= 2
!tokens.is_empty()
&& tokens[0].is_open_bracket()
&& tokens[tokens.len() - 1].is_close_bracket()
}
fn build_item_op(
@ -25,8 +24,9 @@ impl ItemOpFactory<Box<dyn ItemOp>> for BracketsItemOpFactory {
dict: &LanguageDictionary,
) -> Result<Box<dyn ItemOp>, SyntaxError> {
assert_token_raw(Token::OpenBracket, tokens)?;
assert_token_raw_back(Token::CloseBracket, tokens)?;
factory.try_build_item_statement(tokens, dict)
let inner = factory.try_build_item_statement(tokens, dict)?;
assert_token_raw(Token::CloseBracket, tokens)?;
Ok(inner)
}
}

6
interpreter/src/lang/vocabulary/item_ops/branch.rs
View file

@ -126,10 +126,8 @@ impl ItemOpFactory<BranchItemOp> for BranchItemOpFactory {
let end_tokens = tokens.split_off(next_close_curly);
let mut inner_if_ops = Vec::new();
while !tokens.is_empty() {
if let Some(next_comma) = find_next_comma(tokens) {
let end_tokens = tokens.split_off(next_comma);
if find_next_comma(tokens).is_some() {
inner_if_ops.push(factory.try_build_item_statement(tokens, dict)?);
tokens.extend(end_tokens);
assert_token_raw(Token::Comma, tokens)?;
} else {
inner_if_ops.push(factory.try_build_item_statement(tokens, dict)?);
@ -234,6 +232,8 @@ fn find_next_comma(tokens: &VecDeque<Token>) -> Option<usize> {
let token = &tokens[i];
if token.is_comma() && bracket_depth == 0 && curly_depth == 0 {
return Some(i);
} else if token.is_comma() && (bracket_depth < 0 || curly_depth < 0) {
return None;
} else if token.is_open_bracket() {
bracket_depth += 1;
} else if token.is_close_bracket() && bracket_depth != 0 {

3
interpreter/src/lang/vocabulary/item_ops/compare.rs
View file

@ -123,6 +123,9 @@ fn find_first_comparison(tokens: &VecDeque<Token>) -> Option<usize> {
{
return Some(i);
}
},
Token::Comma if curly_depth == 0 && bracket_depth == 0 => {
return None;
}
_ => {}
}

2
interpreter/src/lang/vocabulary/item_ops/constant.rs
View file

@ -38,7 +38,7 @@ pub struct ConstantItemOpFactory;
impl ItemOpFactory<ConstantItemOp> for ConstantItemOpFactory {
fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
tokens.len() == 1 && check_is_type(&tokens[0])
!tokens.is_empty() && check_is_type(&tokens[0])
}
fn build_item_op(

11
interpreter/src/lang/vocabulary/item_ops/constructor.rs
View file

@ -3,7 +3,7 @@ use std::collections::VecDeque;
use std::fmt::{Debug, Display, Error, Formatter};
use crate::lang::utility::{
assert_name, assert_token, assert_token_raw, assert_token_raw_back, check_name,
assert_name, assert_token, assert_token_raw, check_name,
};
use crate::lang::LanguageDictionary;
use crate::lang::{ItemBlockFactory, ItemOp, ItemOpFactory};
@ -83,9 +83,11 @@ impl ItemOpFactory<ConstructorItemOp> for ConstructorItemOpFactory {
) -> Result<ConstructorItemOp, SyntaxError> {
assert_name("Item", tokens)?;
assert_token_raw(Token::OpenBracket, tokens)?;
assert_token_raw_back(Token::CloseBracket, tokens)?;
let mut field_descriptors = Vec::new();
while !tokens.is_empty() {
if tokens[0].is_close_bracket() {
break;
}
let field_name = assert_token(
|t| match t {
Token::Name(n) => Some(n),
@ -96,10 +98,8 @@ impl ItemOpFactory<ConstructorItemOp> for ConstructorItemOpFactory {
)?;
assert_token_raw(Token::Equals, tokens)?;
let field_val;
if let Some(comma_pos) = find_next_comma(tokens) {
let end_tokens = tokens.split_off(comma_pos);
if find_next_comma(tokens).is_some() {
field_val = factory.try_build_item_statement(tokens, dict)?;
tokens.extend(end_tokens);
assert_token_raw(Token::Comma, tokens)?;
} else {
field_val = factory.try_build_item_statement(tokens, dict)?;
@ -109,6 +109,7 @@ impl ItemOpFactory<ConstructorItemOp> for ConstructorItemOpFactory {
value: field_val,
});
}
assert_token_raw(Token::CloseBracket, tokens)?;
Ok(ConstructorItemOp {
fields: field_descriptors,
})

4
interpreter/src/lang/vocabulary/item_ops/empty.rs
View file

@ -36,10 +36,8 @@ pub struct EmptyItemOpFactory;
impl ItemOpFactory<EmptyItemOp> for EmptyItemOpFactory {
fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
tokens.len() == 3
!tokens.is_empty()
&& check_name("empty", &tokens[0])
&& tokens[1].is_open_bracket()
&& tokens[2].is_close_bracket()
}
fn build_item_op(

4
interpreter/src/lang/vocabulary/item_ops/field_assign.rs
View file

@ -61,12 +61,12 @@ pub struct FieldAssignItemOpFactory;
impl ItemOpFactory<FieldAssignItemOp> for FieldAssignItemOpFactory {
fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
(tokens.len() > 4
(tokens.len() > 3
&& tokens[0].is_name()
&& tokens[1].is_dot()
&& tokens[2].is_name()
&& tokens[3].is_equals())
|| (tokens.len() > 3
|| (tokens.len() > 2
&& tokens[0].is_dot()
&& tokens[1].is_name()
&& tokens[2].is_equals())

11
interpreter/src/lang/vocabulary/item_ops/logical_and.rs
View file

@ -73,11 +73,12 @@ impl ItemOpFactory<AndItemOp> for AndItemOpFactory {
dict: &LanguageDictionary,
) -> Result<AndItemOp, SyntaxError> {
let and_location = first_and(tokens).unwrap();
let mut end_tokens = tokens.split_off(and_location);
let end_tokens = tokens.split_off(and_location);
let lhs_op = factory.try_build_item_statement(tokens, dict)?;
assert_token_raw(Token::Ampersand, &mut end_tokens)?;
assert_token_raw(Token::Ampersand, &mut end_tokens)?;
let rhs_op = factory.try_build_item_statement(&mut end_tokens, dict)?;
tokens.extend(end_tokens);
assert_token_raw(Token::Ampersand, tokens)?;
assert_token_raw(Token::Ampersand, tokens)?;
let rhs_op = factory.try_build_item_statement(tokens, dict)?;
Ok(AndItemOp {
lhs: lhs_op,
rhs: rhs_op,
@ -95,6 +96,8 @@ fn first_and(tokens: &VecDeque<Token>) -> Option<usize> {
bracket_depth += 1;
} else if token.is_close_bracket() && bracket_depth != 0 {
bracket_depth -= 1;
} else if token.is_comma() && bracket_depth == 0 {
return None;
}
}
None

11
interpreter/src/lang/vocabulary/item_ops/logical_or.rs
View file

@ -73,11 +73,12 @@ impl ItemOpFactory<OrItemOp> for OrItemOpFactory {
dict: &LanguageDictionary,
) -> Result<OrItemOp, SyntaxError> {
let or_location = first_or(tokens).unwrap();
let mut end_tokens = tokens.split_off(or_location);
let end_tokens = tokens.split_off(or_location);
let lhs_op = factory.try_build_item_statement(tokens, dict)?;
assert_token_raw(Token::Pipe, &mut end_tokens)?;
assert_token_raw(Token::Pipe, &mut end_tokens)?;
let rhs_op = factory.try_build_item_statement(&mut end_tokens, dict)?;
tokens.extend(end_tokens);
assert_token_raw(Token::Pipe, tokens)?;
assert_token_raw(Token::Pipe, tokens)?;
let rhs_op = factory.try_build_item_statement(tokens, dict)?;
Ok(OrItemOp {
lhs: lhs_op,
rhs: rhs_op,
@ -95,6 +96,8 @@ fn first_or(tokens: &VecDeque<Token>) -> Option<usize> {
bracket_depth += 1;
} else if token.is_close_bracket() && bracket_depth != 0 {
bracket_depth -= 1;
} else if token.is_comma() && bracket_depth == 0 {
return None;
}
}
None

2
interpreter/src/lang/vocabulary/item_ops/mod.rs
View file

@ -13,6 +13,7 @@ mod logical_or;
mod negate;
mod not;
mod remove_variable;
mod retrieve_field;
mod retrieve_variable;
mod string_interpolate;
mod subtract;
@ -34,6 +35,7 @@ pub use logical_or::{OrItemOp, OrItemOpFactory};
pub use negate::{NegateItemOp, NegateItemOpFactory};
pub use not::{NotItemOp, NotItemOpFactory};
pub use remove_variable::{RemoveItemOp, RemoveItemOpFactory};
pub use retrieve_field::FieldRetrieveItemOpFactory;
pub use retrieve_variable::{VariableRetrieveItemOp, VariableRetrieveItemOpFactory};
pub use string_interpolate::{InterpolateStringItemOp, InterpolateStringItemOpFactory};
pub use subtract::{SubtractItemOp, SubtractItemOpFactory};

2
interpreter/src/lang/vocabulary/item_ops/negate.rs
View file

@ -46,7 +46,7 @@ pub struct NegateItemOpFactory;
impl ItemOpFactory<NegateItemOp> for NegateItemOpFactory {
fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
tokens.len() >= 2 && tokens[0].is_minus()
!tokens.is_empty() && tokens[0].is_minus()
}
fn build_item_op(

2
interpreter/src/lang/vocabulary/item_ops/not.rs
View file

@ -46,7 +46,7 @@ pub struct NotItemOpFactory;
impl ItemOpFactory<NotItemOp> for NotItemOpFactory {
fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
tokens.len() >= 2 && tokens[0].is_exclamation()
!tokens.is_empty() && tokens[0].is_exclamation()
}
fn build_item_op(

4
interpreter/src/lang/vocabulary/item_ops/remove_variable.rs
View file

@ -57,7 +57,7 @@ pub struct RemoveItemOpFactory;
impl ItemOpFactory<RemoveItemOp> for RemoveItemOpFactory {
fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
(tokens.len() == 2 || tokens.len() == 4) && check_name("remove", &tokens[0])
!tokens.is_empty() && check_name("remove", &tokens[0])
}
fn build_item_op(
@ -75,7 +75,7 @@ impl ItemOpFactory<RemoveItemOp> for RemoveItemOpFactory {
Token::Name("variable_name".into()),
tokens,
)?;
let field_opt = if tokens.is_empty() {
let field_opt = if tokens.is_empty() || !tokens[0].is_dot() {
None
} else {
assert_token_raw(Token::Dot, tokens)?;

49
interpreter/src/lang/vocabulary/item_ops/retrieve_field.rs Normal file
View file

@ -0,0 +1,49 @@
use std::collections::VecDeque;
use crate::lang::utility::{assert_token, assert_token_raw};
use crate::lang::LanguageDictionary;
use crate::lang::{ItemBlockFactory, ItemOpFactory};
use crate::lang::SyntaxError;
use crate::tokens::Token;
use super::VariableRetrieveItemOp;
pub struct FieldRetrieveItemOpFactory;
impl ItemOpFactory<VariableRetrieveItemOp> for FieldRetrieveItemOpFactory {
fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
tokens.len() > 2
&& tokens[0].is_name()
&& tokens[1].is_dot()
&& tokens[2].is_name()
}
fn build_item_op(
&self,
tokens: &mut VecDeque<Token>,
_factory: &ItemBlockFactory,
_dict: &LanguageDictionary,
) -> Result<VariableRetrieveItemOp, SyntaxError> {
let var_name = assert_token(
|t| match t {
Token::Name(s) => Some(s),
_ => None,
},
Token::Name("variable_name".into()),
tokens,
)?;
assert_token_raw(Token::Dot, tokens)?;
let field = assert_token(
|t| match t {
Token::Name(s) => Some(s),
_ => None,
},
Token::Name("field_name".into()),
tokens,
)?;
Ok(VariableRetrieveItemOp {
variable_name: var_name,
field_name: Some(field),
})
}
}

27
interpreter/src/lang/vocabulary/item_ops/retrieve_variable.rs
View file

@ -2,7 +2,7 @@ use core::ops::Deref;
use std::collections::VecDeque;
use std::fmt::{Debug, Display, Error, Formatter};
use crate::lang::utility::{assert_token, assert_token_raw};
use crate::lang::utility::assert_token;
use crate::lang::LanguageDictionary;
use crate::lang::{ItemBlockFactory, ItemOp, ItemOpFactory};
use crate::lang::{RuntimeMsg, SyntaxError};
@ -12,8 +12,8 @@ use crate::Context;
#[derive(Debug)]
pub struct VariableRetrieveItemOp {
variable_name: String,
field_name: Option<String>,
pub(super) variable_name: String,
pub(super) field_name: Option<String>,
}
impl Deref for VariableRetrieveItemOp {
@ -65,11 +65,7 @@ pub struct VariableRetrieveItemOpFactory;
impl ItemOpFactory<VariableRetrieveItemOp> for VariableRetrieveItemOpFactory {
fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
(tokens.len() == 1 && tokens[0].is_name())
|| (tokens.len() == 3
&& tokens[0].is_name()
&& tokens[1].is_dot()
&& tokens[2].is_name())
!tokens.is_empty() && tokens[0].is_name()
}
fn build_item_op(
@ -86,22 +82,9 @@ impl ItemOpFactory<VariableRetrieveItemOp> for VariableRetrieveItemOpFactory {
Token::Name("variable_name".into()),
tokens,
)?;
let field_opt = if tokens.is_empty() {
None
} else {
assert_token_raw(Token::Dot, tokens)?;
Some(assert_token(
|t| match t {
Token::Name(s) => Some(s),
_ => None,
},
Token::Name("field_name".into()),
tokens,
)?)
};
Ok(VariableRetrieveItemOp {
variable_name: var_name,
field_name: field_opt,
field_name: None,
})
}
}

9
interpreter/src/lang/vocabulary/item_ops/subtract.rs
View file

@ -69,10 +69,11 @@ impl ItemOpFactory<SubtractItemOp> for SubtractItemOpFactory {
dict: &LanguageDictionary,
) -> Result<SubtractItemOp, SyntaxError> {
let minus_location = first_minus(tokens).unwrap();
let mut end_tokens = tokens.split_off(minus_location);
let end_tokens = tokens.split_off(minus_location);
let lhs_op = factory.try_build_item_statement(tokens, dict)?;
assert_token_raw(Token::Minus, &mut end_tokens)?;
let rhs_op = factory.try_build_item_statement(&mut end_tokens, dict)?;
tokens.extend(end_tokens);
assert_token_raw(Token::Minus, tokens)?;
let rhs_op = factory.try_build_item_statement(tokens, dict)?;
Ok(SubtractItemOp {
lhs: lhs_op,
rhs: rhs_op,
@ -90,6 +91,8 @@ fn first_minus(tokens: &VecDeque<Token>) -> Option<usize> {
bracket_depth += 1;
} else if token.is_close_bracket() && bracket_depth != 0 {
bracket_depth -= 1;
} else if token.is_comma() && bracket_depth == 0 {
return None;
}
}
None

2
interpreter/src/lang/vocabulary/item_ops/variable_declare.rs
View file

@ -72,7 +72,7 @@ impl ItemOpFactory<VariableDeclareItemOp> for VariableDeclareItemOpFactory {
Token::Name("variable_name".into()),
tokens,
)?;
let inner_op: Option<Box<dyn ItemOp>> = if !tokens.is_empty() {
let inner_op: Option<Box<dyn ItemOp>> = if !tokens.is_empty() && tokens[0].is_equals() {
assert_token_raw(Token::Equals, tokens)?;
Some(factory.try_build_item_statement(tokens, dict)?)
} else {

2
interpreter/src/lang/vocabulary/mod.rs
View file

@ -12,6 +12,7 @@ mod sql_query;
mod sql_simple_query;
mod union;
mod variable_assign;
mod variable_iter;
pub use empties::{empties_function_factory, EmptiesStatementFactory};
pub use empty::{empty_function_factory, EmptyStatementFactory};
@ -26,6 +27,7 @@ pub use sql_query::{sql_function_factory, SqlStatementFactory};
pub use sql_simple_query::{simple_sql_function_factory, SimpleSqlStatementFactory};
pub use union::{union_function_factory, UnionStatementFactory};
pub use variable_assign::{AssignStatement, AssignStatementFactory};
pub use variable_iter::{VariableRetrieveStatement, VariableRetrieveStatementFactory};
pub mod filters;
pub mod item_ops;

39
interpreter/src/lang/vocabulary/repeat.rs
View file

@ -15,7 +15,6 @@ use crate::lang::{RuntimeError, SyntaxError};
pub struct RepeatStatement {
inner_statement: PseudoOp,
inner_done: bool,
context: Option<Context>,
cache: Vec<Item>,
cache_position: usize,
repetitions: usize,
@ -42,7 +41,6 @@ impl std::clone::Clone for RepeatStatement {
Self {
inner_statement: self.inner_statement.clone(),
inner_done: self.inner_done,
context: None,
cache: self.cache.clone(),
cache_position: self.cache_position,
repetitions: self.repetitions,
@ -60,11 +58,6 @@ impl Iterator for RepeatStatement {
Err(e) => return Some(Err(e)),
Ok(real) => real,
};
// give context to inner (should only occur on first run)
if self.context.is_some() {
let ctx = self.context.take().unwrap();
real_op.enter(ctx);
}
if real_op.is_resetable() {
while self.loop_forever || !self.inner_done {
if let Some(item) = real_op.next() {
@ -73,8 +66,6 @@ impl Iterator for RepeatStatement {
if !self.loop_forever {
if self.repetitions == 0 {
self.inner_done = true;
// take context from inner (should only occur when inner is no longer needed)
self.context = Some(real_op.escape());
} else {
self.repetitions -= 1;
if let Err(e) = real_op.reset() {
@ -88,17 +79,10 @@ impl Iterator for RepeatStatement {
}
}
}
if self.context.is_none() {
self.context = Some(real_op.escape());
}
None
} else {
// cache items in RepeatStatement since inner_statement cannot be reset
if !self.inner_done {
if self.context.is_some() {
let ctx = self.context.take().unwrap();
real_op.enter(ctx);
}
let inner_item = real_op.next();
match inner_item {
Some(x) => {
@ -113,7 +97,6 @@ impl Iterator for RepeatStatement {
None => {
// inner has completed it's only run
self.inner_done = true;
self.context = Some(real_op.escape());
}
}
}
@ -156,27 +139,23 @@ impl Iterator for RepeatStatement {
impl Op for RepeatStatement {
fn enter(&mut self, ctx: Context) {
self.context = Some(ctx)
self.inner_statement.try_real().unwrap().enter(ctx)
}
fn escape(&mut self) -> Context {
if self.context.is_some() {
self.context.take().unwrap()
} else {
self.inner_statement.try_real().unwrap().escape()
}
self.inner_statement.try_real().unwrap().escape()
}
fn is_resetable(&self) -> bool {
true
if let Ok(real_op) = self.inner_statement.try_real_ref() {
real_op.is_resetable()
} else {
false
}
}
fn reset(&mut self) -> Result<(), RuntimeError> {
let real_op = self.inner_statement.try_real()?;
if self.context.is_some() {
let ctx = self.context.take().unwrap();
real_op.enter(ctx);
}
if real_op.is_resetable() {
real_op.reset()?;
if self.original_repetitions == 0 {
@ -204,7 +183,6 @@ impl Op for RepeatStatement {
let clone = Self {
inner_statement: PseudoOp::from(self.inner_statement.try_real_ref().unwrap().dup()),
inner_done: self.original_repetitions == 0,
context: None,
cache: Vec::new(),
cache_position: 0,
repetitions: if self.original_repetitions != 0 {
@ -239,7 +217,7 @@ impl FunctionFactory<RepeatStatement> for RepeatFunctionFactory {
tokens.extend(end_tokens);
let mut count: Option<usize> = None;
let mut inner_done = false;
if !tokens.is_empty() {
if !tokens.is_empty() && tokens[0].is_comma() {
// repititions specified
assert_token_raw(Token::Comma, tokens)?;
count = Some(assert_token(
@ -264,7 +242,6 @@ impl FunctionFactory<RepeatStatement> for RepeatFunctionFactory {
Ok(RepeatStatement {
inner_statement: inner_statement.into(),
inner_done,
context: None,
cache: Vec::new(),
cache_position: 0,
repetitions: count.unwrap_or(0),

2
interpreter/src/lang/vocabulary/sorters/empty_sorter.rs
View file

@ -32,7 +32,7 @@ pub struct EmptySorterFactory;
impl SorterFactory<EmptySorter> for EmptySorterFactory {
fn is_sorter(&self, tokens: &VecDeque<&Token>) -> bool {
tokens.is_empty()
!tokens.is_empty() && tokens[0].is_close_bracket()
}
fn build_sorter(

13
interpreter/src/lang/vocabulary/union.rs
View file

@ -163,15 +163,10 @@ impl FunctionFactory<UnionStatement> for UnionFunctionFactory {
) -> Result<UnionStatement, SyntaxError> {
// union(op1, op2, ...)
let operations = repeated_tokens(
|tokens| {
if let Some(comma_pos) = next_comma(tokens) {
let end_tokens = tokens.split_off(comma_pos);
let op = dict.try_build_statement(tokens);
tokens.extend(end_tokens);
Ok(Some(PseudoOp::from(op?)))
} else {
Ok(Some(PseudoOp::from(dict.try_build_statement(tokens)?)))
}
|tokens| if tokens[0].is_close_bracket() {
Ok(None)
} else {
Ok(Some(PseudoOp::from(dict.try_build_statement(tokens)?)))
},
Token::Comma,
)

177
interpreter/src/lang/vocabulary/variable_iter.rs Normal file
View file

@ -0,0 +1,177 @@
use std::collections::VecDeque;
use std::fmt::{Debug, Display, Error, Formatter};
use std::iter::Iterator;
use crate::tokens::Token;
use crate::Context;
use crate::lang::utility::assert_token;
use crate::lang::LanguageDictionary;
use crate::lang::{BoxedOpFactory, IteratorItem, Op, OpFactory, PseudoOp};
use crate::lang::{RuntimeError, RuntimeOp, SyntaxError};
use crate::processing::general::Type;
#[derive(Debug)]
pub struct VariableRetrieveStatement {
variable_name: String,
context: Option<Context>,
is_tried: bool,
}
impl Display for VariableRetrieveStatement {
fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
write!(f, "{}", self.variable_name)
}
}
impl std::clone::Clone for VariableRetrieveStatement {
fn clone(&self) -> Self {
Self {
variable_name: self.variable_name.clone(),
context: None,
is_tried: self.is_tried,
}
}
}
impl Iterator for VariableRetrieveStatement {
type Item = IteratorItem;
fn next(&mut self) -> Option<Self::Item> {
if self.is_tried {
return None;
}
let var = self.context.as_mut()
.unwrap()
.variables.remove(&self.variable_name)
.map_err(|e| e.with(RuntimeOp(PseudoOp::from_printable(self))));
match var {
Ok(Type::Op(mut op)) => {
op.enter(self.context.take().unwrap());
let next_item = op.next();
self.enter(op.escape());
if let Err(e) = self.context.as_mut().unwrap().variables.declare(&self.variable_name, Type::Op(op)) {
return Some(Err(e.with(RuntimeOp(PseudoOp::from_printable(self)))));
}
next_item
},
Ok(Type::Item(item)) => {
self.is_tried = true;
if let Err(e) = self.context.as_mut().unwrap().variables.declare(&self.variable_name, Type::Item(item.clone())) {
return Some(Err(e.with(RuntimeOp(PseudoOp::from_printable(self)))));
}
Some(Ok(item))
},
Ok(Type::Primitive(p)) => {
self.is_tried = true;
let err_msg = format!("Cannot iterate over primitive `{}` ({})", self.variable_name, p);
if let Err(e) = self.context.as_mut().unwrap().variables.declare(&self.variable_name, Type::Primitive(p)) {
return Some(Err(e.with(RuntimeOp(PseudoOp::from_printable(self)))));
}
Some(Err(
RuntimeError {
line: 0,
op: PseudoOp::from_printable(self),
msg: err_msg,
}
))
}
Err(e) => {
self.is_tried = true;
Some(Err(e))
}
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
(0, None)
}
}
impl Op for VariableRetrieveStatement {
fn enter(&mut self, ctx: Context) {
self.context = Some(ctx)
}
fn escape(&mut self) -> Context {
self.context.take().unwrap()
}
fn is_resetable(&self) -> bool {
if let Some(ctx) = &self.context {
let var = ctx.variables.get(&self.variable_name);
match var {
Ok(Type::Op(op)) => op.is_resetable(),
Ok(Type::Primitive(_)) => false,
Ok(Type::Item(_)) => true,
Err(_) => false,
}
} else {
false
}
}
fn reset(&mut self) -> Result<(), RuntimeError> {
self.is_tried = false;
let runtime_op = RuntimeOp(PseudoOp::from_printable(self));
let var = self.context.as_mut()
.unwrap()
.variables.get_mut(&self.variable_name)
.map_err(|e| e.with(runtime_op))?;
if let Type::Op(op) = var {
op.reset()?;
}
Ok(())
}
fn dup(&self) -> Box<dyn Op> {
Box::new(Self {
variable_name: self.variable_name.clone(),
context: None,
is_tried: false,
})
}
}
pub struct VariableRetrieveStatementFactory;
impl OpFactory<VariableRetrieveStatement> for VariableRetrieveStatementFactory {
fn is_op(&self, tokens: &VecDeque<Token>) -> bool {
!tokens.is_empty() && tokens[0].is_name()
}
fn build_op(
&self,
tokens: &mut VecDeque<Token>,
_dict: &LanguageDictionary,
) -> Result<VariableRetrieveStatement, SyntaxError> {
let name = assert_token(
|t| match t {
Token::Name(s) => Some(s),
_ => None,
},
Token::Name("variable_name".into()),
tokens,
)?;
Ok(VariableRetrieveStatement {
variable_name: name,
context: None,
is_tried: false,
})
}
}
impl BoxedOpFactory for VariableRetrieveStatementFactory {
fn build_op_boxed(
&self,
tokens: &mut VecDeque<Token>,
dict: &LanguageDictionary,
) -> Result<Box<dyn Op>, SyntaxError> {
self.build_box(tokens, dict)
}
fn is_op_boxed(&self, tokens: &VecDeque<Token>) -> bool {
self.is_op(tokens)
}
}

23
interpreter/tests/single_line.rs
View file

@ -315,7 +315,14 @@ fn execute_replacefilter_line() -> Result<(), InterpreterError> {
true,
)?;
execute_single_line(
"files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).(if 4: item.() else files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).(0 || 1).(if 200: files() else repeat(item.(), 2)))",
r#"files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)
.(
if 4: item.()
else files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)
.(0 || 1)
.(if 200: files() else repeat(item.(), 2))
)"#,
false,
true,
)
@ -323,11 +330,6 @@ fn execute_replacefilter_line() -> Result<(), InterpreterError> {
#[test]
fn execute_emptysort_line() -> Result<(), InterpreterError> {
execute_single_line(
"files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).sort()",
false,
true,
)?;
execute_single_line(
"files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)~()",
false,
@ -367,7 +369,7 @@ fn execute_fieldsort_line() -> Result<(), InterpreterError> {
true,
)?;
execute_single_line(
"files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).sort(.not_a_field)",
"files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)~(.not_a_field)",
false,
true,
)
@ -661,6 +663,13 @@ fn execute_bracketsitemop_line() -> Result<(), InterpreterError> {
#[test]
fn execute_stringifyitemop_line() -> Result<(), InterpreterError> {
execute_single_line(
"files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).{
item.title = ~`test out: {track_number}` item
}",
false,
true,
)?;
execute_single_line(
"files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).{
item.filepath = ~`test out: {test}` item,

1
player/Cargo.toml
View file

@ -15,6 +15,7 @@ muss-interpreter = { path = "../interpreter", version = "0.9.0" }
[target.'cfg(target_os = "linux")'.dependencies]
#dbus = { version = "^0.9" }
# TODO replace mpris-player, maybe with mpris2-zbus https://github.com/pop-os/dbus-settings-bindings/tree/main/mpris2
mpris-player = { version = "^0.6", path = "../mpris-player", optional = true }
base64 = { version = "0.13", optional = true }