Refactor operation emitter to distinguish filters/sorters/itemops from standalone functions

interpreter/src/faye.rs

@@ -134,6 +134,7 @@ where
             Ok(stmt) => stmt,
             Err(e) => return Some(Err(error_with_ctx(e, self.tokenizer.current_line()))),
         };
+        //println!("Final parsed op: {}", stmt);
         #[cfg(debug_assertions)]
         if !self.buffer.is_empty() {
             panic!("Token buffer was not emptied! (rem: {:?})", self.buffer)

interpreter/src/interpretor.rs

@@ -4,10 +4,9 @@ use super::lang::LanguageDictionary;
 pub(crate) fn standard_vocab(vocabulary: &mut LanguageDictionary) {
     vocabulary
         // filters
-        .add(crate::lang::vocabulary::filters::empty_filter())
+        .add_transform(crate::lang::vocabulary::filters::empty_filter())
-        .add(crate::lang::vocabulary::filters::unique_filter()) // accepts .(unique)
+        .add_transform(crate::lang::vocabulary::filters::range_filter())
-        .add(crate::lang::vocabulary::filters::range_filter())
+        .add_transform( // accepts any .(.something)
-        .add( // accepts any .(.something)
             crate::lang::vocabulary::filters::field::FieldFilterBlockFactory::new()
                 .push(crate::lang::vocabulary::filters::field::FieldFilterComparisonFactory)
                 .push(crate::lang::vocabulary::filters::field::FieldFilterMaybeFactory)
@@ -15,39 +14,41 @@ pub(crate) fn standard_vocab(vocabulary: &mut LanguageDictionary) {
                 .push(crate::lang::vocabulary::filters::field::FieldRegexFilterFactory)
                 .to_statement_factory()
         )
-        .add(crate::lang::vocabulary::filters::index_filter())
+        .add_transform(crate::lang::vocabulary::filters::unique_field_filter())
-        .add(crate::lang::vocabulary::filters::unique_field_filter())
+        .add_transform(crate::lang::vocabulary::filters::unique_filter())
-        .add(crate::lang::vocabulary::filters::nonempty_filter())
+        .add_transform(crate::lang::vocabulary::filters::nonempty_filter())
+        .add_transform(crate::lang::vocabulary::filters::index_filter())
         // sorters
-        .add(crate::lang::vocabulary::sorters::empty_sort())
+        .add_transform(crate::lang::vocabulary::sorters::empty_sort())
-        .add(crate::lang::vocabulary::sorters::shuffle_sort()) // accepts ~(~shuffle)
+        .add_transform(crate::lang::vocabulary::sorters::shuffle_sort()) // accepts ~(~shuffle)
-        .add(crate::lang::vocabulary::sorters::bliss_sort())
+        .add_transform(crate::lang::vocabulary::sorters::bliss_sort())
-        .add(crate::lang::vocabulary::sorters::bliss_next_sort())
+        .add_transform(crate::lang::vocabulary::sorters::bliss_next_sort())
-        .add(crate::lang::vocabulary::sorters::radio_sort())
+        .add_transform(crate::lang::vocabulary::sorters::radio_sort())
-        .add(crate::lang::vocabulary::sorters::field_sort()) // accepts any ~(.name)
+        .add_transform(crate::lang::vocabulary::sorters::field_sort()) // accepts any ~(.name)
         // iter blocks
-        .add(
+        .add_transform(
             crate::lang::ItemBlockFactory::new()
-                .push(crate::lang::vocabulary::item_ops::ConstantItemOpFactory)
-                .push(crate::lang::vocabulary::item_ops::VariableAssignItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::FieldAssignItemOpFactory)
-                .push(crate::lang::vocabulary::item_ops::FileItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::VariableAssignItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::VariableDeclareItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::FileItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::InterpolateStringItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::BranchItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::IterItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::ConstructorItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::EmptyItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::RemoveItemOpFactory)
-                .push(crate::lang::vocabulary::item_ops::VariableRetrieveItemOpFactory)
-                .push(crate::lang::vocabulary::item_ops::NegateItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::NotItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::CompareItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::NegateItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::AddItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::SubtractItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::OrItemOpFactory)
                 .push(crate::lang::vocabulary::item_ops::AndItemOpFactory)
-                .push(crate::lang::vocabulary::item_ops::BracketsItemOpFactory),
+                .push(crate::lang::vocabulary::item_ops::BracketsItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::FieldRetrieveItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::ConstantItemOpFactory)
+                .push(crate::lang::vocabulary::item_ops::VariableRetrieveItemOpFactory)
         )
         // functions and misc
         // functions don't enforce bracket coherence
@@ -64,5 +65,6 @@ pub(crate) fn standard_vocab(vocabulary: &mut LanguageDictionary) {
         .add(crate::lang::vocabulary::empties_function_factory())
         .add(crate::lang::vocabulary::reset_function_factory())
         .add(crate::lang::vocabulary::union_function_factory())
-        .add(crate::lang::vocabulary::intersection_function_factory());
+        .add(crate::lang::vocabulary::intersection_function_factory())
+        .add(crate::lang::vocabulary::VariableRetrieveStatementFactory);
 }

interpreter/src/lang/dictionary.rs

@@ -1,26 +1,33 @@
 use std::collections::VecDeque;
 
 use super::SyntaxError;
-use super::{BoxedOpFactory, Op};
+use super::{BoxedOpFactory, Op, BoxedTransformOpFactory};
 use crate::tokens::Token;
 
 pub struct LanguageDictionary {
-    vocabulary: Vec<Box<dyn BoxedOpFactory>>,
+    root_vocabulary: Vec<Box<dyn BoxedOpFactory>>,
+    transform_vocabulary: Vec<Box<dyn BoxedTransformOpFactory>>,
 }
 
 impl LanguageDictionary {
     pub fn add<T: BoxedOpFactory + 'static>(&mut self, factory: T) -> &mut Self {
-        self.vocabulary
+        self.root_vocabulary
             .push(Box::new(factory) as Box<dyn BoxedOpFactory>);
         self
     }
 
-    pub fn try_build_statement(
+    pub fn add_transform<T: BoxedTransformOpFactory + 'static>(&mut self, factory: T) -> &mut Self {
+        self.transform_vocabulary
+            .push(Box::new(factory) as Box<dyn BoxedTransformOpFactory>);
+        self
+    }
+
+    fn try_build_root_statement(
         &self,
         tokens: &mut VecDeque<Token>,
     ) -> Result<Box<dyn Op>, SyntaxError> {
-        //println!("try_build_statement with tokens {:?}", tokens);
+        //println!("building root op with tokens {:?}", tokens);
-        for factory in &self.vocabulary {
+        for factory in &self.root_vocabulary {
             if factory.is_op_boxed(tokens) {
                 return factory.build_op_boxed(tokens, self);
             }
@@ -40,9 +47,47 @@ impl LanguageDictionary {
         })
     }
 
+    fn try_build_transformed_statement(
+        &self,
+        mut op: Box<dyn Op>,
+        tokens: &mut VecDeque<Token>,
+    ) -> Result<Box<dyn Op>, SyntaxError> {
+        //println!("building transformer for op {} with tokens {:?}", op, tokens);
+        let mut op_found = true;
+        while op_found && !tokens.is_empty() {
+            (op, op_found) = self.try_build_one_transform(op, tokens)?;
+        }
+        //println!("built transformed op {}, remaining tokens {:?}", op, tokens);
+        Ok(op)
+    }
+
+    pub fn try_build_one_transform(
+        &self,
+        mut op: Box<dyn Op>,
+        tokens: &mut VecDeque<Token>,
+    ) -> Result<(Box<dyn Op>, bool), SyntaxError> {
+        for factory in &self.transform_vocabulary {
+            if factory.is_transform_op(tokens) {
+                op = factory.build_transform_op(tokens, self, op)?;
+                return Ok((op, true))
+            }
+        }
+        Ok((op, false))
+    }
+
+    pub fn try_build_statement(
+        &self,
+        tokens: &mut VecDeque<Token>,
+    ) -> Result<Box<dyn Op>, SyntaxError> {
+        let root = self.try_build_root_statement(tokens)?;
+        //println!("built root op {}, remaining tokens {:?}", root, tokens);
+        self.try_build_transformed_statement(root, tokens)
+    }
+
     pub fn new() -> Self {
         Self {
-            vocabulary: Vec::new(),
+            root_vocabulary: Vec::new(),
+            transform_vocabulary: Vec::new(),
         }
     }
 
@@ -57,7 +102,8 @@ impl LanguageDictionary {
 impl Default for LanguageDictionary {
     fn default() -> Self {
         Self {
-            vocabulary: Vec::new(),
+            root_vocabulary: Vec::new(),
+            transform_vocabulary: Vec::new(),
         }
     }
 }

interpreter/src/lang/filter.rs

@@ -3,11 +3,11 @@ use std::fmt::{Debug, Display, Error, Formatter};
 use std::iter::Iterator;
 use std::marker::PhantomData;
 
-use crate::lang::utility::{assert_name, assert_token, assert_token_raw, check_name};
+use crate::lang::utility::{assert_name, assert_token_raw, check_name};
 use crate::lang::FilterReplaceStatement;
 use crate::lang::LanguageDictionary;
 use crate::lang::SingleItem;
-use crate::lang::{BoxedOpFactory, IteratorItem, Op, PseudoOp};
+use crate::lang::{BoxedTransformOpFactory, IteratorItem, Op, PseudoOp};
 use crate::lang::{RuntimeError, RuntimeMsg, RuntimeOp, SyntaxError};
 use crate::processing::general::Type;
 use crate::tokens::Token;
@@ -34,25 +34,10 @@ pub trait FilterFactory<P: FilterPredicate + 'static>: Send + Sync {
     ) -> Result<P, SyntaxError>;
 }
 
-#[derive(Debug, Clone)]
-pub(super) enum VariableOrOp {
-    Variable(String),
-    Op(PseudoOp),
-}
-
-impl Display for VariableOrOp {
-    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
-        match self {
-            Self::Variable(s) => write!(f, "{}", s),
-            Self::Op(op) => write!(f, "{}", op),
-        }
-    }
-}
-
 #[derive(Debug)]
 pub struct FilterStatement<P: FilterPredicate + 'static> {
     predicate: P,
-    iterable: VariableOrOp,
+    iterable: PseudoOp,
     context: Option<Context>,
     other_filters: Option<PseudoOp>,
     is_failing: bool,
@@ -75,7 +60,7 @@ impl<P: FilterPredicate + 'static> Display for FilterStatement<P> {
         if let Some(other_filters) = &self.other_filters {
             write!(
                 f,
-                "{}.({} || (like) {})",
+                "{}.({} || (retconned) {})",
                 self.iterable, self.predicate, other_filters
             )
         } else {
@@ -95,20 +80,8 @@ impl<P: FilterPredicate + 'static> Op for FilterStatement<P> {
 
     fn is_resetable(&self) -> bool {
         let is_iterable_resetable = match &self.iterable {
-            VariableOrOp::Variable(s) => {
+            PseudoOp::Real(op) => op.is_resetable(),
-                if self.context.is_some() {
+            PseudoOp::Fake(_) => false,
-                    let var = self.context.as_ref().unwrap().variables.get_opt(s);
-                    if let Some(Type::Op(var)) = var {
-                        var.is_resetable()
-                    } else {
-                        false
-                    }
-                } else {
-                    true
-                } // ASSUMPTION
-            }
-            VariableOrOp::Op(PseudoOp::Real(op)) => op.is_resetable(),
-            VariableOrOp::Op(PseudoOp::Fake(_)) => false,
         };
         let is_other_filter_resetable =
             if let Some(PseudoOp::Real(other_filter)) = &self.other_filters {
@@ -126,45 +99,13 @@ impl<P: FilterPredicate + 'static> Op for FilterStatement<P> {
             .reset()
             .map_err(|x| x.with(RuntimeOp(fake.clone())))?;
         match &mut self.iterable {
-            VariableOrOp::Variable(s) => {
+            PseudoOp::Real(op) => {
-                if self.context.as_mut().unwrap().variables.exists(s) {
-                    let mut var = self
-                        .context
-                        .as_mut()
-                        .unwrap()
-                        .variables
-                        .remove(s)
-                        .map_err(|e| e.with(RuntimeOp(fake.clone())))?;
-                    let result = if let Type::Op(var) = &mut var {
-                        var.enter(self.context.take().unwrap());
-                        let result = var.reset();
-                        self.context = Some(var.escape());
-                        result
-                    } else {
-                        Err(RuntimeError {
-                            line: 0,
-                            op: fake.clone(),
-                            msg: "Cannot reset non-iterable filter variable".to_string(),
-                        })
-                    };
-                    self.context
-                        .as_mut()
-                        .unwrap()
-                        .variables
-                        .declare(s, var)
-                        .map_err(|e| e.with(RuntimeOp(fake)))?;
-                    result
-                } else {
-                    Ok(())
-                }
-            }
-            VariableOrOp::Op(PseudoOp::Real(op)) => {
                 op.enter(self.context.take().unwrap());
                 let result = op.reset();
                 self.context = Some(op.escape());
                 result
             }
-            VariableOrOp::Op(PseudoOp::Fake(_)) => Err(RuntimeError {
+            PseudoOp::Fake(_) => Err(RuntimeError {
                 line: 0,
                 op: fake,
                 msg: "Cannot reset PseudoOp::Fake filter".to_string(),
@@ -183,10 +124,7 @@ impl<P: FilterPredicate + 'static> Op for FilterStatement<P> {
     fn dup(&self) -> Box<dyn Op> {
         Box::new(Self {
             predicate: self.predicate.clone(),
-            iterable: match &self.iterable {
+            iterable: self.iterable.try_real_ref().unwrap().dup().into(),
-                VariableOrOp::Variable(s) => VariableOrOp::Variable(s.clone()),
-                VariableOrOp::Op(op) => VariableOrOp::Op(op.try_real_ref().unwrap().dup().into()),
-            },
             context: None,
             other_filters: self
                 .other_filters
@@ -199,9 +137,7 @@ impl<P: FilterPredicate + 'static> Op for FilterStatement<P> {
 
 impl <P: FilterPredicate + 'static> FilterStatement<P> {
     fn next_item(&mut self) -> Option<IteratorItem> {
-        let fake_op = PseudoOp::from_printable(self);
+        match self.iterable.try_real() {
-        match &mut self.iterable {
-             VariableOrOp::Op(op) => match op.try_real() {
             Ok(real_op) => {
                 let ctx = self.context.take().unwrap();
                 real_op.enter(ctx);
@@ -210,46 +146,6 @@ impl <P: FilterPredicate + 'static> FilterStatement<P> {
                 item
             }
             Err(e) => return Some(Err(e)),
-             },
-             VariableOrOp::Variable(variable_name) => {
-                let mut variable = match self
-                    .context
-                    .as_mut()
-                    .unwrap()
-                    .variables
-                    .remove(variable_name)
-                {
-                    Ok(Type::Op(op)) => op,
-                    Ok(x) => {
-                        return Some(Err(RuntimeError {
-                            line: 0,
-                            op: fake_op,
-                            msg: format!(
-                                "Expected operation/iterable type in variable {}, got {}",
-                                &variable_name, x
-                            ),
-                        }))
-                    }
-                    Err(e) => {
-                        self.is_failing = true;
-                        return Some(Err(e.with(RuntimeOp(PseudoOp::from_printable(self)))));
-                    },
-                };
-
-                variable.enter(self.context.take().unwrap());
-                let item = variable.next();
-                self.context = Some(variable.escape());
-                if let Err(e) = self
-                    .context
-                    .as_mut()
-                    .unwrap()
-                    .variables
-                    .declare(variable_name, Type::Op(variable))
-                {
-                    return Some(Err(e.with(RuntimeOp(PseudoOp::from_printable(self)))));
-                }
-                item
-            }
         }
     }
 }
@@ -258,6 +154,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterStatement<P> {
     type Item = IteratorItem;
 
     fn next(&mut self) -> Option<Self::Item> {
+        //println!("In FilterStatement {}", self.predicate);
         if (self.predicate.is_complete() && self.other_filters.is_none()) || self.is_failing {
             return None;
         }
@@ -278,6 +175,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterStatement<P> {
                     if let Some(inner) = &mut self.other_filters {
                         // handle other filters
                         // make fake inner item
+                        //println!("Making fake inner variable `{}`", INNER_VARIABLE_NAME);
                         let single_op = SingleItem::new_ok(item.clone());
                         let preexisting_var = self.context.as_mut().unwrap()
                             .variables
@@ -310,18 +208,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterStatement<P> {
     }
 
     fn size_hint(&self) -> (usize, Option<usize>) {
-        match &self.iterable {
+        self.iterable.try_real_ref().map(|x| x.size_hint()).ok().unwrap_or((0, None))
-            VariableOrOp::Variable(s) => self
-                .context
-                .as_ref()
-                .and_then(|x| x.variables.get_opt(s))
-                .and_then(|x| match x {
-                    Type::Op(op) => Some(op.size_hint()),
-                    _ => None,
-                }),
-            VariableOrOp::Op(op) => op.try_real_ref().map(|x| x.size_hint()).ok(),
-        }
-        .unwrap_or((0, None))
     }
 }
 
@@ -339,130 +226,64 @@ impl<P: FilterPredicate + 'static, F: FilterFactory<P> + 'static> FilterStatemen
     }
 }
 
-impl<P: FilterPredicate + 'static, F: FilterFactory<P> + 'static> BoxedOpFactory
+impl<P: FilterPredicate + 'static, F: FilterFactory<P> + 'static> BoxedTransformOpFactory
-    for FilterStatementFactory<P, F>
+    for FilterStatementFactory<P, F> {
-{
+    fn build_transform_op(
-    #[allow(clippy::unnecessary_unwrap)]
-    fn is_op_boxed(&self, tokens: &VecDeque<Token>) -> bool {
-        let tokens_len = tokens.len();
-        if is_correct_format(tokens) {
-            let start_of_predicate = last_dot_before_open_bracket(tokens) + 2; // .(predicate)
-            if start_of_predicate > tokens_len - 1 {
-                false
-            } else {
-                let pipe_location_opt = first_double_pipe(tokens, 1);
-                if pipe_location_opt.is_some() && pipe_location_opt.unwrap() > start_of_predicate {
-                    let pipe_location = pipe_location_opt.unwrap();
-                    // filters combined by OR operations
-                    let tokens2: VecDeque<&Token> =
-                        VecDeque::from_iter(tokens.range(start_of_predicate..pipe_location));
-                    self.filter_factory.is_filter(&tokens2)
-                } else {
-                    // single filter
-                    let tokens2: VecDeque<&Token> =
-                        VecDeque::from_iter(tokens.range(start_of_predicate..tokens_len - 1));
-                    if !tokens2.is_empty() && check_name("if", tokens2[0]) {
-                        // replacement filter
-                        if let Some(colon_location) = first_colon2(&tokens2) {
-                            let tokens3 = VecDeque::from_iter(tokens.range(
-                                start_of_predicate + 1..start_of_predicate + colon_location,
-                            ));
-                            self.filter_factory.is_filter(&tokens3)
-                        } else {
-                            false
-                        }
-                    } else {
-                        // regular filter
-                        self.filter_factory.is_filter(&tokens2)
-                    }
-                }
-            }
-        } else {
-            false
-        }
-    }
-
-    fn build_op_boxed(
         &self,
         tokens: &mut VecDeque<Token>,
         dict: &LanguageDictionary,
+        op: Box<dyn Op>,
     ) -> Result<Box<dyn Op>, SyntaxError> {
-        let start_of_op = last_dot_before_open_bracket(tokens);
-        let op = if start_of_op == 1 && tokens[0].is_name() {
-            // variable_name.(predicate)
-            let variable_name = assert_token(
-                |t| match t {
-                    Token::Name(s) => Some(s),
-                    _ => None,
-                },
-                Token::Name("variable_name".into()),
-                tokens,
-            )?;
-            VariableOrOp::Variable(variable_name)
-        } else {
-            // <some other op>.(predicate)
-            //let mut new_tokens = tokens.range(0..start_of_op).map(|x| x.to_owned()).collect();
-            let end_tokens = tokens.split_off(start_of_op); // don't parse filter in inner statement
-            let inner_op = dict.try_build_statement(tokens)?;
-            tokens.extend(end_tokens);
-            VariableOrOp::Op(inner_op.into())
-        };
         assert_token_raw(Token::Dot, tokens)?;
         assert_token_raw(Token::OpenBracket, tokens)?;
         if !tokens.is_empty() && check_name("if", &tokens[0]) {
-            return {
             // replacement filter
-                //println!("Building replacement filter from tokens {:?}", tokens);
             assert_name("if", tokens)?;
-                if let Some(colon_location) = first_colon(tokens) {
-                    let end_tokens = tokens.split_off(colon_location);
             let filter = self.filter_factory.build_filter(tokens, dict)?;
-                    tokens.extend(end_tokens);
             assert_token_raw(Token::Colon, tokens)?;
             let mut else_op: Option<PseudoOp> = None;
-                    let if_op: PseudoOp;
+            let if_op: PseudoOp = dict.try_build_statement(tokens)?.into();
-                    if let Some(else_location) = first_else_not_in_bracket(tokens) {
+            if check_name("else", &tokens[0]) {
-                        let end_tokens = tokens.split_off(else_location);
+                assert_name("else", tokens)?;
+                else_op = Some(dict.try_build_statement(tokens)?.into());
+            }
+            /*if let Some(else_location) = first_else_not_in_bracket(tokens) {
+                //println!("First else found at {}; {:?}", else_location, tokens);
+                //let end_tokens = tokens.split_off(else_location);
                 // build replacement system
                 if_op = dict.try_build_statement(tokens)?.into();
-                        tokens.extend(end_tokens);
+                //tokens.extend(end_tokens);
                 assert_name("else", tokens)?;
-                        let end_tokens = tokens.split_off(tokens.len() - 1); // up to ending close bracket
+                //let end_tokens = tokens.split_off(tokens.len() - 1); // up to ending close bracket
                                                                         // build replacement system
                 else_op = Some(dict.try_build_statement(tokens)?.into());
-                        tokens.extend(end_tokens);
+                //tokens.extend(end_tokens);
             } else {
-                        let end_tokens = tokens.split_off(tokens.len() - 1);
+                //let end_tokens = tokens.split_off(tokens.len() - 1);
                 // build replacement system
                 if_op = dict.try_build_statement(tokens)?.into();
-                        tokens.extend(end_tokens);
+                //tokens.extend(end_tokens);
-                    }
+            }*/
             assert_token_raw(Token::CloseBracket, tokens)?;
             Ok(Box::new(FilterReplaceStatement {
                 predicate: filter,
-                        iterable: op,
+                iterable:  op.into(),
                 context: None,
                 op_if: if_op,
                 op_else: else_op,
                 item_cache: super::filter_replace::item_cache_deque(),
             }))
         } else {
-                    Err(SyntaxError {
+            // regular filter
-                        line: 0,
-                        token: Token::Colon,
-                        got: None,
-                    })
-                }
-            };
-        } else {
             let mut another_filter = None;
-            let (has_or, end_tokens) = if let Some(pipe_location) = first_double_pipe(tokens, 0) {
+            let has_or = first_double_pipe(tokens, 0).is_some();
+            /*let (has_or, end_tokens) = if let Some(pipe_location) = first_double_pipe(tokens, 0) {
                 (true, tokens.split_off(pipe_location)) // parse up to OR operator
             } else {
                 (false, tokens.split_off(tokens.len() - 1)) // don't parse closing bracket in filter
-            };
+            };*/
             let filter = self.filter_factory.build_filter(tokens, dict)?;
-            tokens.extend(end_tokens);
+            //tokens.extend(end_tokens);
             if has_or {
                 // recursively build other filters for OR operation
                 assert_token_raw(Token::Pipe, tokens)?;
@@ -470,66 +291,57 @@ impl<P: FilterPredicate + 'static, F: FilterFactory<P> + 'static> BoxedOpFactory
                 // emit fake filter syntax
                 tokens.push_front(Token::OpenBracket);
                 tokens.push_front(Token::Dot);
-                tokens.push_front(Token::Name(INNER_VARIABLE_NAME.into())); // impossible to obtain through parsing on purpose
+                //tokens.push_front(Token::Name(INNER_VARIABLE_NAME.into())); // impossible to obtain through parsing on purpose
-                another_filter = Some(dict.try_build_statement(tokens)?.into());
+                /*let inner_op = Box::new(crate::lang::vocabulary::VariableRetrieveStatement {
+                    variable_name: INNER_VARIABLE_NAME.into(),
+                    context: None,
+                    is_tried: false,
+                });*/
+                let mut inner_tokens = VecDeque::with_capacity(1);
+                inner_tokens.push_front(Token::Name(INNER_VARIABLE_NAME.into()));
+                let inner_op = dict.try_build_statement(&mut inner_tokens)?;
+                let (inner_op, op_transformed) = dict.try_build_one_transform(inner_op, tokens)?;
+                if !op_transformed {
+                    return Err(
+                        SyntaxError {
+                            token: Token::Name(INNER_VARIABLE_NAME.into()),
+                            got: Some(Token::Name(INNER_VARIABLE_NAME.into())),
+                            line: 0,
+                        }
+                    )
+                }
+                //println!("Built 2nd filter: {}", inner_op);
+                another_filter = Some(inner_op.into());
             } else {
+                //println!("filter tokens before final bracket: {:?}", tokens);
                 assert_token_raw(Token::CloseBracket, tokens)?; // remove closing bracket
             }
             Ok(Box::new(FilterStatement {
                 predicate: filter,
-                iterable: op,
+                iterable:  op.into(),
                 context: None,
                 other_filters: another_filter,
                 is_failing: false,
             }))
         }
     }
-}
 
-fn is_correct_format(tokens: &VecDeque<Token>) -> bool {
+    fn is_transform_op(&self, tokens: &VecDeque<Token>) -> bool {
-    let mut inside_brackets = 0;
+        let result = if tokens.len() > 2 && tokens[0].is_dot() && tokens[1].is_open_bracket() {
-    let mut open_bracket_found = false;
+            if check_name("if", &tokens[2]) {
-    let mut close_bracket = 0;
+                let tokens2: VecDeque<&Token> =
-    for i in (0..tokens.len()).rev() {
+                    VecDeque::from_iter(tokens.range(3..));
-        if tokens[i].is_close_bracket() {
+                self.filter_factory.is_filter(&tokens2)
-            if inside_brackets == 0 {
-                close_bracket = i;
-            }
-            inside_brackets += 1;
-        } else if tokens[i].is_open_bracket() {
-            if inside_brackets == 1 {
-                open_bracket_found = true;
-            } else if inside_brackets != 0 {
-                inside_brackets -= 1;
-            }
-        } else if open_bracket_found {
-            return tokens[i].is_dot() && (close_bracket == 0 || close_bracket == tokens.len() - 1);
-        }
-    }
-    false
-}
-
-fn last_dot_before_open_bracket(tokens: &VecDeque<Token>) -> usize {
-    let mut inside_brackets = 0;
-    let mut open_bracket_found = false;
-    for i in (0..tokens.len()).rev() {
-        if tokens[i].is_close_bracket() {
-            inside_brackets += 1;
-        } else if tokens[i].is_open_bracket() {
-            if inside_brackets == 1 {
-                open_bracket_found = true;
-            } else if inside_brackets != 0 {
-                inside_brackets -= 1;
-            }
-        } else if open_bracket_found {
-            if tokens[i].is_dot() {
-                return i;
             } else {
-                return 0;
+                let tokens2: VecDeque<&Token> =
+                    VecDeque::from_iter(tokens.range(2..));
+                self.filter_factory.is_filter(&tokens2)
             }
+        } else {
+            false
+        };
+        result
     }
-    }
-    0
 }
 
 fn first_double_pipe(tokens: &VecDeque<Token>, in_brackets: usize) -> Option<usize> {
@@ -554,25 +366,7 @@ fn first_double_pipe(tokens: &VecDeque<Token>, in_brackets: usize) -> Option<usi
     None
 }
 
-fn first_colon(tokens: &VecDeque<Token>) -> Option<usize> {
+/*fn first_else_not_in_bracket(tokens: &VecDeque<Token>) -> Option<usize> {
-    for i in 0..tokens.len() {
-        if tokens[i].is_colon() {
-            return Some(i);
-        }
-    }
-    None
-}
-
-fn first_colon2(tokens: &VecDeque<&Token>) -> Option<usize> {
-    for i in 0..tokens.len() {
-        if tokens[i].is_colon() {
-            return Some(i);
-        }
-    }
-    None
-}
-
-fn first_else_not_in_bracket(tokens: &VecDeque<Token>) -> Option<usize> {
     let mut inside_brackets = 0;
     for i in 0..tokens.len() {
         if check_name("else", &tokens[i]) && inside_brackets == 0 {
@@ -584,4 +378,4 @@ fn first_else_not_in_bracket(tokens: &VecDeque<Token>) -> Option<usize> {
         }
     }
     None
-}
+}*/

interpreter/src/lang/filter_replace.rs

@@ -3,7 +3,7 @@ use std::fmt::{Debug, Display, Error, Formatter};
 use std::iter::Iterator;
 
 use crate::lang::SingleItem;
-use crate::lang::{filter::VariableOrOp, FilterPredicate};
+use crate::lang::FilterPredicate;
 use crate::lang::{IteratorItem, Op, PseudoOp};
 use crate::lang::{RuntimeError, RuntimeMsg, RuntimeOp};
 use crate::processing::general::Type;
@@ -21,7 +21,7 @@ pub(super) fn item_cache_deque() -> VecDeque<Result<Item, RuntimeError>> {
 #[derive(Debug)]
 pub struct FilterReplaceStatement<P: FilterPredicate + 'static> {
     pub(super) predicate: P,
-    pub(super) iterable: VariableOrOp,
+    pub(super) iterable: PseudoOp,
     pub(super) context: Option<Context>,
     pub(super) op_if: PseudoOp,
     pub(super) op_else: Option<PseudoOp>,
@@ -70,20 +70,8 @@ impl<P: FilterPredicate + 'static> Op for FilterReplaceStatement<P> {
 
     fn is_resetable(&self) -> bool {
         match &self.iterable {
-            VariableOrOp::Variable(s) => {
+            PseudoOp::Real(op) => op.is_resetable(),
-                if self.context.is_some() {
+            PseudoOp::Fake(_) => false,
-                    let var = self.context.as_ref().unwrap().variables.get_opt(s);
-                    if let Some(Type::Op(var)) = var {
-                        var.is_resetable()
-                    } else {
-                        false
-                    }
-                } else {
-                    true
-                } // ASSUMPTION
-            }
-            VariableOrOp::Op(PseudoOp::Real(op)) => op.is_resetable(),
-            VariableOrOp::Op(PseudoOp::Fake(_)) => false,
         }
     }
 
@@ -94,45 +82,13 @@ impl<P: FilterPredicate + 'static> Op for FilterReplaceStatement<P> {
             .reset()
             .map_err(|x| x.with(RuntimeOp(fake.clone())))?;
         match &mut self.iterable {
-            VariableOrOp::Variable(s) => {
+            PseudoOp::Real(op) => {
-                if self.context.as_mut().unwrap().variables.exists(s) {
-                    let mut var = self
-                        .context
-                        .as_mut()
-                        .unwrap()
-                        .variables
-                        .remove(s)
-                        .map_err(|e| e.with(RuntimeOp(fake.clone())))?;
-                    let result = if let Type::Op(var) = &mut var {
-                        var.enter(self.context.take().unwrap());
-                        let result = var.reset();
-                        self.context = Some(var.escape());
-                        result
-                    } else {
-                        Err(RuntimeError {
-                            line: 0,
-                            op: fake.clone(),
-                            msg: "Cannot reset non-iterable filter variable".to_string(),
-                        })
-                    };
-                    self.context
-                        .as_mut()
-                        .unwrap()
-                        .variables
-                        .declare(s, var)
-                        .map_err(|e| e.with(RuntimeOp(fake)))?;
-                    result
-                } else {
-                    Ok(())
-                }
-            }
-            VariableOrOp::Op(PseudoOp::Real(op)) => {
                 op.enter(self.context.take().unwrap());
                 let result = op.reset();
                 self.context = Some(op.escape());
                 result
             }
-            VariableOrOp::Op(PseudoOp::Fake(_)) => Err(RuntimeError {
+            PseudoOp::Fake(_) => Err(RuntimeError {
                 line: 0,
                 op: fake,
                 msg: "Cannot reset PseudoOp::Fake filter".to_string(),
@@ -143,10 +99,7 @@ impl<P: FilterPredicate + 'static> Op for FilterReplaceStatement<P> {
     fn dup(&self) -> Box<dyn Op> {
         Box::new(Self {
             predicate: self.predicate.clone(),
-            iterable: match &self.iterable {
+            iterable: self.iterable.try_real_ref().unwrap().dup().into(),
-                VariableOrOp::Variable(s) => VariableOrOp::Variable(s.clone()),
-                VariableOrOp::Op(op) => VariableOrOp::Op(op.try_real_ref().unwrap().dup().into()),
-            },
             context: None,
             op_if: PseudoOp::from(self.op_if.try_real_ref().unwrap().dup()),
             op_else: self
@@ -167,8 +120,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterReplaceStatement<P> {
         }
         let fake = PseudoOp::Fake(format!("{}", self));
         // get next item in iterator
-        let next_item = match &mut self.iterable {
+        let next_item = match self.iterable.try_real() {
-            VariableOrOp::Op(op) => match op.try_real() {
             Ok(real_op) => {
                 let ctx = self.context.take().unwrap();
                 real_op.enter(ctx);
@@ -177,43 +129,6 @@ impl<P: FilterPredicate + 'static> Iterator for FilterReplaceStatement<P> {
                 item
             }
             Err(e) => return Some(Err(e)),
-            },
-            VariableOrOp::Variable(variable_name) => {
-                let mut variable = match self
-                    .context
-                    .as_mut()
-                    .unwrap()
-                    .variables
-                    .remove(variable_name)
-                {
-                    Ok(Type::Op(op)) => op,
-                    Ok(x) => {
-                        return Some(Err(RuntimeError {
-                            line: 0,
-                            op: fake,
-                            msg: format!(
-                                "Expected operation/iterable type in variable {}, got {}",
-                                &variable_name, x
-                            ),
-                        }))
-                    }
-                    Err(e) => return Some(Err(e.with(RuntimeOp(fake)))),
-                };
-                let ctx = self.context.take().unwrap();
-                variable.enter(ctx);
-                let item = variable.next();
-                self.context = Some(variable.escape());
-                if let Err(e) = self
-                    .context
-                    .as_mut()
-                    .unwrap()
-                    .variables
-                    .declare(variable_name, Type::Op(variable))
-                {
-                    return Some(Err(e.with(RuntimeOp(fake))));
-                }
-                item
-            }
         };
         // process item
         match next_item {
@@ -242,6 +157,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterReplaceStatement<P> {
                                     real_op.enter(self.context.take().unwrap());
                                     if real_op.is_resetable() {
                                         if let Err(e) = real_op.reset() {
+                                            self.context = Some(real_op.escape());
                                             return Some(Err(e));
                                         }
                                     }
@@ -269,6 +185,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterReplaceStatement<P> {
                                 replacement
                             }
                         } else if let Some(op_else) = &mut self.op_else {
+                            println!("op_else {}\n{:?}", op_else, op_else);
                             // unwrap inner operation
                             match op_else.try_real() {
                                 Ok(real_op) => {
@@ -286,6 +203,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterReplaceStatement<P> {
                                     real_op.enter(self.context.take().unwrap());
                                     if real_op.is_resetable() {
                                         if let Err(e) = real_op.reset() {
+                                            self.context = Some(real_op.escape());
                                             return Some(Err(e));
                                         }
                                     }
@@ -325,18 +243,7 @@ impl<P: FilterPredicate + 'static> Iterator for FilterReplaceStatement<P> {
     }
 
     fn size_hint(&self) -> (usize, Option<usize>) {
-        match &self.iterable {
+        self.iterable.try_real_ref().map(|x| x.size_hint()).ok().unwrap_or((0, None))
-            VariableOrOp::Variable(s) => self
-                .context
-                .as_ref()
-                .and_then(|x| x.variables.get_opt(s))
-                .and_then(|x| match x {
-                    Type::Op(op) => Some(op.size_hint()),
-                    _ => None,
-                }),
-            VariableOrOp::Op(op) => op.try_real_ref().map(|x| x.size_hint()).ok(),
-        }
-        .unwrap_or((0, None))
     }
 }
 

interpreter/src/lang/function.rs

@@ -45,7 +45,6 @@ impl<O: Op + 'static, F: FunctionFactory<O> + 'static> BoxedOpFactory
                 Token::Name(n) => {
                     self.op_factory.is_function(n)
                         && tokens[1].is_open_bracket()
-                        && tokens[tokens_len - 1].is_close_bracket()
                 }
                 _ => false,
             }
@@ -66,9 +65,9 @@ impl<O: Op + 'static, F: FunctionFactory<O> + 'static> BoxedOpFactory
             tokens,
         )?;
         assert_token_raw(Token::OpenBracket, tokens)?;
-        let end_tokens = tokens.split_off(tokens.len() - 1);
+        //let end_tokens = tokens.split_off(tokens.len() - 1);
         let func = self.op_factory.build_function_params(name, tokens, dict)?;
-        tokens.extend(end_tokens);
+        //tokens.extend(end_tokens);
         assert_token_raw(Token::CloseBracket, tokens)?;
         Ok(Box::new(func))
     }

interpreter/src/lang/iter_block.rs

@@ -6,9 +6,9 @@ use std::iter::Iterator;
 use std::marker::PhantomData;
 use std::sync::Arc;
 
-use crate::lang::utility::{assert_token_raw, assert_token_raw_back};
+use crate::lang::utility::assert_token_raw;
 use crate::lang::LanguageDictionary;
-use crate::lang::{BoxedOpFactory, IteratorItem, Op, PseudoOp};
+use crate::lang::{BoxedTransformOpFactory, IteratorItem, Op, PseudoOp};
 use crate::lang::{RuntimeError, RuntimeMsg, RuntimeOp, SyntaxError};
 use crate::processing::general::Type;
 use crate::tokens::Token;
@@ -72,7 +72,7 @@ impl Display for ItemBlockStatement {
             writeln!(f)?;
         }
         for statement in self.statements.iter() {
-            writeln!(f, "{}", statement)?;
+            writeln!(f, "{},", statement)?;
         }
         write!(f, "}}")
     }
@@ -227,7 +227,7 @@ impl ItemBlockFactory {
         tokens: &mut VecDeque<Token>,
         dict: &LanguageDictionary,
     ) -> Result<Box<dyn ItemOp>, SyntaxError> {
-        for factory in &self.vocabulary {
+        for (i, factory) in self.vocabulary.iter().enumerate() {
             if factory.is_item_op(tokens) {
                 return factory.build_item_op(tokens, self, dict);
             }
@@ -253,48 +253,44 @@ impl ItemBlockFactory {
     }
 }
 
-impl BoxedOpFactory for ItemBlockFactory {
+impl BoxedTransformOpFactory for ItemBlockFactory {
-    fn is_op_boxed(&self, tokens: &VecDeque<Token>) -> bool {
+    fn build_transform_op(
-        find_last_open_curly(tokens).is_some()
-    }
-
-    fn build_op_boxed(
         &self,
         tokens: &mut VecDeque<Token>,
         dict: &LanguageDictionary,
+        op: Box<dyn Op>,
     ) -> Result<Box<dyn Op>, SyntaxError> {
-        let open_curly_pos = if let Some(pos) = find_last_open_curly(tokens) {
-            Ok(pos)
-        } else {
-            Err(SyntaxError {
-                line: 0,
-                token: Token::OpenCurly,
-                got: tokens.pop_front(),
-            })
-        }?;
-        let block_tokens = tokens.split_off(open_curly_pos - 1); // . always before {
-        let inner_op = dict.try_build_statement(tokens)?;
-        tokens.extend(block_tokens);
         assert_token_raw(Token::Dot, tokens)?;
         assert_token_raw(Token::OpenCurly, tokens)?;
-        assert_token_raw_back(Token::CloseCurly, tokens)?;
         let mut item_ops = Vec::with_capacity(tokens.len() / 8);
         while !tokens.is_empty() {
-            if let Some(next_comma) = find_next_comma(tokens) {
+            if tokens[0].is_close_curly() {
-                let end_tokens = tokens.split_off(next_comma);
+                break;
+            }
             item_ops.push(Arc::new(self.try_build_item_statement(tokens, dict)?));
-                tokens.extend(end_tokens);
+            if !tokens.is_empty() {
+                if tokens[0].is_comma() {
                     assert_token_raw(Token::Comma, tokens)?;
+                }
             } else {
-                item_ops.push(Arc::new(self.try_build_item_statement(tokens, dict)?));
+                return Err(SyntaxError {
+                    got: tokens.pop_front(),
+                    token: Token::Literal(", or }".into()),
+                    line: 0,
+                })
             }
         }
+        assert_token_raw(Token::CloseCurly, tokens)?;
         Ok(Box::new(ItemBlockStatement {
             statements: item_ops,
-            iterable: inner_op.into(),
+            iterable: op.into(),
             last_item: None,
         }))
     }
+
+    fn is_transform_op(&self, tokens: &VecDeque<Token>) -> bool {
+        tokens.len() > 1 && tokens[0].is_dot() && tokens[1].is_open_curly()
+    }
 }
 
 fn replace_item_var(ctx: &mut Context, item: Type) -> Option<Type> {
@@ -320,42 +316,7 @@ fn restore_item_var(ctx: &mut Context, old_var: Option<Type>) -> Result<Option<T
     Ok(new_var)
 }
 
-fn find_last_open_curly(tokens: &VecDeque<Token>) -> Option<usize> {
+/*fn find_next_comma(tokens: &VecDeque<Token>) -> Option<usize> {
-    let mut bracket_depth = 0;
-    let mut curly_found = false;
-    for i in (0..tokens.len()).rev() {
-        let token = &tokens[i];
-        match token {
-            Token::OpenCurly => {
-                if bracket_depth != 0 {
-                    bracket_depth -= 1;
-                }
-            }
-            Token::CloseCurly => {
-                bracket_depth += 1;
-            }
-            Token::Dot => {
-                if bracket_depth == 0 && curly_found {
-                    return Some(i + 1);
-                }
-            }
-            Token::OpenBracket | Token::CloseBracket => {
-                if bracket_depth == 0 {
-                    return None;
-                }
-            }
-            _ => {}
-        }
-        if token.is_open_curly() {
-            curly_found = true;
-        } else {
-            curly_found = false;
-        }
-    }
-    None
-}
-
-fn find_next_comma(tokens: &VecDeque<Token>) -> Option<usize> {
     let mut bracket_depth = 0;
     let mut curly_depth = 0;
     for i in 0..tokens.len() {
@@ -373,4 +334,4 @@ fn find_next_comma(tokens: &VecDeque<Token>) -> Option<usize> {
         }
     }
     None
-}
+}*/

interpreter/src/lang/mod.rs

@@ -29,7 +29,7 @@ pub use function::{FunctionFactory, FunctionStatementFactory};
 pub use generator_op::GeneratorOp;
 pub use iter_block::{ItemBlockFactory, ItemOp, ItemOpFactory};
 pub use lookup::Lookup;
-pub use operation::{BoxedOpFactory, IteratorItem, Op, OpFactory, SimpleOpFactory};
+pub use operation::{BoxedOpFactory, IteratorItem, Op, OpFactory, SimpleOpFactory, BoxedTransformOpFactory};
 pub use pseudo_op::PseudoOp;
 pub use repeated_meme::{repeated_tokens, RepeatedTokens};
 pub use single_op::SingleItem;

interpreter/src/lang/operation.rs

@@ -12,20 +12,22 @@ use crate::Item;
 // TODO change API to allow for accumulating modifiers
 // e.g. build_op(&self, Option<Box<dyn Op>>, tokens) -> ...
 
-pub trait SimpleOpFactory<T: Op + 'static> {
+type TokenList = VecDeque<Token>;
-    fn is_op_simple(&self, tokens: &VecDeque<Token>) -> bool;
 
-    fn build_op_simple(&self, tokens: &mut VecDeque<Token>) -> Result<T, SyntaxError>;
+pub trait SimpleOpFactory<T: Op + 'static> {
+    fn is_op_simple(&self, tokens: &TokenList) -> bool;
+
+    fn build_op_simple(&self, tokens: &mut TokenList) -> Result<T, SyntaxError>;
 }
 
 impl<T: Op + 'static, X: SimpleOpFactory<T> + 'static> OpFactory<T> for X {
-    fn is_op(&self, tokens: &VecDeque<Token>) -> bool {
+    fn is_op(&self, tokens: &TokenList) -> bool {
         self.is_op_simple(tokens)
     }
 
     fn build_op(
         &self,
-        tokens: &mut VecDeque<Token>,
+        tokens: &mut TokenList,
         _dict: &LanguageDictionary,
     ) -> Result<T, SyntaxError> {
         self.build_op_simple(tokens)
@@ -33,18 +35,18 @@ impl<T: Op + 'static, X: SimpleOpFactory<T> + 'static> OpFactory<T> for X {
 }
 
 pub trait OpFactory<T: Op + 'static> {
-    fn is_op(&self, tokens: &VecDeque<Token>) -> bool;
+    fn is_op(&self, tokens: &TokenList) -> bool;
 
     fn build_op(
         &self,
-        tokens: &mut VecDeque<Token>,
+        tokens: &mut TokenList,
         dict: &LanguageDictionary,
     ) -> Result<T, SyntaxError>;
 
     #[inline]
     fn build_box(
         &self,
-        tokens: &mut VecDeque<Token>,
+        tokens: &mut TokenList,
         dict: &LanguageDictionary,
     ) -> Result<Box<dyn Op>, SyntaxError> {
         Ok(Box::new(self.build_op(tokens, dict)?))
@@ -54,11 +56,22 @@ pub trait OpFactory<T: Op + 'static> {
 pub trait BoxedOpFactory: Send {
     fn build_op_boxed(
         &self,
-        tokens: &mut VecDeque<Token>,
+        tokens: &mut TokenList,
         dict: &LanguageDictionary,
     ) -> Result<Box<dyn Op>, SyntaxError>;
 
-    fn is_op_boxed(&self, tokens: &VecDeque<Token>) -> bool;
+    fn is_op_boxed(&self, tokens: &TokenList) -> bool;
+}
+
+pub trait BoxedTransformOpFactory: Send {
+    fn build_transform_op(
+        &self,
+        tokens: &mut TokenList,
+        dict: &LanguageDictionary,
+        op: Box<dyn Op>,
+    ) -> Result<Box<dyn Op>, SyntaxError>;
+
+    fn is_transform_op(&self, tokens: &TokenList) -> bool;
 }
 
 pub type IteratorItem = Result<Item, RuntimeError>;

interpreter/src/lang/sorter.rs

@@ -3,9 +3,9 @@ use std::fmt::{Debug, Display, Error, Formatter};
 use std::iter::Iterator;
 use std::marker::PhantomData;
 
-use crate::lang::utility::{assert_name, assert_token_raw, check_name};
+use crate::lang::utility::assert_token_raw;
 use crate::lang::LanguageDictionary;
-use crate::lang::{BoxedOpFactory, IteratorItem, Op, PseudoOp};
+use crate::lang::{IteratorItem, Op, PseudoOp, BoxedTransformOpFactory};
 use crate::lang::{RuntimeError, RuntimeMsg, RuntimeOp, SyntaxError};
 use crate::tokens::Token;
 use crate::Context;
@@ -125,58 +125,16 @@ impl<S: Sorter + 'static, F: SorterFactory<S> + 'static> SortStatementFactory<S,
     }
 }
 
-impl<S: Sorter + 'static, F: SorterFactory<S> + 'static> BoxedOpFactory
+impl<S: Sorter + 'static, F: SorterFactory<S> + 'static> BoxedTransformOpFactory
     for SortStatementFactory<S, F>
 {
-    fn is_op_boxed(&self, tokens: &VecDeque<Token>) -> bool {
+    fn build_transform_op(
-        let tokens_len = tokens.len();
-        if let Some(tilde_location) = last_tilde(tokens, 0) {
-            // iterable~(sorter)
-            if tokens_len > tilde_location + 2 {
-                let tokens2: VecDeque<&Token> =
-                    VecDeque::from_iter(tokens.range(tilde_location + 2..tokens_len - 1));
-                tokens[tokens_len - 1].is_close_bracket() && self.sort_factory.is_sorter(&tokens2)
-            } else {
-                false
-            }
-        } else if let Some(dot_location) = last_dot_sort(tokens, 1) {
-            // iterable.sort(sorter)
-            if tokens_len > dot_location + 3 {
-                let tokens2: VecDeque<&Token> =
-                    VecDeque::from_iter(tokens.range(dot_location + 3..tokens_len - 1));
-                tokens[tokens_len - 1].is_close_bracket() && self.sort_factory.is_sorter(&tokens2)
-            } else {
-                false
-            }
-        } else {
-            false
-        }
-    }
-
-    fn build_op_boxed(
         &self,
         tokens: &mut VecDeque<Token>,
         dict: &LanguageDictionary,
+        op: Box<dyn Op>,
     ) -> Result<Box<dyn Op>, SyntaxError> {
-        let inner_op;
-        if let Some(tilde_location) = last_tilde(tokens, 0) {
-            let end_tokens = tokens.split_off(tilde_location);
-            inner_op = dict.try_build_statement(tokens)?;
-            tokens.extend(end_tokens);
         assert_token_raw(Token::Tilde, tokens)?;
-        } else if let Some(dot_location) = last_dot_sort(tokens, 1) {
-            let end_tokens = tokens.split_off(dot_location);
-            inner_op = dict.try_build_statement(tokens)?;
-            tokens.extend(end_tokens);
-            assert_token_raw(Token::Dot, tokens)?;
-            assert_name("sort", tokens)?;
-        } else {
-            return Err(SyntaxError {
-                line: 0,
-                token: Token::Name(".|~".into()),
-                got: tokens.pop_front(),
-            });
-        }
         assert_token_raw(Token::OpenBracket, tokens)?;
         let end_tokens = tokens.split_off(tokens.len() - 1);
         let sorter = self.sort_factory.build_sorter(tokens, dict)?;
@@ -184,58 +142,19 @@ impl<S: Sorter + 'static, F: SorterFactory<S> + 'static> BoxedOpFactory
         assert_token_raw(Token::CloseBracket, tokens)?;
         Ok(Box::new(SortStatement {
             orderer: sorter,
-            iterable: inner_op.into(),
+            iterable: op.into(),
             item_cache: VecDeque::with_capacity(SORTER_ITEM_CACHE_SIZE),
         }))
     }
-}
 
-fn last_tilde(tokens: &VecDeque<Token>, target_depth: usize) -> Option<usize> {
+    fn is_transform_op(&self, tokens: &VecDeque<Token>) -> bool {
-    let mut bracket_depth = 0;
+        if tokens.len() > 2 {
-    for i in (0..tokens.len()).rev() {
+            let tokens2: VecDeque<&Token> =
-        let current_token = &tokens[i];
+                    VecDeque::from_iter(tokens.range(2..));
-        if current_token.is_close_bracket() {
+            tokens[0].is_tilde() && self.sort_factory.is_sorter(&tokens2)
-            bracket_depth += 1;
+        } else {
-        } else if current_token.is_open_bracket() && bracket_depth != 0 {
+            false
-            bracket_depth -= 1;
-        } else if current_token.is_tilde() && bracket_depth == target_depth {
-            return Some(i);
         }
-    }
-    None
-}
 
-fn last_dot_sort(tokens: &VecDeque<Token>, target_depth: usize) -> Option<usize> {
-    let mut bracket_depth = 0;
-    let mut sort_found = false;
-    let mut bracket_found = false;
-    for i in (0..tokens.len()).rev() {
-        let current_token = &tokens[i];
-        if sort_found {
-            return {
-                if current_token.is_dot() {
-                    Some(i)
-                } else {
-                    None
     }
-            };
-        } else if bracket_found {
-            if check_name("sort", current_token) {
-                sort_found = true;
-            } else {
-                bracket_found = false;
-            }
-        }
-        if current_token.is_close_bracket() {
-            bracket_depth += 1;
-        } else if current_token.is_open_bracket() {
-            if target_depth == bracket_depth {
-                bracket_found = true;
-            }
-            if bracket_depth != 0 {
-                bracket_depth -= 1;
-            }
-        }
-    }
-    None
 }

interpreter/src/lang/utility.rs

@@ -49,29 +49,6 @@ pub fn assert_token_raw(token: Token, tokens: &mut VecDeque<Token>) -> Result<To
     }
 }
 
-pub fn assert_token_raw_back(
-    token: Token,
-    tokens: &mut VecDeque<Token>,
-) -> Result<Token, SyntaxError> {
-    let result = match tokens.pop_back() {
-        Some(x) => Ok(x),
-        None => Err(SyntaxError {
-            line: 0,
-            token: token.clone(),
-            got: None,
-        }),
-    }?;
-    if std::mem::discriminant(&token) == std::mem::discriminant(&result) {
-        Ok(result)
-    } else {
-        Err(SyntaxError {
-            line: 0,
-            token,
-            got: Some(result),
-        })
-    }
-}
-
 pub fn check_token_raw(token: Token, token_target: &Token) -> bool {
     std::mem::discriminant(&token) == std::mem::discriminant(token_target)
 }

interpreter/src/lang/vocabulary/files.rs

@@ -141,11 +141,11 @@ impl FunctionFactory<FilesStatement> for FilesFunctionFactory {
         tokens: &mut VecDeque<Token>,
         _dict: &LanguageDictionary,
     ) -> Result<FilesStatement, SyntaxError> {
-        // files([folder|dir=]"path", [regex|re = "pattern"], [recursive = true|false])
+        // files([folder|dir=]"path", [regex|re = "pattern",] [recursive = true|false,])
         let mut root_path = None;
         let mut pattern = None;
         let mut recursive = None;
-        if !tokens.is_empty() {
+        if !tokens.is_empty() && !tokens[0].is_close_bracket() {
             if tokens[0].is_literal() {
                 // folder is specified without keyword
                 root_path = Some(assert_token(
@@ -162,7 +162,7 @@ impl FunctionFactory<FilesStatement> for FilesFunctionFactory {
             }
             // parse keyword function parameters
             let ingest = |tokens2: &mut VecDeque<Token>| {
-                if tokens2.len() < 3 {
+                if tokens2[0].is_close_bracket() {
                     return Ok(None);
                 } // nothing wrong, nothing left to ingest
                 let param_name = assert_token(

interpreter/src/lang/vocabulary/filters/empty_filter.rs

@@ -35,7 +35,7 @@ pub struct EmptyFilterFactory;
 
 impl FilterFactory<EmptyFilter> for EmptyFilterFactory {
     fn is_filter(&self, tokens: &VecDeque<&Token>) -> bool {
-        tokens.is_empty()
+        !tokens.is_empty() && tokens[0].is_close_bracket()
     }
 
     fn build_filter(

interpreter/src/lang/vocabulary/filters/field/field_match_filter.rs

@@ -167,7 +167,7 @@ impl FieldFilterFactory<FieldRegexFilter> for FieldRegexFilterFactory {
 fn regex_flags(tokens: &mut VecDeque<Token>) -> Result<u8, SyntaxError> {
     // syntax: , "flags"
     let mut result = 0_u8;
-    if tokens.is_empty() {
+    if tokens.is_empty() || !tokens[0].is_comma() {
         Ok(result)
     } else {
         assert_token_raw(Token::Comma, tokens)?;

interpreter/src/lang/vocabulary/filters/index_filter.rs

@@ -64,8 +64,8 @@ pub struct IndexFilterFactory;
 
 impl FilterFactory<IndexFilter> for IndexFilterFactory {
     fn is_filter(&self, tokens: &VecDeque<&Token>) -> bool {
-        (tokens.len() == 1 && Lookup::check_is(tokens[0]))
+        (tokens.len() >= 1 && Lookup::check_is(tokens[0]))
-            || (tokens.len() == 2 && tokens[0].is_exclamation() && Lookup::check_is(tokens[1]))
+            || (tokens.len() >= 2 && tokens[0].is_exclamation() && Lookup::check_is(tokens[1]))
     }
 
     fn build_filter(

interpreter/src/lang/vocabulary/filters/range_filter.rs

@@ -101,9 +101,9 @@ pub struct RangeFilterFactory;
 
 impl FilterFactory<RangeFilter> for RangeFilterFactory {
     fn is_filter(&self, tokens: &VecDeque<&Token>) -> bool {
-        tokens.len() >= 2
+        tokens.len() > 1
-            && ((tokens.len() >= 2 && tokens[0].is_dot() && tokens[1].is_dot())
+            && ((tokens[0].is_dot() && tokens[1].is_dot())
-                || (tokens.len() >= 3
+                || (tokens.len() > 2
                     && Lookup::check_is(tokens[0])
                     && tokens[1].is_dot()
                     && tokens[2].is_dot()))
@@ -132,7 +132,7 @@ impl FilterFactory<RangeFilter> for RangeFilterFactory {
             false
         };
         // end index
-        let end = if !tokens.is_empty() {
+        let end = if !tokens.is_empty() && Lookup::check_is(&tokens[0]) {
             Some(Lookup::parse(tokens)?)
         } else {
             None

interpreter/src/lang/vocabulary/filters/unique.rs

@@ -90,7 +90,7 @@ pub struct UniqueFilterFactory;
 
 impl FilterFactory<UniqueFieldFilter> for UniqueFilterFactory {
     fn is_filter(&self, tokens: &VecDeque<&Token>) -> bool {
-        tokens.len() >= 2 && check_name("unique", tokens[0]) && tokens[1].is_dot()
+        tokens.len() > 1 && check_name("unique", tokens[0]) && tokens[1].is_dot()
     }
 
     fn build_filter(
@@ -108,7 +108,7 @@ impl FilterFactory<UniqueFieldFilter> for UniqueFilterFactory {
             Token::Name("field_name".into()),
             tokens,
         )?;
-        let error_handling = if !tokens.is_empty() {
+        let error_handling = if !tokens.is_empty() && (tokens[0].is_exclamation() || tokens[0].is_interrogation()) {
             if tokens[0].is_exclamation() {
                 assert_token_raw(Token::Exclamation, tokens)?;
                 FieldFilterErrorHandling::Ignore
@@ -129,7 +129,7 @@ impl FilterFactory<UniqueFieldFilter> for UniqueFilterFactory {
 
 impl FilterFactory<UniqueFilter> for UniqueFilterFactory {
     fn is_filter(&self, tokens: &VecDeque<&Token>) -> bool {
-        tokens.len() == 1 && check_name("unique", tokens[0])
+        tokens.len() > 1 && check_name("unique", tokens[0])
     }
 
     fn build_filter(

interpreter/src/lang/vocabulary/item_ops/add.rs

@@ -69,10 +69,11 @@ impl ItemOpFactory<AddItemOp> for AddItemOpFactory {
         dict: &LanguageDictionary,
     ) -> Result<AddItemOp, SyntaxError> {
         let plus_location = first_plus(tokens).unwrap();
-        let mut end_tokens = tokens.split_off(plus_location);
+        let end_tokens = tokens.split_off(plus_location);
         let lhs_op = factory.try_build_item_statement(tokens, dict)?;
-        assert_token_raw(Token::Plus, &mut end_tokens)?;
+        tokens.extend(end_tokens);
-        let rhs_op = factory.try_build_item_statement(&mut end_tokens, dict)?;
+        assert_token_raw(Token::Plus, tokens)?;
+        let rhs_op = factory.try_build_item_statement(tokens, dict)?;
         Ok(AddItemOp {
             lhs: lhs_op,
             rhs: rhs_op,
@@ -90,6 +91,8 @@ fn first_plus(tokens: &VecDeque<Token>) -> Option<usize> {
             bracket_depth += 1;
         } else if token.is_close_bracket() && bracket_depth != 0 {
             bracket_depth -= 1;
+        } else if token.is_comma() && bracket_depth == 0 {
+            return None;
         }
     }
     None

interpreter/src/lang/vocabulary/item_ops/brackets.rs

@@ -1,7 +1,7 @@
 use std::collections::VecDeque;
 use std::convert::AsRef;
 
-use crate::lang::utility::{assert_token_raw, assert_token_raw_back};
+use crate::lang::utility::assert_token_raw;
 use crate::lang::LanguageDictionary;
 use crate::lang::{ItemBlockFactory, ItemOp, ItemOpFactory};
 use crate::lang::{RuntimeMsg, SyntaxError};
@@ -13,9 +13,8 @@ pub struct BracketsItemOpFactory;
 
 impl ItemOpFactory<Box<dyn ItemOp>> for BracketsItemOpFactory {
     fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
-        tokens.len() >= 2
+        !tokens.is_empty()
             && tokens[0].is_open_bracket()
-            && tokens[tokens.len() - 1].is_close_bracket()
     }
 
     fn build_item_op(
@@ -25,8 +24,9 @@ impl ItemOpFactory<Box<dyn ItemOp>> for BracketsItemOpFactory {
         dict: &LanguageDictionary,
     ) -> Result<Box<dyn ItemOp>, SyntaxError> {
         assert_token_raw(Token::OpenBracket, tokens)?;
-        assert_token_raw_back(Token::CloseBracket, tokens)?;
+        let inner = factory.try_build_item_statement(tokens, dict)?;
-        factory.try_build_item_statement(tokens, dict)
+        assert_token_raw(Token::CloseBracket, tokens)?;
+        Ok(inner)
     }
 }
 

interpreter/src/lang/vocabulary/item_ops/branch.rs

@@ -126,10 +126,8 @@ impl ItemOpFactory<BranchItemOp> for BranchItemOpFactory {
         let end_tokens = tokens.split_off(next_close_curly);
         let mut inner_if_ops = Vec::new();
         while !tokens.is_empty() {
-            if let Some(next_comma) = find_next_comma(tokens) {
+            if find_next_comma(tokens).is_some() {
-                let end_tokens = tokens.split_off(next_comma);
                 inner_if_ops.push(factory.try_build_item_statement(tokens, dict)?);
-                tokens.extend(end_tokens);
                 assert_token_raw(Token::Comma, tokens)?;
             } else {
                 inner_if_ops.push(factory.try_build_item_statement(tokens, dict)?);
@@ -234,6 +232,8 @@ fn find_next_comma(tokens: &VecDeque<Token>) -> Option<usize> {
         let token = &tokens[i];
         if token.is_comma() && bracket_depth == 0 && curly_depth == 0 {
             return Some(i);
+        } else if token.is_comma() && (bracket_depth < 0 || curly_depth < 0) {
+            return None;
         } else if token.is_open_bracket() {
             bracket_depth += 1;
         } else if token.is_close_bracket() && bracket_depth != 0 {

interpreter/src/lang/vocabulary/item_ops/compare.rs

@@ -123,6 +123,9 @@ fn find_first_comparison(tokens: &VecDeque<Token>) -> Option<usize> {
                 {
                     return Some(i);
                 }
+            },
+            Token::Comma if curly_depth == 0 && bracket_depth == 0 => {
+                return None;
             }
             _ => {}
         }

interpreter/src/lang/vocabulary/item_ops/constant.rs

@@ -38,7 +38,7 @@ pub struct ConstantItemOpFactory;
 
 impl ItemOpFactory<ConstantItemOp> for ConstantItemOpFactory {
     fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
-        tokens.len() == 1 && check_is_type(&tokens[0])
+        !tokens.is_empty() && check_is_type(&tokens[0])
     }
 
     fn build_item_op(

interpreter/src/lang/vocabulary/item_ops/constructor.rs

@@ -3,7 +3,7 @@ use std::collections::VecDeque;
 use std::fmt::{Debug, Display, Error, Formatter};
 
 use crate::lang::utility::{
-    assert_name, assert_token, assert_token_raw, assert_token_raw_back, check_name,
+    assert_name, assert_token, assert_token_raw, check_name,
 };
 use crate::lang::LanguageDictionary;
 use crate::lang::{ItemBlockFactory, ItemOp, ItemOpFactory};
@@ -83,9 +83,11 @@ impl ItemOpFactory<ConstructorItemOp> for ConstructorItemOpFactory {
     ) -> Result<ConstructorItemOp, SyntaxError> {
         assert_name("Item", tokens)?;
         assert_token_raw(Token::OpenBracket, tokens)?;
-        assert_token_raw_back(Token::CloseBracket, tokens)?;
         let mut field_descriptors = Vec::new();
         while !tokens.is_empty() {
+            if tokens[0].is_close_bracket() {
+                break;
+            }
             let field_name = assert_token(
                 |t| match t {
                     Token::Name(n) => Some(n),
@@ -96,10 +98,8 @@ impl ItemOpFactory<ConstructorItemOp> for ConstructorItemOpFactory {
             )?;
             assert_token_raw(Token::Equals, tokens)?;
             let field_val;
-            if let Some(comma_pos) = find_next_comma(tokens) {
+            if find_next_comma(tokens).is_some() {
-                let end_tokens = tokens.split_off(comma_pos);
                 field_val = factory.try_build_item_statement(tokens, dict)?;
-                tokens.extend(end_tokens);
                 assert_token_raw(Token::Comma, tokens)?;
             } else {
                 field_val = factory.try_build_item_statement(tokens, dict)?;
@@ -109,6 +109,7 @@ impl ItemOpFactory<ConstructorItemOp> for ConstructorItemOpFactory {
                 value: field_val,
             });
         }
+        assert_token_raw(Token::CloseBracket, tokens)?;
         Ok(ConstructorItemOp {
             fields: field_descriptors,
         })

interpreter/src/lang/vocabulary/item_ops/empty.rs

@@ -36,10 +36,8 @@ pub struct EmptyItemOpFactory;
 
 impl ItemOpFactory<EmptyItemOp> for EmptyItemOpFactory {
     fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
-        tokens.len() == 3
+        !tokens.is_empty()
             && check_name("empty", &tokens[0])
-            && tokens[1].is_open_bracket()
-            && tokens[2].is_close_bracket()
     }
 
     fn build_item_op(

interpreter/src/lang/vocabulary/item_ops/field_assign.rs

@@ -61,12 +61,12 @@ pub struct FieldAssignItemOpFactory;
 
 impl ItemOpFactory<FieldAssignItemOp> for FieldAssignItemOpFactory {
     fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
-        (tokens.len() > 4
+        (tokens.len() > 3
             && tokens[0].is_name()
             && tokens[1].is_dot()
             && tokens[2].is_name()
             && tokens[3].is_equals())
-            || (tokens.len() > 3
+            || (tokens.len() > 2
                 && tokens[0].is_dot()
                 && tokens[1].is_name()
                 && tokens[2].is_equals())

interpreter/src/lang/vocabulary/item_ops/logical_and.rs

@@ -73,11 +73,12 @@ impl ItemOpFactory<AndItemOp> for AndItemOpFactory {
         dict: &LanguageDictionary,
     ) -> Result<AndItemOp, SyntaxError> {
         let and_location = first_and(tokens).unwrap();
-        let mut end_tokens = tokens.split_off(and_location);
+        let end_tokens = tokens.split_off(and_location);
         let lhs_op = factory.try_build_item_statement(tokens, dict)?;
-        assert_token_raw(Token::Ampersand, &mut end_tokens)?;
+        tokens.extend(end_tokens);
-        assert_token_raw(Token::Ampersand, &mut end_tokens)?;
+        assert_token_raw(Token::Ampersand, tokens)?;
-        let rhs_op = factory.try_build_item_statement(&mut end_tokens, dict)?;
+        assert_token_raw(Token::Ampersand, tokens)?;
+        let rhs_op = factory.try_build_item_statement(tokens, dict)?;
         Ok(AndItemOp {
             lhs: lhs_op,
             rhs: rhs_op,
@@ -95,6 +96,8 @@ fn first_and(tokens: &VecDeque<Token>) -> Option<usize> {
             bracket_depth += 1;
         } else if token.is_close_bracket() && bracket_depth != 0 {
             bracket_depth -= 1;
+        } else if token.is_comma() && bracket_depth == 0 {
+            return None;
         }
     }
     None

interpreter/src/lang/vocabulary/item_ops/logical_or.rs

@@ -73,11 +73,12 @@ impl ItemOpFactory<OrItemOp> for OrItemOpFactory {
         dict: &LanguageDictionary,
     ) -> Result<OrItemOp, SyntaxError> {
         let or_location = first_or(tokens).unwrap();
-        let mut end_tokens = tokens.split_off(or_location);
+        let end_tokens = tokens.split_off(or_location);
         let lhs_op = factory.try_build_item_statement(tokens, dict)?;
-        assert_token_raw(Token::Pipe, &mut end_tokens)?;
+        tokens.extend(end_tokens);
-        assert_token_raw(Token::Pipe, &mut end_tokens)?;
+        assert_token_raw(Token::Pipe, tokens)?;
-        let rhs_op = factory.try_build_item_statement(&mut end_tokens, dict)?;
+        assert_token_raw(Token::Pipe, tokens)?;
+        let rhs_op = factory.try_build_item_statement(tokens, dict)?;
         Ok(OrItemOp {
             lhs: lhs_op,
             rhs: rhs_op,
@@ -95,6 +96,8 @@ fn first_or(tokens: &VecDeque<Token>) -> Option<usize> {
             bracket_depth += 1;
         } else if token.is_close_bracket() && bracket_depth != 0 {
             bracket_depth -= 1;
+        } else if token.is_comma() && bracket_depth == 0 {
+            return None;
         }
     }
     None

interpreter/src/lang/vocabulary/item_ops/mod.rs

@@ -13,6 +13,7 @@ mod logical_or;
 mod negate;
 mod not;
 mod remove_variable;
+mod retrieve_field;
 mod retrieve_variable;
 mod string_interpolate;
 mod subtract;
@@ -34,6 +35,7 @@ pub use logical_or::{OrItemOp, OrItemOpFactory};
 pub use negate::{NegateItemOp, NegateItemOpFactory};
 pub use not::{NotItemOp, NotItemOpFactory};
 pub use remove_variable::{RemoveItemOp, RemoveItemOpFactory};
+pub use retrieve_field::FieldRetrieveItemOpFactory;
 pub use retrieve_variable::{VariableRetrieveItemOp, VariableRetrieveItemOpFactory};
 pub use string_interpolate::{InterpolateStringItemOp, InterpolateStringItemOpFactory};
 pub use subtract::{SubtractItemOp, SubtractItemOpFactory};

interpreter/src/lang/vocabulary/item_ops/negate.rs

@@ -46,7 +46,7 @@ pub struct NegateItemOpFactory;
 
 impl ItemOpFactory<NegateItemOp> for NegateItemOpFactory {
     fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
-        tokens.len() >= 2 && tokens[0].is_minus()
+        !tokens.is_empty() && tokens[0].is_minus()
     }
 
     fn build_item_op(

interpreter/src/lang/vocabulary/item_ops/not.rs

@@ -46,7 +46,7 @@ pub struct NotItemOpFactory;
 
 impl ItemOpFactory<NotItemOp> for NotItemOpFactory {
     fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
-        tokens.len() >= 2 && tokens[0].is_exclamation()
+        !tokens.is_empty() && tokens[0].is_exclamation()
     }
 
     fn build_item_op(

interpreter/src/lang/vocabulary/item_ops/remove_variable.rs

@@ -57,7 +57,7 @@ pub struct RemoveItemOpFactory;
 
 impl ItemOpFactory<RemoveItemOp> for RemoveItemOpFactory {
     fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
-        (tokens.len() == 2 || tokens.len() == 4) && check_name("remove", &tokens[0])
+        !tokens.is_empty() && check_name("remove", &tokens[0])
     }
 
     fn build_item_op(
@@ -75,7 +75,7 @@ impl ItemOpFactory<RemoveItemOp> for RemoveItemOpFactory {
             Token::Name("variable_name".into()),
             tokens,
         )?;
-        let field_opt = if tokens.is_empty() {
+        let field_opt = if tokens.is_empty() || !tokens[0].is_dot() {
             None
         } else {
             assert_token_raw(Token::Dot, tokens)?;

interpreter/src/lang/vocabulary/item_ops/retrieve_field.rs

@@ -0,0 +1,49 @@
+use std::collections::VecDeque;
+
+use crate::lang::utility::{assert_token, assert_token_raw};
+use crate::lang::LanguageDictionary;
+use crate::lang::{ItemBlockFactory, ItemOpFactory};
+use crate::lang::SyntaxError;
+use crate::tokens::Token;
+
+use super::VariableRetrieveItemOp;
+
+pub struct FieldRetrieveItemOpFactory;
+
+impl ItemOpFactory<VariableRetrieveItemOp> for FieldRetrieveItemOpFactory {
+    fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
+        tokens.len() > 2
+            && tokens[0].is_name()
+            && tokens[1].is_dot()
+            && tokens[2].is_name()
+    }
+
+    fn build_item_op(
+        &self,
+        tokens: &mut VecDeque<Token>,
+        _factory: &ItemBlockFactory,
+        _dict: &LanguageDictionary,
+    ) -> Result<VariableRetrieveItemOp, SyntaxError> {
+        let var_name = assert_token(
+            |t| match t {
+                Token::Name(s) => Some(s),
+                _ => None,
+            },
+            Token::Name("variable_name".into()),
+            tokens,
+        )?;
+        assert_token_raw(Token::Dot, tokens)?;
+        let field = assert_token(
+            |t| match t {
+                Token::Name(s) => Some(s),
+                _ => None,
+            },
+            Token::Name("field_name".into()),
+            tokens,
+        )?;
+        Ok(VariableRetrieveItemOp {
+            variable_name: var_name,
+            field_name: Some(field),
+        })
+    }
+}

interpreter/src/lang/vocabulary/item_ops/retrieve_variable.rs

@@ -2,7 +2,7 @@ use core::ops::Deref;
 use std::collections::VecDeque;
 use std::fmt::{Debug, Display, Error, Formatter};
 
-use crate::lang::utility::{assert_token, assert_token_raw};
+use crate::lang::utility::assert_token;
 use crate::lang::LanguageDictionary;
 use crate::lang::{ItemBlockFactory, ItemOp, ItemOpFactory};
 use crate::lang::{RuntimeMsg, SyntaxError};
@@ -12,8 +12,8 @@ use crate::Context;
 
 #[derive(Debug)]
 pub struct VariableRetrieveItemOp {
-    variable_name: String,
+    pub(super) variable_name: String,
-    field_name: Option<String>,
+    pub(super) field_name: Option<String>,
 }
 
 impl Deref for VariableRetrieveItemOp {
@@ -65,11 +65,7 @@ pub struct VariableRetrieveItemOpFactory;
 
 impl ItemOpFactory<VariableRetrieveItemOp> for VariableRetrieveItemOpFactory {
     fn is_item_op(&self, tokens: &VecDeque<Token>) -> bool {
-        (tokens.len() == 1 && tokens[0].is_name())
+        !tokens.is_empty() && tokens[0].is_name()
-            || (tokens.len() == 3
-                && tokens[0].is_name()
-                && tokens[1].is_dot()
-                && tokens[2].is_name())
     }
 
     fn build_item_op(
@@ -86,22 +82,9 @@ impl ItemOpFactory<VariableRetrieveItemOp> for VariableRetrieveItemOpFactory {
             Token::Name("variable_name".into()),
             tokens,
         )?;
-        let field_opt = if tokens.is_empty() {
-            None
-        } else {
-            assert_token_raw(Token::Dot, tokens)?;
-            Some(assert_token(
-                |t| match t {
-                    Token::Name(s) => Some(s),
-                    _ => None,
-                },
-                Token::Name("field_name".into()),
-                tokens,
-            )?)
-        };
         Ok(VariableRetrieveItemOp {
             variable_name: var_name,
-            field_name: field_opt,
+            field_name: None,
         })
     }
 }

interpreter/src/lang/vocabulary/item_ops/subtract.rs

@@ -69,10 +69,11 @@ impl ItemOpFactory<SubtractItemOp> for SubtractItemOpFactory {
         dict: &LanguageDictionary,
     ) -> Result<SubtractItemOp, SyntaxError> {
         let minus_location = first_minus(tokens).unwrap();
-        let mut end_tokens = tokens.split_off(minus_location);
+        let end_tokens = tokens.split_off(minus_location);
         let lhs_op = factory.try_build_item_statement(tokens, dict)?;
-        assert_token_raw(Token::Minus, &mut end_tokens)?;
+        tokens.extend(end_tokens);
-        let rhs_op = factory.try_build_item_statement(&mut end_tokens, dict)?;
+        assert_token_raw(Token::Minus, tokens)?;
+        let rhs_op = factory.try_build_item_statement(tokens, dict)?;
         Ok(SubtractItemOp {
             lhs: lhs_op,
             rhs: rhs_op,
@@ -90,6 +91,8 @@ fn first_minus(tokens: &VecDeque<Token>) -> Option<usize> {
             bracket_depth += 1;
         } else if token.is_close_bracket() && bracket_depth != 0 {
             bracket_depth -= 1;
+        } else if token.is_comma() && bracket_depth == 0 {
+            return None;
         }
     }
     None

interpreter/src/lang/vocabulary/item_ops/variable_declare.rs

@@ -72,7 +72,7 @@ impl ItemOpFactory<VariableDeclareItemOp> for VariableDeclareItemOpFactory {
             Token::Name("variable_name".into()),
             tokens,
         )?;
-        let inner_op: Option<Box<dyn ItemOp>> = if !tokens.is_empty() {
+        let inner_op: Option<Box<dyn ItemOp>> = if !tokens.is_empty() && tokens[0].is_equals() {
             assert_token_raw(Token::Equals, tokens)?;
             Some(factory.try_build_item_statement(tokens, dict)?)
         } else {

interpreter/src/lang/vocabulary/mod.rs

@@ -12,6 +12,7 @@ mod sql_query;
 mod sql_simple_query;
 mod union;
 mod variable_assign;
+mod variable_iter;
 
 pub use empties::{empties_function_factory, EmptiesStatementFactory};
 pub use empty::{empty_function_factory, EmptyStatementFactory};
@@ -26,6 +27,7 @@ pub use sql_query::{sql_function_factory, SqlStatementFactory};
 pub use sql_simple_query::{simple_sql_function_factory, SimpleSqlStatementFactory};
 pub use union::{union_function_factory, UnionStatementFactory};
 pub use variable_assign::{AssignStatement, AssignStatementFactory};
+pub use variable_iter::{VariableRetrieveStatement, VariableRetrieveStatementFactory};
 
 pub mod filters;
 pub mod item_ops;

interpreter/src/lang/vocabulary/repeat.rs

@@ -15,7 +15,6 @@ use crate::lang::{RuntimeError, SyntaxError};
 pub struct RepeatStatement {
     inner_statement: PseudoOp,
     inner_done: bool,
-    context: Option<Context>,
     cache: Vec<Item>,
     cache_position: usize,
     repetitions: usize,
@@ -42,7 +41,6 @@ impl std::clone::Clone for RepeatStatement {
         Self {
             inner_statement: self.inner_statement.clone(),
             inner_done: self.inner_done,
-            context: None,
             cache: self.cache.clone(),
             cache_position: self.cache_position,
             repetitions: self.repetitions,
@@ -60,11 +58,6 @@ impl Iterator for RepeatStatement {
             Err(e) => return Some(Err(e)),
             Ok(real) => real,
         };
-        // give context to inner (should only occur on first run)
-        if self.context.is_some() {
-            let ctx = self.context.take().unwrap();
-            real_op.enter(ctx);
-        }
         if real_op.is_resetable() {
             while self.loop_forever || !self.inner_done {
                 if let Some(item) = real_op.next() {
@@ -73,8 +66,6 @@ impl Iterator for RepeatStatement {
                 if !self.loop_forever {
                     if self.repetitions == 0 {
                         self.inner_done = true;
-                        // take context from inner (should only occur when inner is no longer needed)
-                        self.context = Some(real_op.escape());
                     } else {
                         self.repetitions -= 1;
                         if let Err(e) = real_op.reset() {
@@ -88,17 +79,10 @@ impl Iterator for RepeatStatement {
                     }
                 }
             }
-            if self.context.is_none() {
-                self.context = Some(real_op.escape());
-            }
             None
         } else {
             // cache items in RepeatStatement since inner_statement cannot be reset
             if !self.inner_done {
-                if self.context.is_some() {
-                    let ctx = self.context.take().unwrap();
-                    real_op.enter(ctx);
-                }
                 let inner_item = real_op.next();
                 match inner_item {
                     Some(x) => {
@@ -113,7 +97,6 @@ impl Iterator for RepeatStatement {
                     None => {
                         // inner has completed it's only run
                         self.inner_done = true;
-                        self.context = Some(real_op.escape());
                     }
                 }
             }
@@ -156,27 +139,23 @@ impl Iterator for RepeatStatement {
 
 impl Op for RepeatStatement {
     fn enter(&mut self, ctx: Context) {
-        self.context = Some(ctx)
+        self.inner_statement.try_real().unwrap().enter(ctx)
     }
 
     fn escape(&mut self) -> Context {
-        if self.context.is_some() {
-            self.context.take().unwrap()
-        } else {
         self.inner_statement.try_real().unwrap().escape()
     }
-    }
 
     fn is_resetable(&self) -> bool {
-        true
+        if let Ok(real_op) = self.inner_statement.try_real_ref() {
+            real_op.is_resetable()
+        } else {
+            false
+        }
     }
 
     fn reset(&mut self) -> Result<(), RuntimeError> {
         let real_op = self.inner_statement.try_real()?;
-        if self.context.is_some() {
-            let ctx = self.context.take().unwrap();
-            real_op.enter(ctx);
-        }
         if real_op.is_resetable() {
             real_op.reset()?;
             if self.original_repetitions == 0 {
@@ -204,7 +183,6 @@ impl Op for RepeatStatement {
         let clone = Self {
             inner_statement: PseudoOp::from(self.inner_statement.try_real_ref().unwrap().dup()),
             inner_done: self.original_repetitions == 0,
-            context: None,
             cache: Vec::new(),
             cache_position: 0,
             repetitions: if self.original_repetitions != 0 {
@@ -239,7 +217,7 @@ impl FunctionFactory<RepeatStatement> for RepeatFunctionFactory {
         tokens.extend(end_tokens);
         let mut count: Option<usize> = None;
         let mut inner_done = false;
-        if !tokens.is_empty() {
+        if !tokens.is_empty() && tokens[0].is_comma() {
             // repititions specified
             assert_token_raw(Token::Comma, tokens)?;
             count = Some(assert_token(
@@ -264,7 +242,6 @@ impl FunctionFactory<RepeatStatement> for RepeatFunctionFactory {
         Ok(RepeatStatement {
             inner_statement: inner_statement.into(),
             inner_done,
-            context: None,
             cache: Vec::new(),
             cache_position: 0,
             repetitions: count.unwrap_or(0),

interpreter/src/lang/vocabulary/sorters/empty_sorter.rs

@@ -32,7 +32,7 @@ pub struct EmptySorterFactory;
 
 impl SorterFactory<EmptySorter> for EmptySorterFactory {
     fn is_sorter(&self, tokens: &VecDeque<&Token>) -> bool {
-        tokens.is_empty()
+        !tokens.is_empty() && tokens[0].is_close_bracket()
     }
 
     fn build_sorter(

interpreter/src/lang/vocabulary/union.rs

@@ -163,15 +163,10 @@ impl FunctionFactory<UnionStatement> for UnionFunctionFactory {
     ) -> Result<UnionStatement, SyntaxError> {
         // union(op1, op2, ...)
         let operations = repeated_tokens(
-            |tokens| {
+            |tokens| if tokens[0].is_close_bracket() {
-                if let Some(comma_pos) = next_comma(tokens) {
+                Ok(None)
-                    let end_tokens = tokens.split_off(comma_pos);
-                    let op = dict.try_build_statement(tokens);
-                    tokens.extend(end_tokens);
-                    Ok(Some(PseudoOp::from(op?)))
             } else {
                 Ok(Some(PseudoOp::from(dict.try_build_statement(tokens)?)))
-                }
             },
             Token::Comma,
         )

interpreter/src/lang/vocabulary/variable_iter.rs

@@ -0,0 +1,177 @@
+use std::collections::VecDeque;
+use std::fmt::{Debug, Display, Error, Formatter};
+use std::iter::Iterator;
+
+use crate::tokens::Token;
+use crate::Context;
+
+use crate::lang::utility::assert_token;
+use crate::lang::LanguageDictionary;
+use crate::lang::{BoxedOpFactory, IteratorItem, Op, OpFactory, PseudoOp};
+use crate::lang::{RuntimeError, RuntimeOp, SyntaxError};
+use crate::processing::general::Type;
+
+#[derive(Debug)]
+pub struct VariableRetrieveStatement {
+    variable_name: String,
+    context: Option<Context>,
+    is_tried: bool,
+}
+
+impl Display for VariableRetrieveStatement {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "{}", self.variable_name)
+    }
+}
+
+impl std::clone::Clone for VariableRetrieveStatement {
+    fn clone(&self) -> Self {
+        Self {
+            variable_name: self.variable_name.clone(),
+            context: None,
+            is_tried: self.is_tried,
+        }
+    }
+}
+
+impl Iterator for VariableRetrieveStatement {
+    type Item = IteratorItem;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.is_tried {
+            return None;
+        }
+        let var = self.context.as_mut()
+            .unwrap()
+            .variables.remove(&self.variable_name)
+            .map_err(|e| e.with(RuntimeOp(PseudoOp::from_printable(self))));
+        match var {
+            Ok(Type::Op(mut op)) => {
+                op.enter(self.context.take().unwrap());
+                let next_item = op.next();
+                self.enter(op.escape());
+                if let Err(e) = self.context.as_mut().unwrap().variables.declare(&self.variable_name, Type::Op(op)) {
+                    return Some(Err(e.with(RuntimeOp(PseudoOp::from_printable(self)))));
+                }
+                next_item
+            },
+            Ok(Type::Item(item)) => {
+                self.is_tried = true;
+                if let Err(e) = self.context.as_mut().unwrap().variables.declare(&self.variable_name, Type::Item(item.clone())) {
+                    return Some(Err(e.with(RuntimeOp(PseudoOp::from_printable(self)))));
+                }
+                Some(Ok(item))
+            },
+            Ok(Type::Primitive(p)) => {
+                self.is_tried = true;
+                let err_msg = format!("Cannot iterate over primitive `{}` ({})", self.variable_name, p);
+                if let Err(e) = self.context.as_mut().unwrap().variables.declare(&self.variable_name, Type::Primitive(p)) {
+                    return Some(Err(e.with(RuntimeOp(PseudoOp::from_printable(self)))));
+                }
+                Some(Err(
+                    RuntimeError {
+                        line: 0,
+                        op: PseudoOp::from_printable(self),
+                        msg: err_msg,
+                    }
+                ))
+            }
+            Err(e) => {
+                self.is_tried = true;
+                Some(Err(e))
+            }
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (0, None)
+    }
+}
+
+impl Op for VariableRetrieveStatement {
+    fn enter(&mut self, ctx: Context) {
+        self.context = Some(ctx)
+    }
+
+    fn escape(&mut self) -> Context {
+        self.context.take().unwrap()
+    }
+
+    fn is_resetable(&self) -> bool {
+        if let Some(ctx) = &self.context {
+            let var = ctx.variables.get(&self.variable_name);
+            match var {
+                Ok(Type::Op(op)) => op.is_resetable(),
+                Ok(Type::Primitive(_)) => false,
+                Ok(Type::Item(_)) => true,
+                Err(_) => false,
+            }
+        } else {
+            false
+        }
+    }
+
+    fn reset(&mut self) -> Result<(), RuntimeError> {
+        self.is_tried = false;
+        let runtime_op = RuntimeOp(PseudoOp::from_printable(self));
+        let var = self.context.as_mut()
+            .unwrap()
+            .variables.get_mut(&self.variable_name)
+            .map_err(|e| e.with(runtime_op))?;
+
+        if let Type::Op(op) = var {
+            op.reset()?;
+        }
+        Ok(())
+    }
+
+    fn dup(&self) -> Box<dyn Op> {
+        Box::new(Self {
+            variable_name: self.variable_name.clone(),
+            context: None,
+            is_tried: false,
+        })
+    }
+}
+
+pub struct VariableRetrieveStatementFactory;
+
+impl OpFactory<VariableRetrieveStatement> for VariableRetrieveStatementFactory {
+    fn is_op(&self, tokens: &VecDeque<Token>) -> bool {
+        !tokens.is_empty() && tokens[0].is_name()
+    }
+
+    fn build_op(
+        &self,
+        tokens: &mut VecDeque<Token>,
+        _dict: &LanguageDictionary,
+    ) -> Result<VariableRetrieveStatement, SyntaxError> {
+        let name = assert_token(
+            |t| match t {
+                Token::Name(s) => Some(s),
+                _ => None,
+            },
+            Token::Name("variable_name".into()),
+            tokens,
+        )?;
+        Ok(VariableRetrieveStatement {
+            variable_name: name,
+            context: None,
+            is_tried: false,
+        })
+    }
+}
+
+impl BoxedOpFactory for VariableRetrieveStatementFactory {
+    fn build_op_boxed(
+        &self,
+        tokens: &mut VecDeque<Token>,
+        dict: &LanguageDictionary,
+    ) -> Result<Box<dyn Op>, SyntaxError> {
+        self.build_box(tokens, dict)
+    }
+
+    fn is_op_boxed(&self, tokens: &VecDeque<Token>) -> bool {
+        self.is_op(tokens)
+    }
+}

interpreter/tests/single_line.rs

@@ -315,7 +315,14 @@ fn execute_replacefilter_line() -> Result<(), InterpreterError> {
         true,
     )?;
     execute_single_line(
-        "files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).(if 4: item.() else files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).(0 || 1).(if 200: files() else repeat(item.(), 2)))",
+        r#"files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)
+    .(
+        if 4: item.()
+        else files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)
+            .(0 || 1)
+
+            .(if 200: files() else repeat(item.(), 2))
+    )"#,
         false,
         true,
     )
@@ -323,11 +330,6 @@ fn execute_replacefilter_line() -> Result<(), InterpreterError> {
 
 #[test]
 fn execute_emptysort_line() -> Result<(), InterpreterError> {
-    execute_single_line(
-        "files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).sort()",
-        false,
-        true,
-    )?;
     execute_single_line(
         "files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)~()",
         false,
@@ -367,7 +369,7 @@ fn execute_fieldsort_line() -> Result<(), InterpreterError> {
         true,
     )?;
     execute_single_line(
-        "files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).sort(.not_a_field)",
+        "files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)~(.not_a_field)",
         false,
         true,
     )
@@ -661,6 +663,13 @@ fn execute_bracketsitemop_line() -> Result<(), InterpreterError> {
 
 #[test]
 fn execute_stringifyitemop_line() -> Result<(), InterpreterError> {
+    execute_single_line(
+        "files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).{
+    item.title = ~`test out: {track_number}` item
+}",
+        false,
+        true,
+    )?;
     execute_single_line(
         "files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`).{
     item.filepath = ~`test out: {test}` item,

player/Cargo.toml

@@ -15,6 +15,7 @@ muss-interpreter = { path = "../interpreter", version = "0.9.0" }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 #dbus = { version = "^0.9" }
+# TODO replace mpris-player, maybe with mpris2-zbus https://github.com/pop-os/dbus-settings-bindings/tree/main/mpris2
 mpris-player = { version = "^0.6", path = "../mpris-player", optional = true }
 base64 = { version = "0.13", optional = true }