Ignore comments when processing tokens instead of at runtime

mps-interpreter/src/interpretor.rs

@@ -48,7 +48,6 @@ pub(crate) fn standard_vocab(vocabulary: &mut MpsLanguageDictionary) {
         // -- function().() is valid despite the ).( in between brackets
         .add(crate::lang::vocabulary::sql_function_factory())
         .add(crate::lang::vocabulary::simple_sql_function_factory())
-        .add(crate::lang::vocabulary::CommentStatementFactory)
         .add(crate::lang::vocabulary::repeat_function_factory())
         .add(crate::lang::vocabulary::AssignStatementFactory)
         .add(crate::lang::vocabulary::sql_init_function_factory())

mps-interpreter/src/lang/vocabulary/comment.rs

@@ -1,110 +0,0 @@
-use std::collections::VecDeque;
-use std::fmt::{Debug, Display, Error, Formatter};
-use std::iter::Iterator;
-
-use crate::tokens::MpsToken;
-use crate::MpsContext;
-
-use crate::lang::utility::assert_token;
-use crate::lang::MpsLanguageDictionary;
-use crate::lang::SyntaxError;
-use crate::lang::{BoxedMpsOpFactory, MpsIteratorItem, MpsOp, MpsOpFactory, SimpleMpsOpFactory};
-
-#[derive(Debug)]
-pub struct CommentStatement {
-    comment: String,
-    context: Option<MpsContext>,
-}
-
-impl CommentStatement {
-    /*fn comment_text(&self) -> String {
-        let mut clone = self.comment.clone();
-        if clone.starts_with("#") {
-            clone.replace_range(..1, ""); // remove "#"
-        } else {
-            clone.replace_range(..2, ""); // remove "//"
-        }
-        clone
-    }*/
-}
-
-impl Display for CommentStatement {
-    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
-        write!(f, "{}", self.comment)
-    }
-}
-
-impl std::clone::Clone for CommentStatement {
-    fn clone(&self) -> Self {
-        Self {
-            comment: self.comment.clone(),
-            context: None,
-        }
-    }
-}
-
-impl Iterator for CommentStatement {
-    type Item = MpsIteratorItem;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        None
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        (0, Some(0))
-    }
-}
-
-impl MpsOp for CommentStatement {
-    fn enter(&mut self, ctx: MpsContext) {
-        self.context = Some(ctx)
-    }
-
-    fn escape(&mut self) -> MpsContext {
-        self.context.take().unwrap()
-    }
-
-    fn dup(&self) -> Box<dyn MpsOp> {
-        Box::new(self.clone())
-    }
-}
-
-pub struct CommentStatementFactory;
-
-impl SimpleMpsOpFactory<CommentStatement> for CommentStatementFactory {
-    fn is_op_simple(&self, tokens: &VecDeque<MpsToken>) -> bool {
-        tokens.len() == 1 && tokens[0].is_comment()
-    }
-
-    fn build_op_simple(
-        &self,
-        tokens: &mut VecDeque<MpsToken>,
-    ) -> Result<CommentStatement, SyntaxError> {
-        let comment = assert_token(
-            |t| match t {
-                MpsToken::Comment(c) => Some(c),
-                _ => None,
-            },
-            MpsToken::Comment("comment".into()),
-            tokens,
-        )?;
-        Ok(CommentStatement {
-            comment,
-            context: None,
-        })
-    }
-}
-
-impl BoxedMpsOpFactory for CommentStatementFactory {
-    fn build_op_boxed(
-        &self,
-        tokens: &mut VecDeque<MpsToken>,
-        dict: &MpsLanguageDictionary,
-    ) -> Result<Box<dyn MpsOp>, SyntaxError> {
-        self.build_box(tokens, dict)
-    }
-
-    fn is_op_boxed(&self, tokens: &VecDeque<MpsToken>) -> bool {
-        self.is_op(tokens)
-    }
-}

mps-interpreter/src/lang/vocabulary/mod.rs

@@ -1,4 +1,3 @@
-mod comment;
 mod empties;
 pub(crate) mod empty;
 mod files;
@@ -11,7 +10,6 @@ mod sql_simple_query;
 mod union;
 mod variable_assign;
 
-pub use comment::{CommentStatement, CommentStatementFactory};
 pub use empties::{empties_function_factory, EmptiesStatementFactory};
 pub use empty::{empty_function_factory, EmptyStatementFactory};
 pub use files::{files_function_factory, FilesStatementFactory};

mps-interpreter/src/tokens/tokenizer.rs

@@ -67,9 +67,10 @@ where
                     bigger_buf.clear();
                 }
                 ReaderStateMachine::EndComment {} => {
-                    let comment = String::from_utf8(bigger_buf.clone())
+                    //let _comment = String::from_utf8(bigger_buf.clone())
-                        .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?;
+                    //    .map_err(|e| self.error(format!("UTF-8 encoding error: {}", e)))?;
-                    buf.push_back(MpsToken::Comment(comment));
+                    // ignore comments
+                    //buf.push_back(MpsToken::Comment(comment));
                     bigger_buf.clear();
                 }
                 ReaderStateMachine::EndToken {} => {

mps-interpreter/tests/single_line.rs

@@ -746,6 +746,19 @@ fn execute_iteritemop_line() -> Result<(), MpsError> {
     )
 }
 
+#[test]
+fn execute_commentitemop_line() -> Result<(), MpsError> {
+    execute_single_line(
+        "empty().{
+    // this is a comment
+    // this is another comment
+    # this is also a comment
+}",
+        true,
+        true,
+    )
+}
+
 #[test]
 fn execute_uniquefieldfilter_line() -> Result<(), MpsError> {
     execute_single_line(