Add randomisation and explicit reset

Cargo.lock

@@ -1133,6 +1133,7 @@ version = "0.4.0"
 dependencies = [
  "bliss-audio",
  "dirs",
+ "rand",
  "regex 1.5.4",
  "rusqlite",
  "shellexpand",

mps-interpreter/Cargo.toml

@@ -12,6 +12,7 @@ symphonia = { version = "0.4.0", optional = true, features = [
 ] }
 dirs = { version = "4.0.0" }
 regex = { version = "1" }
+rand = { version = "0.8" }
 shellexpand = { version = "2.1", optional = true }
 bliss-audio = { version = "0.4", optional = true, path = "../bliss-rs" }
 

mps-interpreter/README.md

@@ -126,6 +126,10 @@ Repeat the iterable count times, or infinite times if count is omitted.
 
 Retrieve all files from a folder, matching a regex pattern.
 
+#### reset(iterable);
+
+Explicitly reset an iterable. This useful for reusing an iterable variable.
+
 #### empty();
 
 Empty iterator. Useful for deleting items using replacement filters.
@@ -137,6 +141,11 @@ Operations to sort the items in an iterable: iterable~(sorter) OR iterable.sort(
 
 Sort by a MpsItem field. Valid field names change depending on what information is available when the MpsItem is populated, but usually title, artist, album, genre, track, filename are valid fields. Items with a missing/incomparable fields will be sorted to the end.
 
+#### shuffle
+#### random shuffle -- e.g. iterable~(shuffle)
+
+Shuffle the songs in the iterator. This is random for up to 2^16 items, and then the randomness degrades (but at that point you won't notice). The more verbose syntax is allowed in preparation for future randomisation strategies.
+
 #### advanced bliss_first -- e.g. iterable~(advanced bliss_first)
 
 Sort by the distance (similarity) from the first song in the iterator. Songs which are more similar (lower distance) to the first song in the iterator will be placed closer to the first song, while less similar songs will be sorted to the end. This uses the [bliss music analyser](https://github.com/polochon-street/bliss-rs), which is a very slow operation and can cause music playback interruptions for large iterators. This requires the `advanced` feature to be enabled (without the feature enabled this is still valid syntax but doesn't change the order).

mps-interpreter/src/interpretor.rs

@@ -155,10 +155,13 @@ pub(crate) fn standard_vocab(vocabulary: &mut MpsLanguageDictionary) {
         .add(crate::lang::vocabulary::filters::field_like_filter())
         // sorters
         .add(crate::lang::vocabulary::sorters::empty_sort())
+        .add(crate::lang::vocabulary::sorters::shuffle_sort()) // accepts valid field ~(shuffle)
         .add(crate::lang::vocabulary::sorters::field_sort())
         .add(crate::lang::vocabulary::sorters::bliss_sort())
         .add(crate::lang::vocabulary::sorters::bliss_next_sort())
         // functions and misc
+        // functions don't enforce bracket coherence
+        // -- function().() is valid despite the ).( in between brackets
         .add(crate::lang::vocabulary::sql_function_factory())
         .add(crate::lang::vocabulary::simple_sql_function_factory())
         .add(crate::lang::vocabulary::CommentStatementFactory)
@@ -166,5 +169,6 @@ pub(crate) fn standard_vocab(vocabulary: &mut MpsLanguageDictionary) {
         .add(crate::lang::vocabulary::AssignStatementFactory)
         .add(crate::lang::vocabulary::sql_init_function_factory())
         .add(crate::lang::vocabulary::files_function_factory())
-        .add(crate::lang::vocabulary::empty_function_factory());
+        .add(crate::lang::vocabulary::empty_function_factory())
+        .add(crate::lang::vocabulary::reset_function_factory());
 }

mps-interpreter/src/lang/vocabulary/mod.rs

@@ -2,6 +2,7 @@ mod comment;
 mod empty;
 mod files;
 mod repeat;
+mod reset;
 mod sql_init;
 mod sql_query;
 mod sql_simple_query;
@@ -11,6 +12,7 @@ pub use comment::{CommentStatement, CommentStatementFactory};
 pub use empty::{empty_function_factory, EmptyStatementFactory};
 pub use files::{files_function_factory, FilesStatementFactory};
 pub use repeat::{repeat_function_factory, RepeatStatementFactory};
+pub use reset::{reset_function_factory, ResetStatementFactory};
 pub use sql_init::{sql_init_function_factory, SqlInitStatementFactory};
 pub use sql_query::{sql_function_factory, SqlStatementFactory};
 pub use sql_simple_query::{simple_sql_function_factory, SimpleSqlStatementFactory};

mps-interpreter/src/lang/vocabulary/reset.rs

@@ -0,0 +1,105 @@
+use std::collections::VecDeque;
+use std::fmt::{Debug, Display, Error, Formatter};
+use std::iter::Iterator;
+
+use crate::tokens::MpsToken;
+use crate::MpsContext;
+
+use crate::lang::MpsLanguageDictionary;
+use crate::lang::PseudoOp;
+use crate::lang::{MpsFunctionFactory, MpsFunctionStatementFactory, MpsIteratorItem, MpsOp};
+use crate::lang::{RuntimeError, SyntaxError};
+
+#[derive(Debug)]
+pub struct ResetStatement {
+    context: Option<MpsContext>,
+    inner: PseudoOp,
+    // state
+    has_tried: bool,
+}
+
+impl Display for ResetStatement {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "reset({})", &self.inner)
+    }
+}
+
+impl std::clone::Clone for ResetStatement {
+    fn clone(&self) -> Self {
+        Self {
+            context: None,
+            inner: self.inner.clone(),
+            has_tried: self.has_tried,
+        }
+    }
+}
+
+impl Iterator for ResetStatement {
+    type Item = MpsIteratorItem;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if !self.has_tried {
+            self.has_tried = true;
+            let inner = match self.inner.try_real() {
+                Ok(x) => x,
+                Err(e) => return Some(Err(e)),
+            };
+            match inner.reset() {
+                Ok(_) => {},
+                Err(e) => return Some(Err(e)),
+            };
+        }
+        None
+    }
+}
+
+impl MpsOp for ResetStatement {
+    fn enter(&mut self, ctx: MpsContext) {
+        self.context = Some(ctx)
+    }
+
+    fn escape(&mut self) -> MpsContext {
+        self.context.take().unwrap()
+    }
+
+    fn is_resetable(&self) -> bool {
+        true
+    }
+
+    fn reset(&mut self) -> Result<(), RuntimeError> {
+        self.has_tried = false;
+        Ok(())
+    }
+}
+
+pub struct ResetFunctionFactory;
+
+impl MpsFunctionFactory<ResetStatement> for ResetFunctionFactory {
+    fn is_function(&self, name: &str) -> bool {
+        name == "reset"
+    }
+
+    fn build_function_params(
+        &self,
+        _name: String,
+        #[allow(unused_variables)]
+        tokens: &mut VecDeque<MpsToken>,
+        dict: &MpsLanguageDictionary,
+    ) -> Result<ResetStatement, SyntaxError> {
+        // reset(var)
+        let inner_op = dict.try_build_statement(tokens)?.into();
+        Ok(ResetStatement {
+            context: None,
+            inner: inner_op,
+            has_tried: false,
+        })
+    }
+}
+
+pub type ResetStatementFactory = MpsFunctionStatementFactory<ResetStatement, ResetFunctionFactory>;
+
+#[inline(always)]
+pub fn reset_function_factory() -> ResetStatementFactory {
+    ResetStatementFactory::new(ResetFunctionFactory)
+}
+

mps-interpreter/src/lang/vocabulary/sorters/mod.rs

@@ -2,8 +2,10 @@ mod bliss_sorter;
 mod bliss_next_sorter;
 mod empty_sorter;
 mod field_sorter;
+mod shuffle;
 
 pub use bliss_sorter::{bliss_sort, BlissSorter, BlissSorterFactory, BlissSorterStatementFactory};
 pub use bliss_next_sorter::{bliss_next_sort, BlissNextSorter, BlissNextSorterFactory, BlissNextSorterStatementFactory};
 pub use empty_sorter::{empty_sort, EmptySorter, EmptySorterFactory, EmptySorterStatementFactory};
 pub use field_sorter::{field_sort, FieldSorter, FieldSorterFactory, FieldSorterStatementFactory};
+pub use shuffle::{shuffle_sort, ShuffleSorter, ShuffleSorterFactory, ShuffleSorterStatementFactory};

mps-interpreter/src/lang/vocabulary/sorters/shuffle.rs

@@ -0,0 +1,99 @@
+use std::collections::VecDeque;
+use std::fmt::{Debug, Display, Error, Formatter};
+
+use rand::{thread_rng, Rng};
+
+use crate::lang::{MpsIteratorItem, MpsLanguageDictionary, MpsOp};
+use crate::lang::{MpsSortStatementFactory, MpsSorter, MpsSorterFactory};
+use crate::lang::{RuntimeError, SyntaxError};
+use crate::lang::utility::{check_name, assert_name};
+use crate::processing::OpGetter;
+use crate::tokens::MpsToken;
+
+const RNG_LIMIT_BITMASK: usize = 0xffff; // bits to preserve in RNG
+// imposes an upper limit in the name of optimisation which reduces randomness past this point
+// this is also an effective item_buf size limit, 2^16 - 1 seems reasonable
+
+#[derive(Debug, Clone)]
+pub struct ShuffleSorter;
+
+impl MpsSorter for ShuffleSorter {
+    fn sort(
+        &mut self,
+        iterator: &mut dyn MpsOp,
+        item_buf: &mut VecDeque<MpsIteratorItem>,
+        _op: &mut OpGetter,
+    ) -> Result<(), RuntimeError> {
+        // iterative shuffling algorithm
+        //
+        // choose a random number r
+        // loop:
+        //   if buffer length > r: return buffer[r] (removing buffer[r])
+        //   else:
+        //     traverse iterator until r - buffer length is encountered
+        //     fill buffer with items as it passes
+        //     if end of iterator encountered: r = r % buffer length, repeat loop
+        //     else: return iterator item
+        //
+        // the following is similar, except using VecDeque.swap_remove_back() to avoid large memory moves
+        let r: usize = thread_rng().gen();
+        let mut random: usize = r & RNG_LIMIT_BITMASK;
+        loop {
+            if item_buf.len() > random {
+                let item = item_buf.swap_remove_back(random).unwrap();
+                item_buf.push_front(item);
+                return Ok(());
+            }
+            let mut iterator_pos = item_buf.len();
+            while let Some(item) = iterator.next() {
+                if iterator_pos == random {
+                    item_buf.push_front(item);
+                    return Ok(());
+                } else {
+                    iterator_pos += 1;
+                    item_buf.push_back(item);
+                }
+            }
+            // end case: everything is completely empty -- end loop without a new result
+            if item_buf.len() == 0 {
+                return Ok(());
+            }
+            random = random % item_buf.len();
+        }
+    }
+}
+
+impl Display for ShuffleSorter {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
+        write!(f, "random shuffle")
+    }
+}
+
+pub struct ShuffleSorterFactory;
+
+impl MpsSorterFactory<ShuffleSorter> for ShuffleSorterFactory {
+    fn is_sorter(&self, tokens: &VecDeque<&MpsToken>) -> bool {
+        (tokens.len() == 1 && check_name("shuffle", &tokens[0]))
+        ||
+        (tokens.len() == 2 && check_name("random", &tokens[0]) && check_name("shuffle", &tokens[1]))
+    }
+
+    fn build_sorter(
+        &self,
+        tokens: &mut VecDeque<MpsToken>,
+        _dict: &MpsLanguageDictionary,
+    ) -> Result<ShuffleSorter, SyntaxError> {
+        if check_name("random", &tokens[0]) {
+            assert_name("random", tokens)?;
+        }
+        assert_name("shuffle", tokens)?;
+        Ok(ShuffleSorter)
+    }
+}
+
+pub type ShuffleSorterStatementFactory = MpsSortStatementFactory<ShuffleSorter, ShuffleSorterFactory>;
+
+#[inline(always)]
+pub fn shuffle_sort() -> ShuffleSorterStatementFactory {
+    ShuffleSorterStatementFactory::new(ShuffleSorterFactory)
+}

mps-interpreter/src/lib.rs

@@ -124,6 +124,10 @@
 //!
 //! Retrieve all files from a folder, matching a regex pattern.
 //!
+//! ### reset(iterable);
+//!
+//! Explicitly reset an iterable. This useful for reusing an iterable variable.
+//!
 //! ### empty();
 //!
 //! Empty iterator. Useful for deleting items using replacement filters.
@@ -135,6 +139,11 @@
 //!
 //! Sort by a MpsItem field. Valid field names change depending on what information is available when the MpsItem is populated, but usually title, artist, album, genre, track, filename are valid fields. Items with a missing/incomparable fields will be sorted to the end.
 //!
+//! ### shuffle
+//! ### random shuffle -- e.g. iterable~(shuffle)
+//!
+//! Shuffle the songs in the iterator. This is random for up to 2^16 items, and then the randomness degrades (but at that point you won't notice). The more verbose syntax is allowed in preparation for future randomisation strategies.
+//!
 //! ### advanced bliss_first -- e.g. iterable~(advanced bliss_first)
 //!
 //! Sort by the distance (similarity) from the first song in the iterator. Songs which are more similar (lower distance) to the first song in the iterator will be placed closer to the first song, while less similar songs will be sorted to the end. This uses the [bliss music analyser](https://github.com/polochon-street/bliss-rs), which is a very slow operation and can cause music playback interruptions for large iterators. This requires the `advanced` feature to be enabled (without the feature enabled this is still valid syntax but doesn't change the order).

mps-interpreter/tests/single_line.rs

@@ -385,3 +385,31 @@ fn execute_emptyfn_line() -> Result<(), Box<dyn MpsLanguageError>> {
         true,
     )
 }
+
+#[test]
+fn execute_resetfn_line() -> Result<(), Box<dyn MpsLanguageError>> {
+    execute_single_line(
+        "reset(empty())",
+        true,
+        true,
+    )
+}
+
+#[test]
+fn execute_shufflesort_line() -> Result<(), Box<dyn MpsLanguageError>> {
+    execute_single_line(
+        "files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)~(random shuffle)",
+        false,
+        true,
+    )?;
+    execute_single_line(
+        "files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)~(shuffle)",
+        false,
+        true,
+    )?;
+    execute_single_line(
+        "empty()~(shuffle)",
+        true,
+        true,
+    )
+}

src/help.rs

@@ -33,6 +33,9 @@ These always return an iterable which can be manipulated.
  files(folder = `path/to/music`, recursive = true|false, regex = `pattern`)
     Retrieve all files from a folder, matching a regex pattern.
 
+ reset(iterable)
+    Explicitly reset an iterable. This useful for reusing an iterable variable.
+
  empty()
     Empty iterator. Useful for deleting items using replacement filters.";
 
@@ -74,6 +77,10 @@ Operations to sort the items in an iterable: iterable~(sorter) OR iterable.sort(
  field -- e.g. iterable~(filename)
     Sort by a MpsItem field. Valid field names change depending on what information is available when the MpsItem is populated, but usually title, artist, album, genre, track, filename are valid fields. Items with a missing/incomparable fields will be sorted to the end.
 
+ shuffle
+ random shuffle -- e.g. iterable~(shuffle)
+    Shuffle the songs in the iterator. This is random for up to 2^16 items, and then the randomness degrades (but at that point you won't notice).
+
  advanced bliss_first -- e.g. iterable~(advanced bliss_first)
     Sort by the distance (similarity) from the first song in the iterator. Songs which are more similar (lower distance) to the first song in the iterator will be placed closer to the first song, while less similar songs will be sorted to the end. This uses the bliss music analyser, which is a very slow operation and can cause music playback interruptions for large iterators. Requires `advanced` mps-interpreter feature.