From 0b55ddda6b07497cada0f15576c7a50dd631e86d Mon Sep 17 00:00:00 2001 From: "NGnius (Graham)" Date: Sun, 30 Jan 2022 21:13:19 -0500 Subject: [PATCH] Add randomisation and explicit reset --- Cargo.lock | 1 + mps-interpreter/Cargo.toml | 1 + mps-interpreter/README.md | 9 ++ mps-interpreter/src/interpretor.rs | 6 +- mps-interpreter/src/lang/vocabulary/mod.rs | 2 + mps-interpreter/src/lang/vocabulary/reset.rs | 105 ++++++++++++++++++ .../src/lang/vocabulary/sorters/mod.rs | 2 + .../src/lang/vocabulary/sorters/shuffle.rs | 99 +++++++++++++++++ mps-interpreter/src/lib.rs | 9 ++ mps-interpreter/tests/single_line.rs | 28 +++++ src/help.rs | 7 ++ 11 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 mps-interpreter/src/lang/vocabulary/reset.rs create mode 100644 mps-interpreter/src/lang/vocabulary/sorters/shuffle.rs diff --git a/Cargo.lock b/Cargo.lock index a03345b..23712b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1133,6 +1133,7 @@ version = "0.4.0" dependencies = [ "bliss-audio", "dirs", + "rand", "regex 1.5.4", "rusqlite", "shellexpand", diff --git a/mps-interpreter/Cargo.toml b/mps-interpreter/Cargo.toml index a3abc44..fcec3d4 100644 --- a/mps-interpreter/Cargo.toml +++ b/mps-interpreter/Cargo.toml @@ -12,6 +12,7 @@ symphonia = { version = "0.4.0", optional = true, features = [ ] } dirs = { version = "4.0.0" } regex = { version = "1" } +rand = { version = "0.8" } shellexpand = { version = "2.1", optional = true } bliss-audio = { version = "0.4", optional = true, path = "../bliss-rs" } diff --git a/mps-interpreter/README.md b/mps-interpreter/README.md index 9ea73ec..92c858c 100644 --- a/mps-interpreter/README.md +++ b/mps-interpreter/README.md @@ -126,6 +126,10 @@ Repeat the iterable count times, or infinite times if count is omitted. Retrieve all files from a folder, matching a regex pattern. +#### reset(iterable); + +Explicitly reset an iterable. This useful for reusing an iterable variable. + #### empty(); Empty iterator. Useful for deleting items using replacement filters. @@ -137,6 +141,11 @@ Operations to sort the items in an iterable: iterable~(sorter) OR iterable.sort( Sort by a MpsItem field. Valid field names change depending on what information is available when the MpsItem is populated, but usually title, artist, album, genre, track, filename are valid fields. Items with a missing/incomparable fields will be sorted to the end. +#### shuffle +#### random shuffle -- e.g. iterable~(shuffle) + +Shuffle the songs in the iterator. This is random for up to 2^16 items, and then the randomness degrades (but at that point you won't notice). The more verbose syntax is allowed in preparation for future randomisation strategies. + #### advanced bliss_first -- e.g. iterable~(advanced bliss_first) Sort by the distance (similarity) from the first song in the iterator. Songs which are more similar (lower distance) to the first song in the iterator will be placed closer to the first song, while less similar songs will be sorted to the end. This uses the [bliss music analyser](https://github.com/polochon-street/bliss-rs), which is a very slow operation and can cause music playback interruptions for large iterators. This requires the `advanced` feature to be enabled (without the feature enabled this is still valid syntax but doesn't change the order). diff --git a/mps-interpreter/src/interpretor.rs b/mps-interpreter/src/interpretor.rs index 8187181..ae3eaf7 100644 --- a/mps-interpreter/src/interpretor.rs +++ b/mps-interpreter/src/interpretor.rs @@ -155,10 +155,13 @@ pub(crate) fn standard_vocab(vocabulary: &mut MpsLanguageDictionary) { .add(crate::lang::vocabulary::filters::field_like_filter()) // sorters .add(crate::lang::vocabulary::sorters::empty_sort()) + .add(crate::lang::vocabulary::sorters::shuffle_sort()) // accepts valid field ~(shuffle) .add(crate::lang::vocabulary::sorters::field_sort()) .add(crate::lang::vocabulary::sorters::bliss_sort()) .add(crate::lang::vocabulary::sorters::bliss_next_sort()) // functions and misc + // functions don't enforce bracket coherence + // -- function().() is valid despite the ).( in between brackets .add(crate::lang::vocabulary::sql_function_factory()) .add(crate::lang::vocabulary::simple_sql_function_factory()) .add(crate::lang::vocabulary::CommentStatementFactory) @@ -166,5 +169,6 @@ pub(crate) fn standard_vocab(vocabulary: &mut MpsLanguageDictionary) { .add(crate::lang::vocabulary::AssignStatementFactory) .add(crate::lang::vocabulary::sql_init_function_factory()) .add(crate::lang::vocabulary::files_function_factory()) - .add(crate::lang::vocabulary::empty_function_factory()); + .add(crate::lang::vocabulary::empty_function_factory()) + .add(crate::lang::vocabulary::reset_function_factory()); } diff --git a/mps-interpreter/src/lang/vocabulary/mod.rs b/mps-interpreter/src/lang/vocabulary/mod.rs index 596b119..53665de 100644 --- a/mps-interpreter/src/lang/vocabulary/mod.rs +++ b/mps-interpreter/src/lang/vocabulary/mod.rs @@ -2,6 +2,7 @@ mod comment; mod empty; mod files; mod repeat; +mod reset; mod sql_init; mod sql_query; mod sql_simple_query; @@ -11,6 +12,7 @@ pub use comment::{CommentStatement, CommentStatementFactory}; pub use empty::{empty_function_factory, EmptyStatementFactory}; pub use files::{files_function_factory, FilesStatementFactory}; pub use repeat::{repeat_function_factory, RepeatStatementFactory}; +pub use reset::{reset_function_factory, ResetStatementFactory}; pub use sql_init::{sql_init_function_factory, SqlInitStatementFactory}; pub use sql_query::{sql_function_factory, SqlStatementFactory}; pub use sql_simple_query::{simple_sql_function_factory, SimpleSqlStatementFactory}; diff --git a/mps-interpreter/src/lang/vocabulary/reset.rs b/mps-interpreter/src/lang/vocabulary/reset.rs new file mode 100644 index 0000000..bc7fef1 --- /dev/null +++ b/mps-interpreter/src/lang/vocabulary/reset.rs @@ -0,0 +1,105 @@ +use std::collections::VecDeque; +use std::fmt::{Debug, Display, Error, Formatter}; +use std::iter::Iterator; + +use crate::tokens::MpsToken; +use crate::MpsContext; + +use crate::lang::MpsLanguageDictionary; +use crate::lang::PseudoOp; +use crate::lang::{MpsFunctionFactory, MpsFunctionStatementFactory, MpsIteratorItem, MpsOp}; +use crate::lang::{RuntimeError, SyntaxError}; + +#[derive(Debug)] +pub struct ResetStatement { + context: Option, + inner: PseudoOp, + // state + has_tried: bool, +} + +impl Display for ResetStatement { + fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { + write!(f, "reset({})", &self.inner) + } +} + +impl std::clone::Clone for ResetStatement { + fn clone(&self) -> Self { + Self { + context: None, + inner: self.inner.clone(), + has_tried: self.has_tried, + } + } +} + +impl Iterator for ResetStatement { + type Item = MpsIteratorItem; + + fn next(&mut self) -> Option { + if !self.has_tried { + self.has_tried = true; + let inner = match self.inner.try_real() { + Ok(x) => x, + Err(e) => return Some(Err(e)), + }; + match inner.reset() { + Ok(_) => {}, + Err(e) => return Some(Err(e)), + }; + } + None + } +} + +impl MpsOp for ResetStatement { + fn enter(&mut self, ctx: MpsContext) { + self.context = Some(ctx) + } + + fn escape(&mut self) -> MpsContext { + self.context.take().unwrap() + } + + fn is_resetable(&self) -> bool { + true + } + + fn reset(&mut self) -> Result<(), RuntimeError> { + self.has_tried = false; + Ok(()) + } +} + +pub struct ResetFunctionFactory; + +impl MpsFunctionFactory for ResetFunctionFactory { + fn is_function(&self, name: &str) -> bool { + name == "reset" + } + + fn build_function_params( + &self, + _name: String, + #[allow(unused_variables)] + tokens: &mut VecDeque, + dict: &MpsLanguageDictionary, + ) -> Result { + // reset(var) + let inner_op = dict.try_build_statement(tokens)?.into(); + Ok(ResetStatement { + context: None, + inner: inner_op, + has_tried: false, + }) + } +} + +pub type ResetStatementFactory = MpsFunctionStatementFactory; + +#[inline(always)] +pub fn reset_function_factory() -> ResetStatementFactory { + ResetStatementFactory::new(ResetFunctionFactory) +} + diff --git a/mps-interpreter/src/lang/vocabulary/sorters/mod.rs b/mps-interpreter/src/lang/vocabulary/sorters/mod.rs index 5267d02..59f67a6 100644 --- a/mps-interpreter/src/lang/vocabulary/sorters/mod.rs +++ b/mps-interpreter/src/lang/vocabulary/sorters/mod.rs @@ -2,8 +2,10 @@ mod bliss_sorter; mod bliss_next_sorter; mod empty_sorter; mod field_sorter; +mod shuffle; pub use bliss_sorter::{bliss_sort, BlissSorter, BlissSorterFactory, BlissSorterStatementFactory}; pub use bliss_next_sorter::{bliss_next_sort, BlissNextSorter, BlissNextSorterFactory, BlissNextSorterStatementFactory}; pub use empty_sorter::{empty_sort, EmptySorter, EmptySorterFactory, EmptySorterStatementFactory}; pub use field_sorter::{field_sort, FieldSorter, FieldSorterFactory, FieldSorterStatementFactory}; +pub use shuffle::{shuffle_sort, ShuffleSorter, ShuffleSorterFactory, ShuffleSorterStatementFactory}; diff --git a/mps-interpreter/src/lang/vocabulary/sorters/shuffle.rs b/mps-interpreter/src/lang/vocabulary/sorters/shuffle.rs new file mode 100644 index 0000000..449ed48 --- /dev/null +++ b/mps-interpreter/src/lang/vocabulary/sorters/shuffle.rs @@ -0,0 +1,99 @@ +use std::collections::VecDeque; +use std::fmt::{Debug, Display, Error, Formatter}; + +use rand::{thread_rng, Rng}; + +use crate::lang::{MpsIteratorItem, MpsLanguageDictionary, MpsOp}; +use crate::lang::{MpsSortStatementFactory, MpsSorter, MpsSorterFactory}; +use crate::lang::{RuntimeError, SyntaxError}; +use crate::lang::utility::{check_name, assert_name}; +use crate::processing::OpGetter; +use crate::tokens::MpsToken; + +const RNG_LIMIT_BITMASK: usize = 0xffff; // bits to preserve in RNG +// imposes an upper limit in the name of optimisation which reduces randomness past this point +// this is also an effective item_buf size limit, 2^16 - 1 seems reasonable + +#[derive(Debug, Clone)] +pub struct ShuffleSorter; + +impl MpsSorter for ShuffleSorter { + fn sort( + &mut self, + iterator: &mut dyn MpsOp, + item_buf: &mut VecDeque, + _op: &mut OpGetter, + ) -> Result<(), RuntimeError> { + // iterative shuffling algorithm + // + // choose a random number r + // loop: + // if buffer length > r: return buffer[r] (removing buffer[r]) + // else: + // traverse iterator until r - buffer length is encountered + // fill buffer with items as it passes + // if end of iterator encountered: r = r % buffer length, repeat loop + // else: return iterator item + // + // the following is similar, except using VecDeque.swap_remove_back() to avoid large memory moves + let r: usize = thread_rng().gen(); + let mut random: usize = r & RNG_LIMIT_BITMASK; + loop { + if item_buf.len() > random { + let item = item_buf.swap_remove_back(random).unwrap(); + item_buf.push_front(item); + return Ok(()); + } + let mut iterator_pos = item_buf.len(); + while let Some(item) = iterator.next() { + if iterator_pos == random { + item_buf.push_front(item); + return Ok(()); + } else { + iterator_pos += 1; + item_buf.push_back(item); + } + } + // end case: everything is completely empty -- end loop without a new result + if item_buf.len() == 0 { + return Ok(()); + } + random = random % item_buf.len(); + } + } +} + +impl Display for ShuffleSorter { + fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { + write!(f, "random shuffle") + } +} + +pub struct ShuffleSorterFactory; + +impl MpsSorterFactory for ShuffleSorterFactory { + fn is_sorter(&self, tokens: &VecDeque<&MpsToken>) -> bool { + (tokens.len() == 1 && check_name("shuffle", &tokens[0])) + || + (tokens.len() == 2 && check_name("random", &tokens[0]) && check_name("shuffle", &tokens[1])) + } + + fn build_sorter( + &self, + tokens: &mut VecDeque, + _dict: &MpsLanguageDictionary, + ) -> Result { + if check_name("random", &tokens[0]) { + assert_name("random", tokens)?; + } + assert_name("shuffle", tokens)?; + Ok(ShuffleSorter) + } +} + +pub type ShuffleSorterStatementFactory = MpsSortStatementFactory; + +#[inline(always)] +pub fn shuffle_sort() -> ShuffleSorterStatementFactory { + ShuffleSorterStatementFactory::new(ShuffleSorterFactory) +} diff --git a/mps-interpreter/src/lib.rs b/mps-interpreter/src/lib.rs index 1abaa70..f2e9904 100644 --- a/mps-interpreter/src/lib.rs +++ b/mps-interpreter/src/lib.rs @@ -124,6 +124,10 @@ //! //! Retrieve all files from a folder, matching a regex pattern. //! +//! ### reset(iterable); +//! +//! Explicitly reset an iterable. This useful for reusing an iterable variable. +//! //! ### empty(); //! //! Empty iterator. Useful for deleting items using replacement filters. @@ -135,6 +139,11 @@ //! //! Sort by a MpsItem field. Valid field names change depending on what information is available when the MpsItem is populated, but usually title, artist, album, genre, track, filename are valid fields. Items with a missing/incomparable fields will be sorted to the end. //! +//! ### shuffle +//! ### random shuffle -- e.g. iterable~(shuffle) +//! +//! Shuffle the songs in the iterator. This is random for up to 2^16 items, and then the randomness degrades (but at that point you won't notice). The more verbose syntax is allowed in preparation for future randomisation strategies. +//! //! ### advanced bliss_first -- e.g. iterable~(advanced bliss_first) //! //! Sort by the distance (similarity) from the first song in the iterator. Songs which are more similar (lower distance) to the first song in the iterator will be placed closer to the first song, while less similar songs will be sorted to the end. This uses the [bliss music analyser](https://github.com/polochon-street/bliss-rs), which is a very slow operation and can cause music playback interruptions for large iterators. This requires the `advanced` feature to be enabled (without the feature enabled this is still valid syntax but doesn't change the order). diff --git a/mps-interpreter/tests/single_line.rs b/mps-interpreter/tests/single_line.rs index abf4c3c..cdd3c16 100644 --- a/mps-interpreter/tests/single_line.rs +++ b/mps-interpreter/tests/single_line.rs @@ -385,3 +385,31 @@ fn execute_emptyfn_line() -> Result<(), Box> { true, ) } + +#[test] +fn execute_resetfn_line() -> Result<(), Box> { + execute_single_line( + "reset(empty())", + true, + true, + ) +} + +#[test] +fn execute_shufflesort_line() -> Result<(), Box> { + execute_single_line( + "files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)~(random shuffle)", + false, + true, + )?; + execute_single_line( + "files(`~/Music/MusicFlac/Bruno Mars/24K Magic/`)~(shuffle)", + false, + true, + )?; + execute_single_line( + "empty()~(shuffle)", + true, + true, + ) +} diff --git a/src/help.rs b/src/help.rs index 72f34af..ae10680 100644 --- a/src/help.rs +++ b/src/help.rs @@ -33,6 +33,9 @@ These always return an iterable which can be manipulated. files(folder = `path/to/music`, recursive = true|false, regex = `pattern`) Retrieve all files from a folder, matching a regex pattern. + reset(iterable) + Explicitly reset an iterable. This useful for reusing an iterable variable. + empty() Empty iterator. Useful for deleting items using replacement filters."; @@ -74,6 +77,10 @@ Operations to sort the items in an iterable: iterable~(sorter) OR iterable.sort( field -- e.g. iterable~(filename) Sort by a MpsItem field. Valid field names change depending on what information is available when the MpsItem is populated, but usually title, artist, album, genre, track, filename are valid fields. Items with a missing/incomparable fields will be sorted to the end. + shuffle + random shuffle -- e.g. iterable~(shuffle) + Shuffle the songs in the iterator. This is random for up to 2^16 items, and then the randomness degrades (but at that point you won't notice). + advanced bliss_first -- e.g. iterable~(advanced bliss_first) Sort by the distance (similarity) from the first song in the iterator. Songs which are more similar (lower distance) to the first song in the iterator will be placed closer to the first song, while less similar songs will be sorted to the end. This uses the bliss music analyser, which is a very slow operation and can cause music playback interruptions for large iterators. Requires `advanced` mps-interpreter feature.