From 34487c02ebef452d372b056129fd31009ffdb033 Mon Sep 17 00:00:00 2001 From: "NGnius (Graham)" Date: Sat, 14 May 2022 11:10:03 -0400 Subject: [PATCH] Improve tag processing and filtering with ?? filter --- mps-interpreter/src/interpretor.rs | 1 + .../src/lang/vocabulary/filters/mod.rs | 4 ++ .../vocabulary/filters/nonempty_filter.rs | 65 +++++++++++++++++++ mps-interpreter/src/lib.rs | 4 ++ mps-interpreter/src/music/tag.rs | 9 +++ mps-interpreter/src/processing/filesystem.rs | 59 ++++++++--------- .../src/processing/music_analysis.rs | 9 ++- mps-interpreter/tests/single_line.rs | 16 +++++ mps-m3u8/src/main.rs | 33 ++++++++-- mps-player/src/player.rs | 24 ++++++- src/help.rs | 5 +- 11 files changed, 187 insertions(+), 42 deletions(-) create mode 100644 mps-interpreter/src/lang/vocabulary/filters/nonempty_filter.rs diff --git a/mps-interpreter/src/interpretor.rs b/mps-interpreter/src/interpretor.rs index 1d60397..f3dace5 100644 --- a/mps-interpreter/src/interpretor.rs +++ b/mps-interpreter/src/interpretor.rs @@ -13,6 +13,7 @@ pub(crate) fn standard_vocab(vocabulary: &mut MpsLanguageDictionary) { .add(crate::lang::vocabulary::filters::field_like_filter()) .add(crate::lang::vocabulary::filters::field_re_filter()) .add(crate::lang::vocabulary::filters::unique_field_filter()) + .add(crate::lang::vocabulary::filters::nonempty_filter()) // sorters .add(crate::lang::vocabulary::sorters::empty_sort()) .add(crate::lang::vocabulary::sorters::shuffle_sort()) // accepts ~(shuffle) diff --git a/mps-interpreter/src/lang/vocabulary/filters/mod.rs b/mps-interpreter/src/lang/vocabulary/filters/mod.rs index 0d48710..fa3dbbf 100644 --- a/mps-interpreter/src/lang/vocabulary/filters/mod.rs +++ b/mps-interpreter/src/lang/vocabulary/filters/mod.rs @@ -4,6 +4,7 @@ mod field_filter_maybe; mod field_like_filter; mod field_match_filter; mod index_filter; +mod nonempty_filter; mod range_filter; mod unique; pub(crate) mod utility; @@ -27,6 +28,9 @@ pub use field_match_filter::{ pub use index_filter::{ index_filter, IndexFilter, IndexFilterFactory, IndexFilterStatementFactory, }; +pub use nonempty_filter::{ + nonempty_filter, NonEmptyFilter, NonEmptyFilterFactory, NonEmptyFilterStatementFactory, +}; pub use range_filter::{ range_filter, RangeFilter, RangeFilterFactory, RangeFilterStatementFactory, }; diff --git a/mps-interpreter/src/lang/vocabulary/filters/nonempty_filter.rs b/mps-interpreter/src/lang/vocabulary/filters/nonempty_filter.rs new file mode 100644 index 0000000..b5286b3 --- /dev/null +++ b/mps-interpreter/src/lang/vocabulary/filters/nonempty_filter.rs @@ -0,0 +1,65 @@ +use std::collections::VecDeque; +use std::fmt::{Debug, Display, Error, Formatter}; + +use crate::lang::MpsLanguageDictionary; +use crate::lang::{MpsFilterFactory, MpsFilterPredicate, MpsFilterStatementFactory}; +use crate::lang::{RuntimeMsg, SyntaxError, utility::assert_token_raw}; +use crate::tokens::MpsToken; +use crate::MpsContext; +use crate::MpsItem; + +#[derive(Debug, Clone)] +pub struct NonEmptyFilter; + +impl Display for NonEmptyFilter { + fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { + write!(f, "[empty]") + } +} + +impl MpsFilterPredicate for NonEmptyFilter { + fn matches(&mut self, item: &MpsItem, _ctx: &mut MpsContext) -> Result { + if item.len() != 0 { + if item.len() == 1 && item.field("filename").is_some() { + Ok(false) // ignore filename field, since that almost always exists + } else { + Ok(true) + } + } else { + Ok(false) + } + } + + fn is_complete(&self) -> bool { + false + } + + fn reset(&mut self) -> Result<(), RuntimeMsg> { + Ok(()) + } +} + +pub struct NonEmptyFilterFactory; + +impl MpsFilterFactory for NonEmptyFilterFactory { + fn is_filter(&self, tokens: &VecDeque<&MpsToken>) -> bool { + tokens.len() >= 2 && tokens[0].is_interrogation() && tokens[1].is_interrogation() + } + + fn build_filter( + &self, + tokens: &mut VecDeque, + _dict: &MpsLanguageDictionary, + ) -> Result { + assert_token_raw(MpsToken::Interrogation, tokens)?; + assert_token_raw(MpsToken::Interrogation, tokens)?; + Ok(NonEmptyFilter) + } +} + +pub type NonEmptyFilterStatementFactory = MpsFilterStatementFactory; + +#[inline(always)] +pub fn nonempty_filter() -> NonEmptyFilterStatementFactory { + NonEmptyFilterStatementFactory::new(NonEmptyFilterFactory) +} diff --git a/mps-interpreter/src/lib.rs b/mps-interpreter/src/lib.rs index b23c84d..50c2664 100644 --- a/mps-interpreter/src/lib.rs +++ b/mps-interpreter/src/lib.rs @@ -89,6 +89,10 @@ //! //! Keep only items which are do not duplicate another item, or keep only items whoes specified field does not duplicate another item's same field. The first non-duplicated instance of an item is always the one that is kept. //! +//! ### ?? -- e.g. `iterable.(??);` +//! +//! Keep only the items that contain at least one field (not including the filename field). +//! //! ## Functions //! Similar to most other languages: `function_name(param1, param2, etc.);`. //! These always return an iterable which can be manipulated with other syntax (filters, sorters, etc.). diff --git a/mps-interpreter/src/music/tag.rs b/mps-interpreter/src/music/tag.rs index 99646b2..c3dc251 100644 --- a/mps-interpreter/src/music/tag.rs +++ b/mps-interpreter/src/music/tag.rs @@ -70,6 +70,15 @@ impl Tags { .map(|s| s.to_string()) } + #[inline] + pub fn albumartist_name(&self) -> Option { + self.data + .get("ALBUMARTIST") + .unwrap_or(&TagType::Unknown) + .str() + .map(|s| s.to_string()) + } + #[inline] pub fn genre_title(&self) -> Option { self.data diff --git a/mps-interpreter/src/processing/filesystem.rs b/mps-interpreter/src/processing/filesystem.rs index 18ff800..b63b019 100644 --- a/mps-interpreter/src/processing/filesystem.rs +++ b/mps-interpreter/src/processing/filesystem.rs @@ -163,35 +163,34 @@ impl FileIter { match crate::music::MpsLibrary::read_media_tags(path) { Ok(tags) => { let mut item = MpsItem::new(); + item.set_field("title", tags.track_title().into()); + if let Some(artist) = tags.artist_name() { + item.set_field("artist", artist.into()); + } + if let Some(albumartist) = tags.albumartist_name() { + item.set_field("albumartist", albumartist.clone().into()); + if let Some(MpsTypePrimitive::String(artist)) = item.field("artist") { + if albumartist.trim() != artist.trim() { + let new_artist = format!("{},{}", artist, albumartist.as_str()); + item.set_field("artist", new_artist.into()); + } + } else { + item.set_field("artist", albumartist.into()); + } + } + if let Some(album) = tags.album_title() { + item.set_field("album", album.into()); + } + if let Some(genre) = tags.genre_title() { + item.set_field("genre", genre.into()); + } + if let Some(track) = tags.track_number() { + item.set_field("track", track.into()); + } + if let Some(year) = tags.track_date() { + item.set_field("year", year.into()); + } self.populate_item_impl_simple(&mut item, path_str, captures, capture_names); - if item.field("title").is_none() { - item.set_field("title", tags.track_title().into()); - } - if item.field("artist").is_none() { - if let Some(artist) = tags.artist_name() { - item.set_field("artist", artist.into()); - } - } - if item.field("album").is_none() { - if let Some(album) = tags.album_title() { - item.set_field("album", album.into()); - } - } - if item.field("genre").is_none() { - if let Some(genre) = tags.genre_title() { - item.set_field("genre", genre.into()); - } - } - if item.field("track").is_none() { - if let Some(track) = tags.track_number() { - item.set_field("track", track.into()); - } - } - if item.field("year").is_none() { - if let Some(year) = tags.track_date() { - item.set_field("year", year.into()); - } - } Some(item) } Err(_) => { @@ -227,7 +226,9 @@ impl FileIter { if let Some(captures) = captures { for name_maybe in capture_names { if let Some(name) = name_maybe { - if let Some(value) = captures.name(name).map(|m| m.as_str().to_string()) { + if item.field(name).is_some() { + // do nothing + } else if let Some(value) = captures.name(name).map(|m| m.as_str().to_string()) { item.set_field(name, MpsTypePrimitive::parse(value)); } } diff --git a/mps-interpreter/src/processing/music_analysis.rs b/mps-interpreter/src/processing/music_analysis.rs index 2ba048a..278490c 100644 --- a/mps-interpreter/src/processing/music_analysis.rs +++ b/mps-interpreter/src/processing/music_analysis.rs @@ -13,10 +13,10 @@ use bliss_audio_symphonia::{BlissError, Song}; const DEFAULT_PARALLELISM: usize = 2; // maximum length of song cache (song objects take up a lot of memory) -const MAX_SONG_CACHE_SIZE: usize = 1000; +const MAX_SONG_CACHE_SIZE: usize = 10000; // maximum length of distance cache (takes up significantly less memory than songs) -const MAX_DISTANCE_CACHE_SIZE: usize = MAX_SONG_CACHE_SIZE * 10; +const MAX_DISTANCE_CACHE_SIZE: usize = MAX_SONG_CACHE_SIZE * MAX_SONG_CACHE_SIZE; use crate::lang::RuntimeMsg; use crate::MpsItem; @@ -252,7 +252,7 @@ impl CacheThread { self.distance_in_progress.remove(&key); if self.distance_cache.len() > MAX_DISTANCE_CACHE_SIZE { // avoid using too much memory - self.song_cache.clear(); + self.distance_cache.clear(); } self.distance_cache.insert(key, distance_result); } @@ -560,6 +560,9 @@ fn worker_distance( song: new_song2.clone(), }) .unwrap_or(()); + if new_song2.is_err() { + eprintln!("Song error on `{}`: {}", path2, new_song2.clone().err().unwrap()); + } new_song2? }; Ok(song1.distance(&song2)) diff --git a/mps-interpreter/tests/single_line.rs b/mps-interpreter/tests/single_line.rs index edbfbb5..40b0b42 100644 --- a/mps-interpreter/tests/single_line.rs +++ b/mps-interpreter/tests/single_line.rs @@ -1,3 +1,5 @@ +//! Integration tests for every syntax feature + use mps_interpreter::tokens::{MpsToken, MpsTokenizer, ParseError}; use mps_interpreter::*; use std::collections::VecDeque; @@ -813,3 +815,17 @@ fn execute_emptiesop_line() -> Result<(), MpsError> { )?; execute_single_line("empties(0)", true, true) } + +#[test] +fn execute_nonemptyfilter_line() -> Result<(), MpsError> { + execute_single_line( + "files().(??)", + false, + true, + )?; + execute_single_line( + "empties(42).(??)", + true, + true, + ) +} diff --git a/mps-m3u8/src/main.rs b/mps-m3u8/src/main.rs index b61a379..12dcaf8 100644 --- a/mps-m3u8/src/main.rs +++ b/mps-m3u8/src/main.rs @@ -11,7 +11,7 @@ use std::path::Path; use m3u8_rs::{MediaPlaylist, MediaSegment}; -use mps_interpreter::MpsRunner; +use mps_interpreter::{MpsFaye, MpsItem}; fn main() { let args = cli::parse(); @@ -29,16 +29,16 @@ fn main() { println!("Executing: {}", &args.input); let in_file = Cursor::new(&args.input); - let runner = MpsRunner::with_stream(in_file); + let runner = MpsFaye::with_stream(in_file); for item in runner { match item { Ok(music) => { if let Some(filename) = - music.field("filename").and_then(|x| x.to_owned().to_str()) + music_filename(&music) { playlist.segments.push(MediaSegment { uri: filename, - title: music.field("title").and_then(|x| x.to_owned().to_str()), + title: music_title(&music), ..Default::default() }); } else { @@ -52,16 +52,16 @@ fn main() { let in_path = Path::new(&args.input); let in_file = BufReader::new(File::open(in_path).expect("Invalid/missing input file")); - let runner = MpsRunner::with_stream(in_file); + let runner = MpsFaye::with_stream(in_file); for item in runner { match item { Ok(music) => { if let Some(filename) = - music.field("filename").and_then(|x| x.to_owned().to_str()) + music_filename(&music) { playlist.segments.push(MediaSegment { uri: filename, - title: music.field("title").and_then(|x| x.to_owned().to_str()), + title: music_title(&music), ..Default::default() }); } else { @@ -82,3 +82,22 @@ fn main() { eprintln!("Playlist save error: {}", e); } } + +fn music_title(item: &MpsItem) -> Option { + item.field("title").and_then(|x| x.to_owned().to_str()) +} + +fn music_filename(item: &MpsItem) -> Option { + if let Some(filename) = item.field("filename") { + if let Ok(cwd) = std::env::current_dir() { + let path: &Path = &cwd; + Some(filename.as_str().replace(path.to_str().unwrap_or(""), "./")) + } else { + Some(filename.to_string()) + } + } else { + None + } +} + + diff --git a/mps-player/src/player.rs b/mps-player/src/player.rs index d5a354c..e007ea9 100644 --- a/mps-player/src/player.rs +++ b/mps-player/src/player.rs @@ -155,11 +155,12 @@ impl<'a, T: MpsTokenReader + 'a> MpsPlayer<'a, T> { match item { Ok(music) => { if let Some(filename) = - music.field("filename").and_then(|x| x.to_owned().to_str()) + music_filename(&music) { + println!("Adding file `{}` to playlist", filename); playlist.segments.push(MediaSegment { uri: filename, - title: music.field("title").and_then(|x| x.to_owned().to_str()), + title: music_title(&music), ..Default::default() }); Ok(()) @@ -198,6 +199,25 @@ impl<'a, T: MpsTokenReader + 'a> MpsPlayer<'a, T> { } } +#[inline] +fn music_title(item: &MpsItem) -> Option { + item.field("title").and_then(|x| x.to_owned().to_str()) +} + +#[inline] +fn music_filename(item: &MpsItem) -> Option { + if let Some(filename) = item.field("filename") { + if let Ok(cwd) = std::env::current_dir() { + let path: &std::path::Path = &cwd; + Some(filename.as_str().replace(path.to_str().unwrap_or(""), "./")) + } else { + Some(filename.to_string()) + } + } else { + None + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/help.rs b/src/help.rs index 47bbf32..485b97b 100644 --- a/src/help.rs +++ b/src/help.rs @@ -85,7 +85,10 @@ Operations to reduce the items in an iterable: iterable.(filter) unique unique field -- e.g. iterable.(unique title) - Keep only items which are do not duplicate another item, or keep only items whoes specified field does not duplicate another item's same field. The first non-duplicated instance of an item is always the one that is kept."; + Keep only items which are do not duplicate another item, or keep only items whoes specified field does not duplicate another item's same field. The first non-duplicated instance of an item is always the one that is kept. + + ?? + Keep only the items that contain at least one field (not including the filename field)."; pub const SORTERS: &str = "SORTERS (?sorters)