Convert ffmpeg functionality to symphonia and fix some related tests

This commit is contained in:
NGnius (Graham) 2023-01-11 19:15:53 -05:00
parent 08aba11e39
commit 66e7333ff1
9 changed files with 457 additions and 339 deletions

227
Cargo.lock generated
View file

@ -52,6 +52,12 @@ version = "1.0.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602"
[[package]]
name = "arrayvec"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
name = "atty"
version = "0.2.14"
@ -70,6 +76,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
<<<<<<< HEAD
name = "bindgen"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@ -89,6 +96,8 @@ dependencies = [
]
[[package]]
=======
>>>>>>> 6fcb00c (Convert ffmpeg functionality to symphonia and fix some related tests)
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@ -104,7 +113,6 @@ dependencies = [
"crossbeam",
"dirs",
"env_logger",
"ffmpeg-next",
"glob",
"indicatif",
"lazy_static 1.4.0",
@ -126,7 +134,11 @@ dependencies = [
"serde_json",
"strum",
"strum_macros",
<<<<<<< HEAD
"tempdir",
=======
"symphonia",
>>>>>>> 6fcb00c (Convert ffmpeg functionality to symphonia and fix some related tests)
"thiserror",
]
@ -168,6 +180,21 @@ dependencies = [
]
[[package]]
<<<<<<< HEAD
=======
name = "byte-tools"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
[[package]]
name = "bytemuck"
version = "1.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439989e6b8c38d1b6570a384ef1e49c8848128f5a97f3914baef02920842712f"
[[package]]
>>>>>>> 6fcb00c (Convert ffmpeg functionality to symphonia and fix some related tests)
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@ -203,15 +230,6 @@ dependencies = [
"jobserver",
]
[[package]]
name = "cexpr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -229,6 +247,7 @@ dependencies = [
]
[[package]]
<<<<<<< HEAD
name = "clang-sys"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@ -240,6 +259,8 @@ dependencies = [
]
[[package]]
=======
>>>>>>> 6fcb00c (Convert ffmpeg functionality to symphonia and fix some related tests)
name = "clap"
version = "2.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@ -430,6 +451,15 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "encoding_rs"
version = "0.8.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df"
dependencies = [
"cfg-if",
]
[[package]]
name = "env_logger"
version = "0.8.4"
@ -456,6 +486,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
<<<<<<< HEAD
name = "ffmpeg-next"
version = "5.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@ -481,6 +512,8 @@ dependencies = [
]
[[package]]
=======
>>>>>>> 6fcb00c (Convert ffmpeg functionality to symphonia and fix some related tests)
name = "fftw-src"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@ -679,12 +712,6 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "libc"
version = "0.2.134"
@ -692,6 +719,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb"
[[package]]
<<<<<<< HEAD
name = "libloading"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@ -704,6 +732,10 @@ dependencies = [
[[package]]
name = "libsqlite3-sys"
version = "0.24.2"
=======
name = "log"
version = "0.4.16"
>>>>>>> 6fcb00c (Convert ffmpeg functionality to symphonia and fix some related tests)
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14"
dependencies = [
@ -837,6 +869,7 @@ dependencies = [
]
[[package]]
<<<<<<< HEAD
name = "nom"
version = "7.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@ -847,6 +880,8 @@ dependencies = [
]
[[package]]
=======
>>>>>>> 6fcb00c (Convert ffmpeg functionality to symphonia and fix some related tests)
name = "num"
version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
@ -944,21 +979,6 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "output_vt100"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66"
dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "peeking_take_while"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]]
name = "pest"
version = "2.4.0"
@ -1267,12 +1287,6 @@ dependencies = [
"smallvec",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustfft"
version = "5.1.1"
@ -1394,6 +1408,139 @@ dependencies = [
"syn",
]
[[package]]
name = "symphonia"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7e5f38aa07e792f4eebb0faa93cee088ec82c48222dd332897aae1569d9a4b7"
dependencies = [
"lazy_static 1.4.0",
"symphonia-bundle-flac",
"symphonia-bundle-mp3",
"symphonia-codec-aac",
"symphonia-codec-pcm",
"symphonia-codec-vorbis",
"symphonia-core",
"symphonia-format-ogg",
"symphonia-format-wav",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-flac"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "116e5412f5fb4e5d07efd6628d50d6fcd7a61ebef43d98f5012f3cf763b25d02"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-bundle-mp3"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec4d97c4a61ece4651751dddb393ebecb7579169d9e758ae808fe507a5250790"
dependencies = [
"bitflags",
"lazy_static 1.4.0",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-codec-aac"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd3d7ab37eb9b7df16ddedd7adb7cc382afe708ff078e525a14dc9b05e57558f"
dependencies = [
"lazy_static 1.4.0",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-pcm"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba1d54738758993546107e3a4be2c1da827f2d4489fcffee0fa47867254e44c7"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-vorbis"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a29ed6748078effb35a05064a451493a78038918981dc1a76bdf5a2752d441fa"
dependencies = [
"log",
"symphonia-core",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-core"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa135e97be0f4a666c31dfe5ef4c75435ba3d355fd6a73d2100aa79b14c104c9"
dependencies = [
"arrayvec",
"bitflags",
"bytemuck",
"lazy_static 1.4.0",
"log",
]
[[package]]
name = "symphonia-format-ogg"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7b2357288a79adfec532cfd86049696cfa5c58efeff83bd51687a528f18a519"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-format-wav"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d9fa5e5b420dea6763ba2547887eb1a02a142c676c5b02ed1b113a247101dad"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-metadata"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5260599daba18d8fe905ca3eb3b42ba210529a6276886632412cc74984e79b1a"
dependencies = [
"encoding_rs",
"lazy_static 1.4.0",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-utils-xiph"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a37026c6948ff842e0bf94b4008579cc71ab16ed0ff9ca70a331f60f4f1e1e9"
dependencies = [
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "syn"
version = "1.0.101"
@ -1548,12 +1695,6 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vec_map"
version = "0.8.2"

View file

@ -1,5 +1,5 @@
[package]
name = "bliss-audio"
name = "bliss-audio-symphonia"
version = "0.6.5"
authors = ["Polochon-street <polochonstreet@gmx.fr>"]
edition = "2018"
@ -17,8 +17,6 @@ no-default-features = true
[features]
default = ["bliss-audio-aubio-rs/static"]
# Build ffmpeg instead of using the host's.
build-ffmpeg = ["ffmpeg-next/build"]
ffmpeg-static = ["ffmpeg-next/static"]
# Use if you want to build python bindings with maturin.
python-bindings = ["bliss-audio-aubio-rs/fftw3"]
# Enable the benchmarks with `cargo +nightly bench --features=bench`
@ -41,7 +39,8 @@ lazy_static = "1.4.0"
rayon = "1.5.0"
crossbeam = "0.8.0"
noisy_float = "0.2.0"
ffmpeg-next = "5.1.1"
symphonia = { version = "0.5", features = ["mp3", "aac"]}
rubato = { version = "0.12" }
log = "0.4.14"
env_logger = "0.8.3"
thiserror = "1.0.24"

View file

@ -1,4 +1,4 @@
use bliss_audio::Song;
use bliss_audio_symphonia::Song;
use std::env;
/**

View file

@ -1,4 +1,4 @@
use bliss_audio::Song;
use bliss_audio_symphonia::Song;
use std::env;
/**

View file

@ -1,6 +1,6 @@
use anyhow::Result;
use bliss_audio::playlist::{closest_to_first_song, dedup_playlist, euclidean_distance};
use bliss_audio::{analyze_paths, Song};
use bliss_audio_symphonia::playlist::{closest_to_first_song, dedup_playlist, euclidean_distance};
use bliss_audio_symphonia::{analyze_paths, Song};
use clap::{App, Arg};
use glob::glob;
use std::env;

View file

@ -201,7 +201,7 @@ mod tests {
let error = songs[0].to_owned().unwrap_err();
assert_eq!(
error,
BlissError::DecodingError("empty audio file associated to CUE sheet".to_string())
BlissError::DecodingError("while opening format: DecodeError(\"wav: chunk length exceeds parent (list) chunk length\").".to_string())
);
}
@ -330,8 +330,7 @@ mod tests {
..Default::default()
}),
Err(BlissError::DecodingError(String::from(
"while opening format for file 'data/not-existing.wav': \
ffmpeg::Error(2: No such file or directory).",
"while opening song: Os { code: 2, kind: NotFound, message: \"No such file or directory\" }.",
))),
];
assert_eq!(expected, songs);

View file

@ -91,7 +91,7 @@ use thiserror::Error;
pub use song::{Analysis, AnalysisIndex, Song, NUMBER_FEATURES};
const CHANNELS: u16 = 1;
//const CHANNELS: u16 = 1;
const SAMPLE_RATE: u32 = 22050;
/// Stores the current version of bliss-rs' features.
/// It is bumped every time one or more feature is added, updated or removed,
@ -291,27 +291,21 @@ mod tests {
false,
PathBuf::from("./data/testcue.cue"),
Some(String::from(
"error happened while decoding file while \
opening format for file './data/not-existing.wav': \
ffmpeg::Error(2: No such file or directory).",
"error happened while decoding file while opening song: Os { code: 2, kind: NotFound, message: \"No such file or directory\" }.",
)),
),
(
false,
PathBuf::from("definitely-not-existing.foo"),
Some(String::from(
"error happened while decoding file while \
opening format for file 'definitely-not-existing\
.foo': ffmpeg::Error(2: No such file or directory).",
"error happened while decoding file while opening song: Os { code: 2, kind: NotFound, message: \"No such file or directory\" }.",
)),
),
(
false,
PathBuf::from("not-existing.foo"),
Some(String::from(
"error happened while decoding file \
while opening format for file 'not-existing.foo': \
ffmpeg::Error(2: No such file or directory).",
"error happened while decoding file while opening song: Os { code: 2, kind: NotFound, message: \"No such file or directory\" }.",
)),
),
(true, PathBuf::from("./data/s16_mono_22_5kHz.flac"), None),

View file

@ -8,7 +8,8 @@
//! a look at Library is instead recommended.
extern crate crossbeam;
extern crate ffmpeg_next as ffmpeg;
//extern crate ffmpeg_next as ffmpeg;
extern crate symphonia;
extern crate ndarray;
extern crate ndarray_npy;
@ -21,19 +22,18 @@ use crate::playlist::{closest_to_first_song, dedup_playlist, euclidean_distance,
use crate::temporal::BPMDesc;
use crate::timbral::{SpectralDesc, ZeroCrossingRateDesc};
use crate::{BlissError, BlissResult, SAMPLE_RATE};
use crate::{CHANNELS, FEATURES_VERSION};
use crate::{FEATURES_VERSION};
use ::log::warn;
use core::ops::Index;
use crossbeam::thread;
use ffmpeg_next::codec::threading::{Config, Type as ThreadingType};
use ffmpeg_next::util::channel_layout::ChannelLayout;
use ffmpeg_next::util::error::Error;
use ffmpeg_next::util::error::EINVAL;
use ffmpeg_next::util::format::sample::{Sample, Type};
use ffmpeg_next::util::frame::audio::Audio;
use ffmpeg_next::util::log;
use ffmpeg_next::util::log::level::Level;
use ffmpeg_next::{media, util};
use symphonia::core::formats::{SeekMode, SeekTo};
use symphonia::core::io::MediaSourceStream;
use symphonia::core::probe::Hint;
use symphonia::core::errors::Error as SymphoniaError;
use symphonia::core::audio::{AudioBuffer, AudioBufferRef, Signal};
use symphonia::core::meta::{MetadataRevision, StandardTagKey};
use rubato::Resampler;
use rubato::FftFixedIn as ResamplerImpl;
use ndarray::{arr1, Array1};
use std::convert::TryInto;
use std::fmt;
@ -133,7 +133,7 @@ pub const NUMBER_FEATURES: usize = AnalysisIndex::COUNT;
/// Only use it if you want to have an in-depth look of what is
/// happening behind the scene, or make a distance metric yourself.
///
/// Under the hood, it is just an array of f32 holding different numeric
/// Under the hood, it is just an array of f32 holding different nsrcumeric
/// features.
///
/// For more info on the different features, build the
@ -422,205 +422,123 @@ impl Song {
}
pub(crate) fn decode(path: &Path) -> BlissResult<InternalSong> {
ffmpeg::init().map_err(|e| {
BlissError::DecodingError(format!(
"ffmpeg init error while decoding file '{}': {:?}.",
path.display(),
e
))
})?;
log::set_level(Level::Quiet);
let registry = symphonia::default::get_codecs();
let probe = symphonia::default::get_probe();
let mut song = InternalSong {
path: path.into(),
..Default::default()
};
let mut ictx = ffmpeg::format::input(&path).map_err(|e| {
BlissError::DecodingError(format!(
"while opening format for file '{}': {:?}.",
path.display(),
e
))
})?;
let (mut decoder, stream, expected_sample_number) = {
let input = ictx.streams().best(media::Type::Audio).ok_or_else(|| {
BlissError::DecodingError(format!(
"No audio stream found for file '{}'.",
path.display()
))
})?;
let mut context = ffmpeg::codec::context::Context::from_parameters(input.parameters())
.map_err(|e| {
BlissError::DecodingError(format!(
"Could not load the codec context for file '{}': {:?}",
path.display(),
e
))
})?;
context.set_threading(Config {
let song_file = std::fs::File::open(path).map_err(|e| BlissError::DecodingError(format!("while opening song: {:?}.", e)))?;
let stream = MediaSourceStream::new(Box::new(song_file), Default::default());
let mut hint = Hint::new();
if let Some(ext) = path.extension().and_then(|x| x.to_str()) {
hint.with_extension(ext);
}
let mut probed = probe.format(&hint, stream, &Default::default(), &Default::default())
.map_err(|e| BlissError::DecodingError(format!("while opening format: {:?}.", e)))?;
let format = &mut probed.format;
/*let mut format = ffmpeg::format::input(&path)
.map_err(|e| BlissError::DecodingError(format!("while opening format: {:?}.", e)))?;*/
let (mut codec, stream_id, expected_sample_number) = {
let track = format
.tracks()
.into_iter()
.find(|t| t.codec_params.codec != symphonia::core::codecs::CODEC_TYPE_NULL)
.ok_or_else(|| BlissError::DecodingError(String::from("No valid audio stream found.")))?;
let track_id = track.id;
/*let stream = format
.streams()
.find(|s| s.codec().medium() == ffmpeg::media::Type::Audio)
.ok_or_else(|| BlissError::DecodingError(String::from("No audio stream found.")))?;*/
/*stream.codec().set_threading(Config {
kind: ThreadingType::Frame,
count: 0,
safe: true,
});
let decoder = context.decoder().audio().map_err(|e| {
BlissError::DecodingError(format!(
"when finding decoder for file '{}': {:?}.",
path.display(),
e
))
})?;
// Add SAMPLE_RATE to have one second margin to avoid reallocating if
// the duration is slightly more than estimated
// TODO>1.0 another way to get the exact number of samples is to decode
// everything once, compute the real number of samples from that,
// allocate the array with that number, and decode again. Check
// what's faster between reallocating, and just have one second
// leeway.
let expected_sample_number = (SAMPLE_RATE as f32 * input.duration() as f32
/ input.time_base().denominator() as f32)
.ceil()
+ SAMPLE_RATE as f32;
(decoder, input.index(), expected_sample_number)
};
let sample_array: Vec<f32> = Vec::with_capacity(expected_sample_number as usize);
if let Some(title) = ictx.metadata().get("title") {
song.title = match title {
"" => None,
t => Some(t.to_string()),
};
};
if let Some(artist) = ictx.metadata().get("artist") {
song.artist = match artist {
"" => None,
a => Some(a.to_string()),
};
};
if let Some(album) = ictx.metadata().get("album") {
song.album = match album {
"" => None,
a => Some(a.to_string()),
};
};
if let Some(genre) = ictx.metadata().get("genre") {
song.genre = match genre {
"" => None,
g => Some(g.to_string()),
};
};
if let Some(track_number) = ictx.metadata().get("track") {
song.track_number = match track_number {
"" => None,
t => Some(t.to_string()),
};
};
if let Some(album_artist) = ictx.metadata().get("album_artist") {
song.album_artist = match album_artist {
"" => None,
t => Some(t.to_string()),
};
};
let in_channel_layout = {
if decoder.channel_layout() == ChannelLayout::empty() {
ChannelLayout::default(decoder.channels().into())
} else {
decoder.channel_layout()
}
};
decoder.set_channel_layout(in_channel_layout);
let (tx, rx) = mpsc::channel();
let in_codec_format = decoder.format();
let in_codec_rate = decoder.rate();
let child = std_thread::spawn(move || {
resample_frame(
rx,
in_codec_format,
in_channel_layout,
in_codec_rate,
sample_array,
)
});
for (s, packet) in ictx.packets() {
if s.index() != stream {
continue;
}
match decoder.send_packet(&packet) {
Ok(_) => (),
Err(Error::Other { errno: EINVAL }) => {
return Err(BlissError::DecodingError(format!(
"wrong codec opened for file '{}.",
path.display(),
)))
}
Err(Error::Eof) => {
warn!(
"Premature EOF reached while decoding file '{}'.",
path.display()
);
drop(tx);
song.sample_array = child.join().unwrap()?;
return Ok(song);
}
Err(e) => warn!("error while decoding file '{}': {}", path.display(), e),
};
loop {
let mut decoded = ffmpeg::frame::Audio::empty();
match decoder.receive_frame(&mut decoded) {
Ok(_) => {
tx.send(decoded).map_err(|e| {
BlissError::DecodingError(format!(
"while sending decoded frame to the resampling thread for file '{}': {:?}",
path.display(),
e,
))
})?;
});*/
let mut decoder = registry
.make(&track.codec_params, &Default::default())
.map_err(|e| BlissError::DecodingError(format!("when finding codec: {:?}.", e)))?;
/*let codec =
stream.codec().decoder().audio().map_err(|e| {
BlissError::DecodingError(format!("when finding codec: {:?}.", e))
})?;*/
let mut expected_sample_number: usize = 0;
// decode once to find sample number
// previously, ffmpeg let us (roughly) calculate this
// symphonia does not make that easy so we just decode twice instead
while let Ok(packet) = format.next_packet() {
if packet.track_id() == track_id {
if let Ok(buffer) = decoder.decode(&packet) {
// errors will only be handled when actually decoding samples
expected_sample_number += sample_buffer_length(buffer);
}
Err(_) => break,
}
} // else ignore packet
}
// reset decoding to start of audio
format.seek(SeekMode::Accurate, SeekTo::Time {
time: symphonia::core::units::Time {
seconds: 0,
frac: 0.0,
},
track_id: Some(track_id),
}).map_err(|e| BlissError::DecodingError(format!("while seeking to start: {}", e)))?;
decoder.reset();
(decoder, track_id, expected_sample_number)
};
let sample_array: Vec<f32> = Vec::with_capacity(expected_sample_number);
// populate song metadata from file's info tags
if let Some(revision) = format.metadata().current() {
// audio format's built-in tags
song.read_tags(revision);
}
if let Some(metadata) = probed.metadata.get() {
if let Some(revision) = metadata.current() {
// audio wrapper's tags
song.read_tags(revision);
}
}
// Flush the stream
let packet = ffmpeg::codec::packet::Packet::empty();
match decoder.send_packet(&packet) {
Ok(_) => (),
Err(Error::Other { errno: EINVAL }) => {
return Err(BlissError::DecodingError(format!(
"wrong codec opened for file '{}'.",
path.display()
)))
}
Err(Error::Eof) => {
warn!(
"Premature EOF reached while decoding file '{}'.",
path.display()
);
drop(tx);
song.sample_array = child.join().unwrap()?;
return Ok(song);
}
Err(e) => warn!("error while decoding {}: {}", path.display(), e),
};
let (tx, rx) = mpsc::channel();
let child = std_thread::spawn(move || {
resample_buffer(
rx,
sample_array,
)
});
loop {
let mut decoded = ffmpeg::frame::Audio::empty();
match decoder.receive_frame(&mut decoded) {
Ok(_) => {
tx.send(decoded).map_err(|e| {
let packet = match format.next_packet() {
Err(SymphoniaError::IoError(_)) => break,
Ok(packet) => packet,
Err(e) => return Err(BlissError::DecodingError(format!("while reading packet: {}", e)))
};
if packet.track_id() != stream_id {
continue;
}
match codec.decode(&packet) {
Ok(decoded) => {
tx.send(convert_sample_buffer(decoded)).map_err(|e| {
BlissError::DecodingError(format!(
"while sending decoded frame to the resampling thread for file '{}': {:?}",
path.display(),
e
))
})?;
},
Err(SymphoniaError::Unsupported(s)) => {
return Err(BlissError::DecodingError(format!("unsupported: {}", s)))
}
Err(_) => break,
}
Err(SymphoniaError::IoError(e)) => {
warn!("IO error occured while decoding: {}", e);
drop(tx);
song.sample_array = child.join().unwrap()?;
return Ok(song);
},
Err(e) => warn!("error while decoding {}: {}", path.display(), e),
};
}
drop(tx);
song.sample_array = child.join().unwrap()?;
song.sample_array = child.join().map_err(|_| BlissError::DecodingError(format!("resampler thread panic!")))??;
let duration_seconds = song.sample_array.len() as f32 / SAMPLE_RATE as f32;
song.duration = Duration::from_nanos((duration_seconds * 1e9_f32).round() as u64);
Ok(song)
@ -640,93 +558,163 @@ pub(crate) struct InternalSong {
pub sample_array: Vec<f32>,
}
fn resample_frame(
rx: Receiver<Audio>,
in_codec_format: Sample,
in_channel_layout: ChannelLayout,
in_rate: u32,
mut sample_array: Vec<f32>,
) -> BlissResult<Vec<f32>> {
let mut resample_context = ffmpeg::software::resampling::context::Context::get(
in_codec_format,
in_channel_layout,
in_rate,
Sample::F32(Type::Packed),
ffmpeg::util::channel_layout::ChannelLayout::MONO,
SAMPLE_RATE,
)
.map_err(|e| {
BlissError::DecodingError(format!(
"while trying to allocate resampling context: {:?}",
e
))
})?;
let mut resampled = ffmpeg::frame::Audio::empty();
let mut something_happened = false;
for decoded in rx.iter() {
if in_codec_format != decoded.format()
|| (in_channel_layout != decoded.channel_layout())
// If the decoded layout is empty, it means we forced the
// "in_channel_layout" to something default, not that
// the format is wrong.
&& (decoded.channel_layout() != ChannelLayout::empty())
|| in_rate != decoded.rate()
{
warn!("received decoded packet with wrong format; file might be corrupted.");
continue;
}
something_happened = true;
resampled = ffmpeg::frame::Audio::empty();
resample_context
.run(&decoded, &mut resampled)
.map_err(|e| {
BlissError::DecodingError(format!("while trying to resample song: {:?}", e))
})?;
push_to_sample_array(&resampled, &mut sample_array);
}
if !something_happened {
return Ok(sample_array);
}
// TODO when ffmpeg-next will be active again: shouldn't we allocate
// `resampled` again?
loop {
match resample_context.flush(&mut resampled).map_err(|e| {
BlissError::DecodingError(format!("while trying to resample song: {:?}", e))
})? {
Some(_) => {
push_to_sample_array(&resampled, &mut sample_array);
}
None => {
if resampled.samples() == 0 {
break;
impl InternalSong {
#[inline]
fn read_tags(&mut self, metadata: &MetadataRevision) {
for tag in metadata.tags() {
if let Some(key) = tag.std_key {
match key {
StandardTagKey::Album => self.album = Some(tag.value.to_string()),
StandardTagKey::AlbumArtist => self.album_artist = Some(tag.value.to_string()),
StandardTagKey::TrackTitle => self.title = Some(tag.value.to_string()),
StandardTagKey::Artist => self.artist = Some(tag.value.to_string()),
StandardTagKey::Genre => self.genre = Some(tag.value.to_string()),
StandardTagKey::TrackNumber => self.track_number = Some(tag.value.to_string()),
_ => {},
}
push_to_sample_array(&resampled, &mut sample_array);
}
};
}
}
Ok(sample_array)
}
fn push_to_sample_array(frame: &ffmpeg::frame::Audio, sample_array: &mut Vec<f32>) {
if frame.samples() == 0 {
return;
fn convert_sample_buffer(buffer_in: AudioBufferRef) -> AudioBuffer<f32> {
match buffer_in {
AudioBufferRef::F32(buf) => buf.into_owned(),
AudioBufferRef::F64(buf) => {
let mut buffer_out = buf.make_equivalent::<f32>();
// symphonia expects frames to already exist (but since we know we're going to replace them right away, rendering doesn't have to do anything)
buffer_out.render(Some(buf.frames()), |_, _| Ok(())).unwrap_or(());
buf.convert(&mut buffer_out);
buffer_out
},
AudioBufferRef::S16(buf) => {
let mut buffer_out = buf.make_equivalent::<f32>();
// symphonia expects frames to already exist
buffer_out.render(Some(buf.frames()), |_, _| Ok(())).unwrap_or(());
buf.convert(&mut buffer_out);
buffer_out
},
AudioBufferRef::S24(buf) => {
let mut buffer_out = buf.make_equivalent::<f32>();
// symphonia expects frames to already exist
buffer_out.render(Some(buf.frames()), |_, _| Ok(())).unwrap_or(());
buf.convert(&mut buffer_out);
buffer_out
},
AudioBufferRef::S32(buf) => {
let mut buffer_out = buf.make_equivalent::<f32>();
// symphonia expects frames to already exist
buffer_out.render(Some(buf.frames()), |_, _| Ok(())).unwrap_or(());
buf.convert(&mut buffer_out);
buffer_out
},
AudioBufferRef::S8(buf) => {
let mut buffer_out = buf.make_equivalent::<f32>();
// symphonia expects frames to already exist
buffer_out.render(Some(buf.frames()), |_, _| Ok(())).unwrap_or(());
buf.convert(&mut buffer_out);
buffer_out
},
AudioBufferRef::U16(buf) => {
let mut buffer_out = buf.make_equivalent::<f32>();
// symphonia expects frames to already exist
buffer_out.render(Some(buf.frames()), |_, _| Ok(())).unwrap_or(());
buf.convert(&mut buffer_out);
buffer_out
},
AudioBufferRef::U24(buf) => {
let mut buffer_out = buf.make_equivalent::<f32>();
// symphonia expects frames to already exist
buffer_out.render(Some(buf.frames()), |_, _| Ok(())).unwrap_or(());
buf.convert(&mut buffer_out);
buffer_out
},
AudioBufferRef::U32(buf) => {
let mut buffer_out = buf.make_equivalent::<f32>();
// symphonia expects frames to already exist
buffer_out.render(Some(buf.frames()), |_, _| Ok(())).unwrap_or(());
buf.convert(&mut buffer_out);
buffer_out
},
AudioBufferRef::U8(buf) => {
let mut buffer_out = buf.make_equivalent::<f32>();
// symphonia expects frames to already exist
buffer_out.render(Some(buf.frames()), |_, _| Ok(())).unwrap_or(());
buf.convert(&mut buffer_out);
buffer_out
},
}
// Account for the padding
let actual_size = util::format::sample::Buffer::size(
Sample::F32(Type::Packed),
CHANNELS,
frame.samples(),
false,
);
let f32_frame: Vec<f32> = frame.data(0)[..actual_size]
.chunks_exact(4)
.map(|x| {
let mut a: [u8; 4] = [0; 4];
a.copy_from_slice(x);
f32::from_le_bytes(a)
})
.collect();
sample_array.extend_from_slice(&f32_frame);
}
fn sample_buffer_length(buffer_in: AudioBufferRef) -> usize {
match buffer_in {
AudioBufferRef::F32(buf) => buf.frames(),
AudioBufferRef::F64(buf) => buf.frames(),
AudioBufferRef::S16(buf) => buf.frames(),
AudioBufferRef::S24(buf) => buf.frames(),
AudioBufferRef::S32(buf) => buf.frames(),
AudioBufferRef::S8(buf) => buf.frames(),
AudioBufferRef::U16(buf) => buf.frames(),
AudioBufferRef::U24(buf) => buf.frames(),
AudioBufferRef::U32(buf) => buf.frames(),
AudioBufferRef::U8(buf) => buf.frames(),
}
}
fn resample_buffer(
rx: Receiver<AudioBuffer<f32>>,
mut sample_array: Vec<f32>,
) -> BlissResult<Vec<f32>> {
let mut resample_buffer = Vec::<f32>::new();
let mut resampler_cache = std::collections::HashMap::<(u32, usize), ResamplerImpl<f32>>::new();
let mut wave_out_buffer = [Vec::<f32>::new()];
for decoded in rx.iter() {
//dbg!(decoded.frames());
if decoded.planes().planes().is_empty() || decoded.frames() < 5 {
// buffers that are too small cause resampler to panic
// due to chunk rounding down to 0
continue;
}
resample_buffer.clear();
// do resampling ourselves since symphonia doesn't
let frame_count = decoded.frames();
let in_sample_rate = decoded.spec().rate;
let audio_planes = decoded.planes();
let planes = audio_planes.planes();
let planes_len = planes.len() as f32;
for i in 0..frame_count {
// average samples into Mono track
let mut avg_sample = 0.0;
for plane in planes {
avg_sample += plane[i] / planes_len;
}
resample_buffer.push(avg_sample);
}
// build resampler
let cache_key = (in_sample_rate, resample_buffer.len());
let resampler = if let Some(resampler) = resampler_cache.get_mut(&cache_key) {
resampler
} else {
let new_resampler = ResamplerImpl::new(
in_sample_rate as _,
SAMPLE_RATE as _,
resample_buffer.len(),
8,
1
).map_err(|e| BlissError::DecodingError(format!("Resampler init failure: {}", e)))?;
resampler_cache.insert(cache_key, new_resampler);
resampler_cache.get_mut(&cache_key).unwrap()
};
// resample
resampler.process_into_buffer(
&[resample_buffer.as_slice()],
&mut wave_out_buffer,
None,
).map_err(|e| BlissError::DecodingError(format!("Resampler processing error: {}", e)))?;
sample_array.append(&mut wave_out_buffer[0]);
}
Ok(sample_array)
}
#[cfg(test)]
@ -929,14 +917,12 @@ mod tests {
assert_eq!(
Song::decode(Path::new("nonexistent")).unwrap_err(),
BlissError::DecodingError(String::from(
"while opening format for file 'nonexistent': ffmpeg::Error(2: No such file or directory)."
"while opening song: Os { code: 2, kind: NotFound, message: \"No such file or directory\" }."
)),
);
assert_eq!(
Song::decode(Path::new("data/picture.png")).unwrap_err(),
BlissError::DecodingError(String::from(
"No audio stream found for file 'data/picture.png'."
)),
BlissError::DecodingError(String::from("while opening format: Unsupported(\"core (probe): no suitable format reader found\").")),
);
}

View file

@ -3,7 +3,6 @@ use ndarray::{arr1, s, Array, Array1, Array2};
use rustfft::num_complex::Complex;
use rustfft::num_traits::Zero;
use rustfft::FftPlanner;
extern crate ffmpeg_next as ffmpeg;
use log::warn;
use std::f32::consts::PI;