diff --git a/src/cli.rs b/src/cli.rs index f6e0ab2..ad8d2b9 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,15 +1,11 @@ use clap::{Parser, Subcommand, Args}; +//use std::io::Write as _; +use std::fmt::Write as _; /// An alternative plugin store #[derive(Parser, Debug, Clone)] #[command(author, version, about, long_about = None, propagate_version = true)] pub struct CliArgs { - /// Proxy offerings from another store - #[arg(name = "store", long)] - pub proxy_store: Option, - /// Proxy main store offerings - #[arg(name = "proxy", short, long)] - pub proxy: bool, /// Cache results for a period #[arg(name = "cache", long)] pub cache_duration: Option, @@ -30,6 +26,41 @@ pub enum StorageArgs { Default, /// Use the filesystem Filesystem(FilesystemArgs), + /// Use an existing online store + Proxy(ProxyArgs), + /// Use no storage system + Empty, + /// Combine multiple storages together + Merge(MergeArgs) +} + +impl StorageArgs { + // A cursed syntax with super simple parsing for describing storage settings + pub fn from_descriptor(chars: &mut std::str::Chars) -> Result { + //let mut chars = descriptor.chars(); + if let Some(char0) = chars.next() { + Ok(match char0 { + 'd' | '_' => Self::Default, + 'f' => Self::Filesystem(FilesystemArgs::from_descriptor(chars)?), + 'p' => Self::Proxy(ProxyArgs::from_descriptor(chars)?), + 'e' | ' ' => Self::Empty, + 'm' | '+' => Self::Merge(MergeArgs::from_descriptor(chars)?), + c => return Err(format!("Unexpected char {}, expected a descriptor prefix from {{d f p e m}}", c)), + }) + } else { + Err(format!("Empty storage descriptor")) + } + } + + pub fn to_descriptor(self) -> String { + match self { + Self::Default => "d".to_owned(), + Self::Filesystem(fs) => format!("f{}", fs.to_descriptor()), + Self::Proxy(px) => format!("p{}", px.to_descriptor()), + Self::Empty => "e".to_owned(), + Self::Merge(ls) => format!("m{}", ls.to_descriptor()), + } + } } #[derive(Args, Debug, Clone)] @@ -41,3 +72,203 @@ pub struct FilesystemArgs { #[arg(name = "stats", long)] pub enable_stats: bool, } + +impl FilesystemArgs { + fn from_descriptor(chars: &mut std::str::Chars) -> Result { + if let Some(char1) = chars.next() { + if char1 != '{' { + return Err(format!("Expected {{, got {}", char1)); + } + } else { + return Err(format!("Filesystem descriptor too short")); + } + let mut root = None; + let mut domain = None; + let mut stats = false; + let mut buffer = Vec::::new(); + let mut for_variable: Option = None; + let mut in_string = false; + for c in chars { + match c { + '}' => return + Ok(Self { + root: root.unwrap_or_else(|| "./store".into()), + domain_root: domain.unwrap_or_else(|| "http://localhost:22252".into()), + enable_stats: stats, + }), + '\'' => in_string = !in_string, + '=' => if !in_string { + let value: String = buffer.drain(..).collect(); + if for_variable.is_some() { + return Err("Unexpected = in filesystem descriptor".to_owned()); + } else { + for_variable = Some(value); + } + }, + ',' => if !in_string { + let value: String = buffer.iter().collect(); + if let Some(var) = for_variable.take() { + let var_trimmed = var.trim(); + match &var_trimmed as &str { + "r" | "root" => root = Some(value), + "d" | "domain" => domain = Some(value), + "s" | "stats" => stats = value == "1" || value == "y", + v => return Err(format!("Unexpected variable name {} in filesystem descriptor", v)), + } + } else { + return Err("Unexpected , in filesystem descriptor".to_owned()) + } + } + c => buffer.push(c), + } + } + Err("Unexpected end of descriptor".to_owned()) + } + + fn to_descriptor(self) -> String { + format!("{{root='{}',domain='{}',stats:{}}}", self.root, self.domain_root, self.enable_stats as u8) + } +} + +#[derive(Args, Debug, Clone)] +pub struct ProxyArgs { + /// Proxy offerings from another store + #[arg(name = "store", long, default_value_t = {"https://plugins.deckbrew.xyz".into()})] + pub proxy_store: String, +} + +impl ProxyArgs { + fn from_descriptor(chars: &mut std::str::Chars) -> Result { + if let Some(char1) = chars.next() { + if char1 != '{' { + return Err(format!("Expected {{, got {}", char1)); + } + } else { + return Err(format!("Proxy descriptor too short")); + } + let mut buffer = Vec::new(); + for c in chars { + match c { + '}' => return + Ok(Self { + proxy_store: if buffer.is_empty() { "https://plugins.deckbrew.xyz".into() } else { buffer.iter().collect() } + }), + c => buffer.push(c), + } + } + Err("Unexpected end of descriptor".to_owned()) + } + + fn to_descriptor(self) -> String { + format!("{{{}}}", self.proxy_store) + } +} + +#[derive(Args, Debug, Clone)] +pub struct MergeArgs { + /// Settings descriptor + pub settings: Vec, +} + +impl MergeArgs { + fn from_descriptor(chars: &mut std::str::Chars) -> Result { + if let Some(char1) = chars.next() { + if char1 != '[' { + return Err(format!("Expected [, got {}", char1)); + } + } else { + return Err(format!("Merge descriptor too short")); + } + let mut others = Vec::new(); + loop { + if let Some(char_n) = chars.next() { + if char_n != '(' { + return Err(format!("Expected (, got {}", char_n)); + } + } + others.push(StorageArgs::from_descriptor(chars)?.to_descriptor()); + if let Some(char_n) = chars.next() { + if char_n != ')' { + return Err(format!("Expected ), got {}", char_n)); + } + } + if let Some(c) = chars.next() { + match c { + ',' => {}, + ']' => { + if others.len() < 2 { + return Err("Merge args too short (0-1 descriptors is useless!)".to_owned()); + } + return Ok(Self { + settings: others, + }); + }, + c => return Err(format!("Unexpected char {}, expected ] or ,", c)) + } + } else { + break; + } + + } + Err("Unexpected end of descriptor".to_owned()) + } + + fn to_descriptor(self) -> String { + let mut out = "[".to_owned(); + for descriptor in self.settings { + write!(&mut out, "({})", descriptor).unwrap(); + } + write!(&mut out, "]").unwrap(); + out + } + + pub fn generate_args(&self) -> Result, String> { + let mut results = Vec::with_capacity(self.settings.len()); + for args in &self.settings { + results.push(StorageArgs::from_descriptor(&mut args.chars())?); + } + Ok(results) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn storage_descriptor() { + let descriptor = "f{root='',domain='',stats:0}"; + let parsed = StorageArgs::from_descriptor(&mut descriptor.chars()); + parsed.expect("StorageArgs parse error"); + let descriptor = "p{}"; + let parsed = StorageArgs::from_descriptor(&mut descriptor.chars()); + parsed.expect("StorageArgs parse error"); + let descriptor = "m[(p{}),(d)]"; + let parsed = StorageArgs::from_descriptor(&mut descriptor.chars()); + parsed.expect("StorageArgs parse error"); + let descriptor = "d"; + let parsed = StorageArgs::from_descriptor(&mut descriptor.chars()); + parsed.expect("StorageArgs parse error"); + } + + #[test] + fn filesys_descriptor() { + let descriptor = "{root='',domain='',stats:0}"; + let parsed = FilesystemArgs::from_descriptor(&mut descriptor.chars()); + parsed.expect("FilesystemArgs parse error"); + } + + #[test] + fn proxy_descriptor() { + let descriptor = "{}"; + let parsed = ProxyArgs::from_descriptor(&mut descriptor.chars()); + parsed.expect("ProxyArgs parse error"); + } + + #[test] + fn merge_descriptor() { + let descriptor = "[(f{}),(p{}),( )]"; + let parsed = MergeArgs::from_descriptor(&mut descriptor.chars()); + parsed.expect("MergeArgs parse error"); + } +} diff --git a/src/main.rs b/src/main.rs index 08d1a31..e9c4abc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,8 +3,6 @@ mod consts; mod not_decky; mod storage; -use crate::storage::IStorageWrap; - use actix_web::{get, web, App, HttpResponse, HttpServer, Responder}; use simplelog::{LevelFilter, WriteLogger}; @@ -13,6 +11,32 @@ async fn hello() -> impl Responder { HttpResponse::Ok().body(format!("{} v{}", consts::PACKAGE_NAME, consts::PACKAGE_VERSION)) } +fn build_storage_box(storage: &cli::StorageArgs) -> Box { + match storage { + cli::StorageArgs::Default => Box::new(storage::FileStorage::new( + "./store".into(), + "http://192.168.0.128:22252".into(), + true, + )), + cli::StorageArgs::Filesystem(fs) => Box::new(storage::FileStorage::new( + fs.root.clone().into(), + fs.domain_root.clone().into(), + fs.enable_stats, + )), + cli::StorageArgs::Proxy(px) => Box::new(storage::ProxiedStorage::new( + px.proxy_store.clone(), + )), + cli::StorageArgs::Empty => Box::new(storage::EmptyStorage), + cli::StorageArgs::Merge(ls) => Box::new(storage::MergedStorage::new( + ls.generate_args() + .expect("Bad descriptor") + .drain(..) + .map(|args| build_storage_box(&args)) + .collect() + )) + } +} + #[actix_web::main] async fn main() -> std::io::Result<()> { let args = cli::CliArgs::get(); @@ -35,17 +59,12 @@ async fn main() -> std::io::Result<()> { .allow_any_header() .expose_any_header(); - let storage_data: Box = match &args.storage { - cli::StorageArgs::Default => storage::FileStorage::new( - "./store".into(), - "http://192.168.0.128:22252".into(), - true, - ).wrap(args.clone()), - cli::StorageArgs::Filesystem(fs) => storage::FileStorage::new( - fs.root.clone().into(), - fs.domain_root.clone().into(), - fs.enable_stats, - ).wrap(args.clone()), + let storage_data: Box = build_storage_box(&args.storage); + + let storage_data = if let Some(cache_duration) = args.cache_duration { + Box::new(storage::CachedStorage::new(cache_duration, storage_data)) + } else { + storage_data }; App::new() diff --git a/src/not_decky/artifact.rs b/src/not_decky/artifact.rs index 479345b..08c76f8 100644 --- a/src/not_decky/artifact.rs +++ b/src/not_decky/artifact.rs @@ -4,6 +4,7 @@ use crate::storage::IStorage; #[get("/plugins/{name}/{version}/{hash}.zip")] pub async fn decky_artifact(data: web::Data>, path: web::Path<(String, String, String)>) -> actix_web::Result { - let zip = data.get_artifact(&path.0, &path.1, &path.2).map_err(|e| actix_web::error::ErrorNotFound(e.to_string()))?; + let zip = web::block(move || data.get_artifact(&path.0, &path.1, &path.2)).await + .map_err(|e| actix_web::error::ErrorNotFound(e.to_string()))?; Ok(zip) } diff --git a/src/not_decky/image.rs b/src/not_decky/image.rs index b4c945e..11d1520 100644 --- a/src/not_decky/image.rs +++ b/src/not_decky/image.rs @@ -4,6 +4,7 @@ use crate::storage::IStorage; #[get("/plugins/{name}.png")] pub async fn decky_image(data: web::Data>, path: web::Path) -> actix_web::Result { - let zip = data.get_image(&path).map_err(|e| actix_web::error::ErrorNotFound(e.to_string()))?; + let zip = web::block(move || data.get_image(&path)).await + .map_err(|e| actix_web::error::ErrorNotFound(e.to_string()))?; Ok(zip) } diff --git a/src/not_decky/plugins.rs b/src/not_decky/plugins.rs index 3ad1e95..b46f705 100644 --- a/src/not_decky/plugins.rs +++ b/src/not_decky/plugins.rs @@ -6,6 +6,6 @@ use crate::storage::IStorage; #[get("/plugins")] pub async fn decky_plugins(data: actix_web::web::Data>) -> impl Responder { - let plugins: StorePluginList = data.plugins(); + let plugins: StorePluginList = web::block(move || data.plugins()).await.unwrap(); web::Json(plugins) } diff --git a/src/storage/cache.rs b/src/storage/cache.rs index 9cb8448..ea511b1 100644 --- a/src/storage/cache.rs +++ b/src/storage/cache.rs @@ -47,7 +47,7 @@ impl Cached { } } -pub struct CachedStorage { +pub struct CachedStorage + Send + Sync> { fallback: S, plugins_cache: Cached, statistics_cache: Cached>, @@ -55,11 +55,11 @@ pub struct CachedStorage { images_cache: Cached>, } -impl CachedStorage { +impl + Send + Sync> CachedStorage { pub fn new(duration: i64, inner: S) -> Self { Self { - plugins_cache: Cached::new(inner.plugins(), duration), - statistics_cache: Cached::new(inner.get_statistics(), duration), + plugins_cache: Cached::new(inner.as_ref().plugins(), duration), + statistics_cache: Cached::new(inner.as_ref().get_statistics(), duration), artifacts_cache: Cached::new(HashMap::new(), duration), images_cache: Cached::new(HashMap::new(), duration), fallback: inner, @@ -67,9 +67,9 @@ impl CachedStorage { } } -impl IStorage for CachedStorage { +impl + Send + Sync> IStorage for CachedStorage { fn plugins(&self) -> StorePluginList { - self.plugins_cache.get(|| self.fallback.plugins()) + self.plugins_cache.get(|| self.fallback.as_ref().plugins()) } fn get_artifact(&self, name: &str, version: &str, hash: &str) -> Result { @@ -77,7 +77,7 @@ impl IStorage for CachedStorage { if let Some(bytes) = cached.get(hash) { Ok(bytes.to_owned()) } else { - let new_artifact = self.fallback.get_artifact(name, version, hash)?; + let new_artifact = self.fallback.as_ref().get_artifact(name, version, hash)?; cached.insert(hash.to_owned(), new_artifact.clone()); self.artifacts_cache.refresh(cached); Ok(new_artifact) @@ -89,7 +89,7 @@ impl IStorage for CachedStorage { if let Some(bytes) = cached.get(name) { Ok(bytes.to_owned()) } else { - let new_image = self.fallback.get_image(name)?; + let new_image = self.fallback.as_ref().get_image(name)?; cached.insert(name.to_owned(), new_image.clone()); self.images_cache.refresh(cached); Ok(new_image) @@ -97,6 +97,6 @@ impl IStorage for CachedStorage { } fn get_statistics(&self) -> std::collections::HashMap { - self.statistics_cache.get(|| self.fallback.get_statistics()) + self.statistics_cache.get(|| self.fallback.as_ref().get_statistics()) } } diff --git a/src/storage/interface.rs b/src/storage/interface.rs index 86bb0f8..a51b630 100644 --- a/src/storage/interface.rs +++ b/src/storage/interface.rs @@ -1,4 +1,4 @@ -pub trait IStorage { +pub trait IStorage: Send + Sync { fn plugins(&self) -> decky_api::StorePluginList; fn get_artifact(&self, _name: &str, _version: &str, _hash: &str) -> Result { @@ -14,33 +14,6 @@ pub trait IStorage { } } -pub trait IStorageWrap: Sized + IStorage { - fn wrap(self, conf: crate::cli::CliArgs) -> Box; -} - -impl IStorageWrap for X { - fn wrap(self, conf: crate::cli::CliArgs) -> Box { - let proxy = if let Some(store) = conf.proxy_store { - Some(store) - } else if conf.proxy { - Some("https://plugins.deckbrew.xyz".to_owned()) - } else { - None - }; - match (proxy, conf.cache_duration) { - (Some(proxy), Some(cache_dur)) => Box::new( - super::CachedStorage::new( - cache_dur, - super::ProxiedStorage::new(proxy, self), - ) - ), - (Some(proxy), None) => Box::new(super::ProxiedStorage::new(proxy, self)), - (None, Some(cache_dur)) => Box::new(super::CachedStorage::new(cache_dur, self)), - (None, None) => Box::new(self), - } - } -} - pub struct EmptyStorage; impl IStorage for EmptyStorage { diff --git a/src/storage/merge.rs b/src/storage/merge.rs new file mode 100644 index 0000000..3ec8bf9 --- /dev/null +++ b/src/storage/merge.rs @@ -0,0 +1,149 @@ +use std::collections::HashMap; +use std::sync::RwLock; + +use decky_api::{StorePluginList, StorePlugin}; + +use super::IStorage; + +struct StoreIndex(usize); +#[derive(Hash, Eq, PartialEq)] +struct StoreName(String); + +#[derive(Hash, Eq, PartialEq)] +struct HashablePluginVersion { + plugin_name: String, + version_name: String, + hash: String, +} + +pub struct MergedStorage + Send + Sync> { + stores: Vec, + store_artifact_map: RwLock>, + store_image_map: RwLock>>, +} + +impl + Send + Sync> MergedStorage { + pub fn new(inner: Vec) -> Self { + Self { + stores: inner, + store_artifact_map: RwLock::new(HashMap::new()), + store_image_map: RwLock::new(HashMap::new()), + } + } + + /*pub fn add(mut self, store: S) -> Self { + self.stores.push(store); + self + }*/ + + fn map_to_vec(plugins: HashMap) -> StorePluginList { + let mut result = Vec::with_capacity(plugins.len()); + for (_, val) in plugins { + result.push(val); + } + result + } + + fn merge_plugins_into(dest: &mut HashMap, source: StorePluginList) { + for mut plugin in source { + let store_name = StoreName(plugin.name.clone()); + if let Some(existing_plugin) = dest.get_mut(&store_name) { + // combine versions if the plugin has the same name as an existing one + existing_plugin.versions.append(&mut plugin.versions); + } else { + // create new plugin entry if not + dest.insert(store_name, plugin); + } + } + } + + fn merge_statistics_into(dest: &mut HashMap, source: HashMap) { + for (entry, val) in source { + if let Some(existing_stat) = dest.get_mut(&entry) { + // combine if already exists + *existing_stat += val; + } else { + // create new plugin entry if not + dest.insert(entry, val); + } + } + } +} + +impl + Send + Sync> IStorage for MergedStorage { + fn plugins(&self) -> StorePluginList { + let mut merged_plugins = HashMap::new(); + log::debug!("Acquiring store map write locks"); + let mut arti_lock = self.store_artifact_map.write().expect("Failed to acquire store_artifact_map write lock"); + let mut img_lock = self.store_image_map.write().expect("Failed to acquire store_image_map write lock"); + for (index, store) in self.stores.iter().enumerate() { + let plugins = store.as_ref().plugins(); + // re-build store mappins + for plugin in &plugins { + for version in &plugin.versions { + let hashable_ver = HashablePluginVersion { + plugin_name: plugin.name.clone(), + version_name: version.name.clone(), + hash: version.hash.clone(), + }; + arti_lock.insert(hashable_ver, StoreIndex(index)); + } + let store_name = StoreName(plugin.name.clone()); + if let Some(stores) = img_lock.get_mut(&store_name) { + stores.push(StoreIndex(index)); + } else { + img_lock.insert(store_name, vec![StoreIndex(index)]); + } + } + Self::merge_plugins_into(&mut merged_plugins, plugins); + } + Self::map_to_vec(merged_plugins) + } + + fn get_artifact(&self, name: &str, version: &str, hash: &str) -> Result { + log::debug!("Acquiring store_artifact_map read lock"); + let lock = self.store_artifact_map.read().expect("Failed to acquire store_artifact_map read lock"); + if let Some(index) = lock.get(&HashablePluginVersion { + plugin_name: name.to_owned(), + version_name: version.to_owned(), + hash: hash.to_owned(), + }) { + if let Some(store) = self.stores.get(index.0) { + store.as_ref().get_artifact(name, version, hash) + } else { + Err(std::io::Error::new(std::io::ErrorKind::NotFound, format!("Store index {} does not exist", index.0))) + } + } else { + Err(std::io::Error::new(std::io::ErrorKind::NotFound, "Plugin version does not exist in any store")) + } + } + + fn get_image(&self, name: &str) -> Result { + log::debug!("Acquiring store_image_map read lock"); + let lock = self.store_image_map.read().expect("Failed to acquire store_image_map read lock"); + if let Some(indices) = lock.get(&StoreName(name.to_owned())) { + for index in indices { + if let Some(store) = self.stores.get(index.0) { + match store.as_ref().get_image(name) { + Ok(img) => return Ok(img), + Err(e) => log::error!("Error retrieving image from store #{}: {}", index.0, e), + } + } + } + + Err(std::io::Error::new(std::io::ErrorKind::NotFound, "Stores do not exist for that plugin")) + } else { + Err(std::io::Error::new(std::io::ErrorKind::NotFound, "Plugin does not exist in any store")) + } + } + + fn get_statistics(&self) -> std::collections::HashMap { + let mut stats = HashMap::new(); + for store in &self.stores { + let new_stats = store.as_ref().get_statistics(); + Self::merge_statistics_into(&mut stats, new_stats); + } + + stats + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 73da24b..1c6128d 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,9 +1,11 @@ mod cache; mod filesystem; mod interface; +mod merge; mod proxy; pub use cache::CachedStorage; pub use filesystem::FileStorage; -pub use interface::{IStorage, EmptyStorage, IStorageWrap}; +pub use interface::{IStorage, EmptyStorage}; +pub use merge::MergedStorage; pub use proxy::ProxiedStorage; diff --git a/src/storage/proxy.rs b/src/storage/proxy.rs index a013a12..a2f6b0b 100644 --- a/src/storage/proxy.rs +++ b/src/storage/proxy.rs @@ -2,17 +2,15 @@ use decky_api::{StorePluginList, StorePluginVersion}; use super::IStorage; -pub struct ProxiedStorage { +pub struct ProxiedStorage { store_url: String, - fallback: S, agent: ureq::Agent, } -impl ProxiedStorage { - pub fn new(target_store: String, inner: S) -> Self { +impl ProxiedStorage { + pub fn new(target_store: String) -> Self { Self { store_url: target_store, - fallback: inner, agent: ureq::Agent::new(), } } @@ -45,7 +43,7 @@ impl ProxiedStorage { } } -impl IStorage for ProxiedStorage { +impl IStorage for ProxiedStorage { fn plugins(&self) -> StorePluginList { let mut proxy = self.proxy_plugins(); for plugin in &mut proxy { @@ -55,12 +53,10 @@ impl IStorage for ProxiedStorage { } } } - let fallback = self.fallback.plugins(); - proxy.extend_from_slice(&fallback); proxy } - fn get_artifact(&self, name: &str, version: &str, hash: &str) -> Result { + /*fn get_artifact(&self, name: &str, version: &str, hash: &str) -> Result { self.fallback.get_artifact(name, version, hash) } @@ -70,5 +66,5 @@ impl IStorage for ProxiedStorage { fn get_statistics(&self) -> std::collections::HashMap { self.fallback.get_statistics() - } + }*/ }