diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 100eea2..de5585d 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1170,7 +1170,7 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "powertools" -version = "2.0.0" +version = "2.0.1" dependencies = [ "async-trait", "chrono", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7d7f6b1..07987c7 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "powertools" -version = "2.0.0" +version = "2.0.1" edition = "2021" authors = ["NGnius (Graham) "] description = "Backend (superuser) functionality for PowerTools" diff --git a/backend/build.sh b/backend/build.sh index 689ae88..416eeb7 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -3,11 +3,11 @@ #cargo build --release --target x86_64-unknown-linux-musl #cargo build --target x86_64-unknown-linux-musl #cross build -cargo build --release -#cargo build +#cargo build --release +cargo build mkdir -p ../bin #cp --preserve=mode ./target/x86_64-unknown-linux-musl/release/powertools ../bin/backend #cp --preserve=mode ./target/x86_64-unknown-linux-musl/debug/powertools ../bin/backend -cp --preserve=mode ./target/release/powertools ../bin/backend -#cp --preserve=mode ./target/debug/powertools ../bin/backend +#cp --preserve=mode ./target/release/powertools ../bin/backend +cp --preserve=mode ./target/debug/powertools ../bin/backend diff --git a/backend/community_settings_srv/Cargo.lock b/backend/community_settings_srv/Cargo.lock index 1cf7fd3..3783d84 100644 --- a/backend/community_settings_srv/Cargo.lock +++ b/backend/community_settings_srv/Cargo.lock @@ -438,7 +438,7 @@ dependencies = [ [[package]] name = "community_settings_srv" -version = "0.1.0" +version = "0.1.1" dependencies = [ "actix-web", "clap", @@ -450,6 +450,7 @@ dependencies = [ "serde_json", "simplelog", "tokio", + "walkdir", ] [[package]] @@ -1017,6 +1018,15 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1319,6 +1329,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/backend/community_settings_srv/Cargo.toml b/backend/community_settings_srv/Cargo.toml index 37cb95c..aee36ff 100644 --- a/backend/community_settings_srv/Cargo.toml +++ b/backend/community_settings_srv/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "community_settings_srv" -version = "0.1.0" +version = "0.1.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -16,6 +16,8 @@ tokio = { version = "1", features = ["full"] } actix-web = { version = "4.4" } mime = { version = "0.3.17" } +walkdir = "2" + # logging log = "0.4" simplelog = "0.12" diff --git a/backend/community_settings_srv/build_arm64.sh b/backend/community_settings_srv/build_arm64.sh new file mode 100755 index 0000000..7010ff6 --- /dev/null +++ b/backend/community_settings_srv/build_arm64.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +cargo build --release --target aarch64-unknown-linux-musl diff --git a/backend/community_settings_srv/src/api/get_game.rs b/backend/community_settings_srv/src/api/get_game.rs index ec19424..8e42556 100644 --- a/backend/community_settings_srv/src/api/get_game.rs +++ b/backend/community_settings_srv/src/api/get_game.rs @@ -60,11 +60,12 @@ fn get_some_settings_by_app_id(steam_app_id: u32, cli: &'static Cli) -> std::io: let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, steam_app_id); let mut files: Vec<_> = app_id_folder.read_dir()? .filter_map(|res| res.ok()) - .filter(|f| f.path().extension().map(|ext| ext == file_util::RON_EXTENSION).unwrap_or(false)) + .filter(|f| f.path().extension().map(|ext| ext == crate::consts::RON_EXTENSION).unwrap_or(false)) .filter_map(|f| f.metadata().ok().map(|meta| (f, meta))) - .filter_map(|(f, meta)| meta.created().ok().map(|time| (f, meta, time))) + .filter_map(|(f, meta)| meta.modified().ok().map(|time| (f, meta, time))) .collect(); files.sort_by(|(_, _, a_created), (_, _, b_created)| a_created.cmp(b_created)); + let files_len = files.len(); let mut results = Vec::with_capacity(MAX_RESULTS); for (_, (f, _, _)) in files.into_iter().enumerate().take_while(|(i, _)| *i < MAX_RESULTS) { @@ -72,12 +73,14 @@ fn get_some_settings_by_app_id(steam_app_id: u32, cli: &'static Cli) -> std::io: let setting = match ron::de::from_reader(reader) { Ok(x) => x, Err(e) => { + log::debug!("Error while reading {}: {}", f.path().display(), e); let e_msg = format!("{}", e); return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); } }; results.push(setting); } + log::debug!("Got {} results (from {} files) for {}", results.len(), files_len, app_id_folder.display()); Ok(results) } @@ -88,7 +91,8 @@ pub async fn get_setting_by_app_id_handler( cli: web::Data<&'static Cli>, ) -> std::io::Result { let id: u32 = *id; - println!("Accept: {}", accept.to_string()); + #[cfg(debug_assertions)] + log::debug!("Accept: {}", accept.to_string()); let preferred = accept.preference(); if super::is_mime_type_ron_capable(&preferred) { // Send RON diff --git a/backend/community_settings_srv/src/api/get_setting.rs b/backend/community_settings_srv/src/api/get_setting.rs index 3121192..9d3a05b 100644 --- a/backend/community_settings_srv/src/api/get_setting.rs +++ b/backend/community_settings_srv/src/api/get_setting.rs @@ -67,7 +67,7 @@ pub async fn get_setting_handler( if super::is_mime_type_ron_capable(&preferred) { // Send RON let ron = if id != 0 { - let path = file_util::setting_path_by_id(&cli.folder, id, file_util::RON_EXTENSION); + let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::RON_EXTENSION); if !path.exists() { return Err(std::io::Error::new(std::io::ErrorKind::NotFound, format!("setting id {} does not exist", id))); } @@ -93,7 +93,7 @@ pub async fn get_setting_handler( } else { // Send JSON (fallback) let json = if id != 0 { - let path = file_util::setting_path_by_id(&cli.folder, id, file_util::JSON_EXTENSION); + let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::JSON_EXTENSION); // TODO? cache this instead of always loading it from file let reader = std::io::BufReader::new(std::fs::File::open(path)?); match serde_json::from_reader(reader) { diff --git a/backend/community_settings_srv/src/api/save_setting.rs b/backend/community_settings_srv/src/api/save_setting.rs index 173364f..6169d06 100644 --- a/backend/community_settings_srv/src/api/save_setting.rs +++ b/backend/community_settings_srv/src/api/save_setting.rs @@ -18,7 +18,7 @@ pub async fn save_setting_handler( Err(_e) => return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "too many bytes in payload")), }; let next_id = file_util::next_setting_id(&cli.folder); - let parsed_data: community_settings_core::v1::Metadata = if super::is_mime_type_ron_capable(&content_type) { + let mut parsed_data: community_settings_core::v1::Metadata = if super::is_mime_type_ron_capable(&content_type) { // Parse as RON match ron::de::from_reader(bytes.as_ref()) { Ok(x) => x, @@ -42,16 +42,18 @@ pub async fn save_setting_handler( } } }; + // Override the ID with the one used by this server + parsed_data.id = next_id.to_string(); // TODO validate user and app id // Reject blocked users and apps - let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, file_util::RON_EXTENSION); + let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::RON_EXTENSION); let writer = std::io::BufWriter::new(std::fs::File::create(&path_ron)?); if let Err(e) = ron::ser::to_writer(writer, &parsed_data) { let e_msg = format!("{}", e); return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); } - let path_json = file_util::setting_path_by_id(&cli.folder, next_id, file_util::JSON_EXTENSION); + let path_json = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::JSON_EXTENSION); let writer = std::io::BufWriter::new(std::fs::File::create(&path_json)?); if let Err(e) = serde_json::to_writer(writer, &parsed_data) { let e_msg = format!("{}", e); @@ -62,57 +64,32 @@ pub async fn save_setting_handler( } } - // create symlinks for other ways of looking up these settings files - let filename_ron = file_util::filename(next_id, file_util::RON_EXTENSION); - let filename_json = file_util::filename(next_id, file_util::JSON_EXTENSION); + log::debug!("Saved to {}, building symlinks", path_ron.display()); - // create symlinks to app id folder - let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, parsed_data.steam_app_id); - if !app_id_folder.exists() { - std::fs::create_dir(&app_id_folder)?; - } - #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained - { - std::os::windows::fs::symlink_file(&path_ron, app_id_folder.join(&filename_ron))?; - std::os::windows::fs::symlink_file(&path_json, app_id_folder.join(&filename_json))?; - } - #[cfg(target_family = "unix")] - { - std::os::unix::fs::symlink(&path_ron, app_id_folder.join(&filename_ron))?; - std::os::unix::fs::symlink(&path_json, app_id_folder.join(&filename_json))?; - } + let to_symlink = file_util::symlinks(&cli.folder, &parsed_data)?; - // create symlinks for user id folder - let user_id_folder = file_util::setting_folder_by_user_id(&cli.folder, parsed_data.steam_user_id); - if !user_id_folder.exists() { - std::fs::create_dir(&user_id_folder)?; - } - #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained - { - std::os::windows::fs::symlink_file(&path_ron, user_id_folder.join(&filename_ron))?; - std::os::windows::fs::symlink_file(&path_json, user_id_folder.join(&filename_json))?; - } - #[cfg(target_family = "unix")] - { - std::os::unix::fs::symlink(&path_ron, user_id_folder.join(&filename_ron))?; - std::os::unix::fs::symlink(&path_json, user_id_folder.join(&filename_json))?; - } - - // create symlinks for each tag - for tag in parsed_data.tags.iter() { - let tag_folder = file_util::setting_folder_by_tag(&cli.folder, tag); - if !tag_folder.exists() { - std::fs::create_dir(&tag_folder)?; - } + let path_ron_canon = path_ron.canonicalize()?; + let path_json_canon = path_json.canonicalize()?; + for ron_link in to_symlink.ron { + log::debug!("Symlinking {} -> {}", ron_link.display(), path_ron_canon.display()); #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { - std::os::windows::fs::symlink_file(&path_ron, tag_folder.join(&filename_ron))?; - std::os::windows::fs::symlink_file(&path_json, tag_folder.join(&filename_json))?; + std::os::windows::fs::symlink_file(&path_ron_canon, &ron_link)?; } #[cfg(target_family = "unix")] { - std::os::unix::fs::symlink(&path_ron, tag_folder.join(&filename_ron))?; - std::os::unix::fs::symlink(&path_json, tag_folder.join(&filename_json))?; + std::os::unix::fs::symlink(&path_ron_canon, &ron_link)?; + } + } + for json_link in to_symlink.json { + //log::debug!("Symlinking {} -> {}", json_link.display(), path_json_canon.display()); + #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained + { + std::os::windows::fs::symlink_file(&path_json_canon, json_link)?; + } + #[cfg(target_family = "unix")] + { + std::os::unix::fs::symlink(&path_json_canon, json_link)?; } } diff --git a/backend/community_settings_srv/src/cli.rs b/backend/community_settings_srv/src/cli.rs index b787355..f960560 100644 --- a/backend/community_settings_srv/src/cli.rs +++ b/backend/community_settings_srv/src/cli.rs @@ -14,6 +14,14 @@ pub struct Cli { /// Log file location #[arg(short, long, default_value = "/tmp/powertools_community_settings_srv.log")] pub log: std::path::PathBuf, + + /// Perform maintenance tasks + #[arg(long)] + pub fix: bool, + + /// Keep up to this many duplicate settings (0 will delete everything!!!) + #[arg(long, default_value_t = 2)] + pub duplicates: usize, } impl Cli { diff --git a/backend/community_settings_srv/src/consts.rs b/backend/community_settings_srv/src/consts.rs new file mode 100644 index 0000000..833fd90 --- /dev/null +++ b/backend/community_settings_srv/src/consts.rs @@ -0,0 +1,8 @@ +pub const RON_EXTENSION: &'static str = "ron"; +pub const JSON_EXTENSION: &'static str = "json"; + +pub const SETTING_FOLDER: &'static str = "settings"; +pub const ID_FOLDER: &'static str = "by_id"; +pub const APP_ID_FOLDER: &'static str = "by_app_id"; +pub const USER_ID_FOLDER: &'static str = "by_user_id"; +pub const TAG_FOLDER: &'static str = "by_tag"; diff --git a/backend/community_settings_srv/src/file_util.rs b/backend/community_settings_srv/src/file_util.rs index 670c608..f776da2 100644 --- a/backend/community_settings_srv/src/file_util.rs +++ b/backend/community_settings_srv/src/file_util.rs @@ -1,41 +1,10 @@ use std::path::{Path, PathBuf}; use std::sync::Mutex; -pub const RON_EXTENSION: &'static str = "ron"; -pub const JSON_EXTENSION: &'static str = "json"; - -const SETTING_FOLDER: &'static str = "settings"; -const ID_FOLDER: &'static str = "by_id"; -const APP_ID_FOLDER: &'static str = "by_app_id"; -const USER_ID_FOLDER: &'static str = "by_user_id"; -const TAG_FOLDER: &'static str = "by_tag"; +use crate::consts::*; static LAST_SETTING_ID: Mutex = Mutex::new(0); -pub fn build_folder_layout(root: impl AsRef) -> std::io::Result<()> { - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(ID_FOLDER) - )?; - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(APP_ID_FOLDER) - )?; - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(USER_ID_FOLDER) - )?; - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(TAG_FOLDER) - )?; - Ok(()) -} - pub fn filename(id: u128, ext: &str) -> String { format!("{}.{}", id, ext) } @@ -54,6 +23,23 @@ pub fn setting_folder_by_app_id(root: impl AsRef, steam_app_id: u32) -> Pa .join(steam_app_id.to_string()) } +pub fn setting_tag_folder_by_app_id(root: impl AsRef, steam_app_id: u32) -> PathBuf { + root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .join(steam_app_id.to_string()) + .join(TAG_FOLDER) +} + +pub fn setting_folder_by_app_id_tag(root: impl AsRef, steam_app_id: u32, tag: &str) -> PathBuf { + root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .join(steam_app_id.to_string()) + .join(TAG_FOLDER) + .join(tag) +} + pub fn setting_folder_by_user_id(root: impl AsRef, steam_user_id: u64) -> PathBuf { root.as_ref() .join(SETTING_FOLDER) @@ -61,6 +47,23 @@ pub fn setting_folder_by_user_id(root: impl AsRef, steam_user_id: u64) -> .join(steam_user_id.to_string()) } +pub fn setting_tag_folder_by_user_id(root: impl AsRef, steam_user_id: u64) -> PathBuf { + root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .join(steam_user_id.to_string()) + .join(TAG_FOLDER) +} + +pub fn setting_folder_by_user_id_tag(root: impl AsRef, steam_user_id: u64, tag: &str) -> PathBuf { + root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .join(steam_user_id.to_string()) + .join(TAG_FOLDER) + .join(tag) +} + pub fn setting_folder_by_tag(root: impl AsRef, tag: &str) -> PathBuf { root.as_ref() .join(SETTING_FOLDER) @@ -73,14 +76,83 @@ pub fn next_setting_id(root: impl AsRef) -> u128 { let mut last_id = *lock; if last_id == 0 { // needs init + last_id = 1; let mut path = setting_path_by_id(root.as_ref(), last_id, RON_EXTENSION); while path.exists() { last_id += 1; path = setting_path_by_id(root.as_ref(), last_id, RON_EXTENSION); } - *lock = last_id; - println!("setting id initialized to {}", last_id); + *lock = last_id - 1; + log::info!("setting id initialized to {}", last_id); } *lock += 1; *lock } + +pub struct ToSymlink { + pub ron: Vec, + pub json: Vec, +} + +pub fn symlinks(root: impl AsRef, meta: &community_settings_core::v1::Metadata) -> std::io::Result { + let mut symlink_locations = ToSymlink { ron: Vec::new(), json: Vec::new() }; + let filename_ron = filename(meta.get_id(), crate::consts::RON_EXTENSION); + let filename_json = filename(meta.get_id(), crate::consts::JSON_EXTENSION); + // build symlinks to app id folder + let app_id_folder = setting_folder_by_app_id(&root, meta.steam_app_id); + log::debug!("App id folder {}", app_id_folder.display()); + if !app_id_folder.exists() { + std::fs::create_dir(&app_id_folder)?; + std::fs::create_dir(setting_tag_folder_by_app_id(&root, meta.steam_app_id))?; + } + symlink_locations.ron.push(app_id_folder.join(&filename_ron)); + symlink_locations.json.push(app_id_folder.join(&filename_json)); + + // create symlinks for user id folder + let user_id_folder = setting_folder_by_user_id(&root, meta.steam_user_id); + if !user_id_folder.exists() { + std::fs::create_dir(&user_id_folder)?; + std::fs::create_dir(setting_tag_folder_by_user_id(&root, meta.steam_user_id))?; + } + symlink_locations.ron.push(user_id_folder.join(&filename_ron)); + symlink_locations.json.push(user_id_folder.join(&filename_json)); + + // create symlinks for each tag + for tag in meta.tags.iter() { + if !str_is_alphanumeric_or_space(&tag){ + continue; + } + // create symlinks for general tag folder + let tag_folder = setting_folder_by_tag(&root, tag); + if !tag_folder.exists() { + std::fs::create_dir(&tag_folder)?; + } + symlink_locations.ron.push(tag_folder.join(&filename_ron)); + symlink_locations.json.push(tag_folder.join(&filename_json)); + + // create symlinks for app id tag folder + let app_tag_folder = setting_folder_by_app_id_tag(&root, meta.steam_app_id, tag); + if !app_tag_folder.exists() { + std::fs::create_dir(&app_tag_folder)?; + } + symlink_locations.ron.push(app_tag_folder.join(&filename_ron)); + symlink_locations.json.push(app_tag_folder.join(&filename_json)); + + // create symlinks for user id tag folder + let user_tag_folder = setting_folder_by_user_id_tag(&root, meta.steam_user_id, tag); + if !user_tag_folder.exists() { + std::fs::create_dir(&user_tag_folder)?; + } + symlink_locations.ron.push(user_tag_folder.join(&filename_ron)); + symlink_locations.json.push(user_tag_folder.join(&filename_json)); + } + Ok(symlink_locations) +} + +fn str_is_alphanumeric_or_space(s: &str) -> bool { + let mut result = true; + for ch in s.chars() { + result &= ch.is_ascii_alphanumeric() || ch == ' '; + } + result +} diff --git a/backend/community_settings_srv/src/main.rs b/backend/community_settings_srv/src/main.rs index 2753bfa..b3f26d0 100644 --- a/backend/community_settings_srv/src/main.rs +++ b/backend/community_settings_srv/src/main.rs @@ -1,6 +1,9 @@ mod api; mod cli; +mod consts; mod file_util; +mod tasks; +mod upgrade; use actix_web::{web, App, HttpServer}; @@ -27,7 +30,22 @@ async fn main() -> std::io::Result<()> { // setup log::debug!("Building folder layout (if not exists) at: {}", &args.folder.display()); - file_util::build_folder_layout(&args.folder)?; + upgrade::build_folder_layout(&args.folder)?; + + // fix things + if args.fix { + log::info!("Fixing old symlinks"); + upgrade::fix_symlinks(&args.folder)?; + log::info!("Creating missing by_tag folders"); + upgrade::make_tag_subfolders(&args.folder)?; + log::info!("Resynchronizing file IDs with file name IDs"); + upgrade::sync_ids(&args.folder)?; + log::info!("Rebuilding missing symlinks"); + upgrade::rebuild_symlinks(&args.folder)?; + return Ok(()) + } + + tasks::start_tasks(args.clone()); let leaked_args: &'static cli::Cli = Box::leak::<'static>(Box::new(args)); HttpServer::new(move || { diff --git a/backend/community_settings_srv/src/tasks/mod.rs b/backend/community_settings_srv/src/tasks/mod.rs new file mode 100644 index 0000000..b922f4b --- /dev/null +++ b/backend/community_settings_srv/src/tasks/mod.rs @@ -0,0 +1,16 @@ +mod task_runner; +mod symlink_cleanup; +mod user_antispam; + +pub fn start_tasks(args: crate::cli::Cli) { + task_runner::TaskRunner::new( + args.clone(), + symlink_cleanup::remove_broken_symlinks, + std::time::Duration::from_secs(1 * 60 /* 1 minute */), + ).run(); + task_runner::TaskRunner::new( + args.clone(), + user_antispam::remove_similar_user_uploads, + std::time::Duration::from_secs(5 * 60 /* 5 minutes */), + ).run(); +} diff --git a/backend/community_settings_srv/src/tasks/symlink_cleanup.rs b/backend/community_settings_srv/src/tasks/symlink_cleanup.rs new file mode 100644 index 0000000..34893e8 --- /dev/null +++ b/backend/community_settings_srv/src/tasks/symlink_cleanup.rs @@ -0,0 +1,39 @@ +//! Realistically this shouldn't be occur unless there's corruption or a settings files is manually deleted + +pub fn remove_broken_symlinks(args: &mut crate::cli::Cli) { + log::info!("Starting broken symlink removal task"); + if let Err(e) = enforce_working_symlinks(&args.folder) { + log::error!("Error in broken symlink task: {}", e); + } + log::info!("Completed broken symlink removal task"); +} + +fn enforce_working_symlinks(root: impl AsRef) -> std::io::Result<()> { + for dir_entry in walkdir::WalkDir::new(root.as_ref().join(crate::consts::SETTING_FOLDER)) { + let dir_entry = dir_entry?; + match check_symlink(dir_entry.path()) { + Ok(true) => {}, + Ok(false) => { + log::info!("Symlink {} seems broken, removing...", dir_entry.path().display()); + if let Err(e) = std::fs::remove_file(dir_entry.path()) { + log::warn!("Failed to delete broken symlink {}: {}", dir_entry.path().display(), e); + } + }, + Err(symlink_e) => { + log::info!("Thing {} seems broken, removing... err: {}", dir_entry.path().display(), symlink_e); + if let Err(e) = std::fs::remove_file(dir_entry.path()) { + log::warn!("Failed to delete broken thing {}: {}", dir_entry.path().display(), e); + } + } + } + } + Ok(()) +} + +fn check_symlink(path: impl AsRef) -> std::io::Result { + if path.as_ref().is_symlink() { + Ok(path.as_ref().read_link().is_ok_and(|link| link.exists())) + } else { + Ok(true) + } +} diff --git a/backend/community_settings_srv/src/tasks/task_runner.rs b/backend/community_settings_srv/src/tasks/task_runner.rs new file mode 100644 index 0000000..ced9533 --- /dev/null +++ b/backend/community_settings_srv/src/tasks/task_runner.rs @@ -0,0 +1,47 @@ +use std::thread::{spawn, JoinHandle, sleep}; +use std::time::{Duration, Instant}; + +pub struct TaskRunner { + task: F, + context: C, + period: Option, +} + +impl TaskRunner { + pub fn new(c: C, f: F, period: Duration) -> Self { + Self { + task: f, + context: c, + period: Some(period), + } + } + + /* + pub fn new_oneshot(c: C, f: F) -> Self { + Self { + task: f, + context: c, + period: None, + } + }*/ + + pub fn run(mut self) -> JoinHandle<()> { + if let Some(period) = self.period { + spawn(move || { + let mut pre_task; + let mut after_task; + loop { + pre_task = Instant::now(); + (self.task)(&mut self.context); + after_task = Instant::now(); + sleep(period - (after_task.duration_since(pre_task))); + } + }) + } else { + spawn(move || { + (self.task)(&mut self.context); + }) + } + + } +} diff --git a/backend/community_settings_srv/src/tasks/user_antispam.rs b/backend/community_settings_srv/src/tasks/user_antispam.rs new file mode 100644 index 0000000..e3379df --- /dev/null +++ b/backend/community_settings_srv/src/tasks/user_antispam.rs @@ -0,0 +1,114 @@ +use crate::file_util; + +pub fn remove_similar_user_uploads(args: &mut crate::cli::Cli) { + log::info!("Starting user antispam task"); + if let Err(e) = enforce_user_dirs(&args.folder, args.duplicates) { + log::error!("Error in user antispam task: {}", e); + } + log::info!("Completed user antispam task"); +} + +fn enforce_user_dirs(root: impl AsRef, keep_duplicates: usize) -> std::io::Result<()> { + let mut to_remove: Vec = Vec::new(); + let mut seen_names = std::collections::HashMap::>::new(); + for dir_entry in root.as_ref() + .join(crate::consts::SETTING_FOLDER) + .join(crate::consts::USER_ID_FOLDER) + .read_dir()? { + match dir_entry { + Ok(dir_entry) => { + log::info!("Scanning {} for user antispam", dir_entry.path().display()); + if dir_entry.file_type()?.is_dir() { + seen_names.clear(); + to_remove.clear(); + for user_entry in dir_entry.path().read_dir()? { + match user_entry { + Ok(user_entry) => { + let f_path = user_entry.path(); + if let Some(ext) = f_path.extension() { + if ext == crate::consts::RON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + match ron::de::from_reader::<_, community_settings_core::v1::Metadata>(reader) { + Ok(x) => { + let sani_name = sanitise_name(&x.name); + if let Some(seen_in_ids) = seen_names.get_mut(&sani_name) { + seen_in_ids.push(x); + } else { + seen_names.insert(sani_name, vec![x]); + } + + }, + Err(e) => log::debug!("Error while reading {}: {}", f_path.display(), e) + } + } + } + }, + Err(e) => log::warn!("Skipping file in {} for user antispam task due to error: {}", dir_entry.path().display(), e), + } + } + for seen in seen_names.values_mut() { + seen.sort_by_key(|meta| meta.get_id()); // sort lowest to highest + // keep the highest id (i.e. latest uploaded) settings + for _ in 0..keep_duplicates { + seen.pop(); + } + to_remove.append(seen); + } + log::info!("Found {} spammy entries in {}", to_remove.len(), dir_entry.path().display()); + // remove settings (and related symlinks) from the filesystem + for meta in to_remove.iter() { + let filename_ron = file_util::filename(meta.get_id(), crate::consts::RON_EXTENSION); + let filename_json = file_util::filename(meta.get_id(), crate::consts::JSON_EXTENSION); + // delete tag symlinks + for tag in meta.tags.iter() { + let app_id_tag_folder = file_util::setting_folder_by_app_id_tag(root.as_ref(), meta.steam_app_id, tag); + let user_id_tag_folder = file_util::setting_folder_by_user_id_tag(root.as_ref(), meta.steam_user_id, tag); + let tag_folder = file_util::setting_folder_by_tag(root.as_ref(), tag); + let paths = [ + app_id_tag_folder.join(&filename_json), + app_id_tag_folder.join(&filename_ron), + user_id_tag_folder.join(&filename_json), + user_id_tag_folder.join(&filename_ron), + tag_folder.join(&filename_json), + tag_folder.join(&filename_ron), + ]; + for path in paths { + if path.exists() { + if let Err(e) = std::fs::remove_file(&path) { + log::warn!("Failed to delete {}: {}", path.display(), e); + } + } + } + } + // delete first-order symlinks and finally the actual files + let app_id_folder = file_util::setting_folder_by_app_id(root.as_ref(), meta.steam_app_id); + let user_id_folder = file_util::setting_folder_by_user_id(root.as_ref(), meta.steam_user_id); + let paths = [ + app_id_folder.join(&filename_json), + app_id_folder.join(&filename_ron), + user_id_folder.join(&filename_json), + user_id_folder.join(&filename_ron), + file_util::setting_path_by_id(root.as_ref(), meta.get_id(), crate::consts::JSON_EXTENSION), + file_util::setting_path_by_id(root.as_ref(), meta.get_id(), crate::consts::RON_EXTENSION), + ]; + for path in paths { + if path.exists() { + if let Err(e) = std::fs::remove_file(&path) { + log::warn!("Failed to delete {}: {}", path.display(), e); + } + } + } + } + } else { + log::info!("Encountered non-folder path in user dir: {}", dir_entry.path().display()); + } + }, + Err(e) => log::warn!("Skipping file for user antispam task due to error: {}", e), + } + } + Ok(()) +} + +fn sanitise_name(name: &str) -> String { + name.trim().to_lowercase() +} diff --git a/backend/community_settings_srv/src/upgrade.rs b/backend/community_settings_srv/src/upgrade.rs new file mode 100644 index 0000000..052cb83 --- /dev/null +++ b/backend/community_settings_srv/src/upgrade.rs @@ -0,0 +1,217 @@ +use std::path::Path; + +use crate::consts::*; + +pub fn build_folder_layout(root: impl AsRef) -> std::io::Result<()> { + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(ID_FOLDER) + )?; + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + )?; + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + )?; + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(TAG_FOLDER) + )?; + Ok(()) +} + +pub fn rebuild_symlinks(root: impl AsRef) -> std::io::Result<()> { + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_file() { + let f_path = dir_entry.path(); + if let Some(ext) = f_path.extension() { + if ext == RON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + let setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) { + Ok(x) => x, + Err(e) => { + log::debug!("Error while reading {}: {}", f_path.display(), e); + let e_msg = format!("{}", e); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); + } + }; + let to_symlink = crate::file_util::symlinks(&root, &setting)?; + let path_ron_canon = f_path.canonicalize()?; + let path_json_canon = crate::file_util::setting_path_by_id(&root, setting.get_id(), JSON_EXTENSION).canonicalize()?; + for ron_link in to_symlink.ron { + if ron_link.exists() { continue; } + log::info!("Rebuilding symlink {} -> {}", ron_link.display(), path_ron_canon.display()); + #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained + { + std::os::windows::fs::symlink_file(&path_ron_canon, &ron_link)?; + } + #[cfg(target_family = "unix")] + { + std::os::unix::fs::symlink(&path_ron_canon, &ron_link)?; + } + } + for json_link in to_symlink.json { + if json_link.exists() { continue; } + log::info!("Rebuilding symlink {} -> {}", json_link.display(), path_json_canon.display()); + #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained + { + std::os::windows::fs::symlink_file(&path_json_canon, json_link)?; + } + #[cfg(target_family = "unix")] + { + std::os::unix::fs::symlink(&path_json_canon, json_link)?; + } + } + } + } + } + } + Ok(()) +} + +pub fn fix_symlinks(root: impl AsRef) -> std::io::Result<()> { + log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display()); + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(TAG_FOLDER).read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + + Ok(()) +} + +fn make_symlinks_absolute_in_dir(root: impl AsRef, dir: impl AsRef) -> std::io::Result<()> { + let abs_root = root.as_ref().canonicalize()?; + assert!(abs_root.is_absolute()); + for dir_entry in dir.as_ref() + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_symlink() { + let path = dir_entry.path(); + let link_path = path.read_link()?; + if !link_path.is_absolute() { + let new_link = abs_root.join( + link_path.strip_prefix(&root).expect("Symlinked path does not begin with root settings folder") + ); + log::info!("Fixing {} -> {} to -> {}", path.display(), link_path.display(), new_link.display()); + std::fs::remove_file(&path)?; + #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained + { + std::os::windows::fs::symlink_file(new_link, &path)?; + } + #[cfg(target_family = "unix")] + { + std::os::unix::fs::symlink(new_link, &path)?; + } + }else { + log::info!("Found already-absolute symlink {} -> {}", path.display(), link_path.display()); + } + } else { + log::info!("Found non-symlink {}: {:?}", dir_entry.path().display(), dir_entry.file_type()?); + } + } + Ok(()) +} + +pub fn sync_ids(root: impl AsRef) -> std::io::Result<()> { + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + let f_path = dir_entry.path(); + if let Some(ext) = f_path.extension() { + let id = f_path.file_stem().map(|os| os.to_string_lossy().to_string()).unwrap(); + if ext == RON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + let mut setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) { + Ok(x) => x, + Err(e) => { + log::debug!("Error while reading {}: {}", f_path.display(), e); + let e_msg = format!("{}", e); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); + } + }; + if setting.id != id { + setting.id = id; + ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + } + } else if ext == JSON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) { + Ok(x) => x, + Err(e) => { + log::debug!("Error while reading {}: {}", f_path.display(), e); + let e_msg = format!("{}", e); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); + } + }; + if setting.id != id { + setting.id = id; + serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + } + } + } + } + Ok(()) +} + +pub fn make_tag_subfolders(root: impl AsRef) -> std::io::Result<()> { + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.metadata()?.is_dir() { + let tag_folder = dir_entry.path().join(TAG_FOLDER); + if !tag_folder.exists() { + std::fs::create_dir(&tag_folder)?; + } + } + } + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.metadata()?.is_dir() { + let tag_folder = dir_entry.path().join(TAG_FOLDER); + if !tag_folder.exists() { + std::fs::create_dir(&tag_folder)?; + } + } + } + // TODO populate folders + Ok(()) +} diff --git a/backend/src/settings/steam_deck/util.rs b/backend/src/settings/steam_deck/util.rs index 88d940c..3f02be1 100644 --- a/backend/src/settings/steam_deck/util.rs +++ b/backend/src/settings/steam_deck/util.rs @@ -31,7 +31,7 @@ const THINGS: &[u8] = &[ 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, ]; -const TIME_UNIT: std::time::Duration = std::time::Duration::from_millis(200); +const TIME_UNIT: std::time::Duration = std::time::Duration::from_millis(250); pub fn flash_led() { use smokepatio::ec::ControllerSet; diff --git a/package.json b/package.json index 56d46cd..c73bfeb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "PowerTools", - "version": "2.0.0", + "version": "2.0.1", "description": "Power tweaks for power users", "scripts": { "build": "shx rm -rf dist && rollup -c", diff --git a/src/store/page.tsx b/src/store/page.tsx index da8f36d..a9badb8 100644 --- a/src/store/page.tsx +++ b/src/store/page.tsx @@ -40,18 +40,22 @@ export class StoreResultsPage extends Component<{onNewVariant: () => void}> { { tr("No results") /* TODO translate */ } ); } else { - // TODO - return ( + return ( +
+ { storeItems.map((meta: backend.StoreMetadata) => ( void}> { )) } - ); + +
); } } else {