From 6f2c2b186fec981548265050346d2e2b8cdcf2da Mon Sep 17 00:00:00 2001 From: "NGnius (Graham)" Date: Fri, 12 Apr 2024 17:31:48 -0400 Subject: [PATCH 1/7] Fix symlinking logic --- backend/community_settings_srv/build_arm64.sh | 3 + .../src/api/get_game.rs | 1 + .../src/api/save_setting.rs | 24 +++---- backend/community_settings_srv/src/cli.rs | 4 ++ .../community_settings_srv/src/file_util.rs | 65 +++++++++++++++++++ backend/community_settings_srv/src/main.rs | 7 ++ 6 files changed, 92 insertions(+), 12 deletions(-) create mode 100755 backend/community_settings_srv/build_arm64.sh diff --git a/backend/community_settings_srv/build_arm64.sh b/backend/community_settings_srv/build_arm64.sh new file mode 100755 index 0000000..7010ff6 --- /dev/null +++ b/backend/community_settings_srv/build_arm64.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +cargo build --release --target aarch64-unknown-linux-musl diff --git a/backend/community_settings_srv/src/api/get_game.rs b/backend/community_settings_srv/src/api/get_game.rs index ec19424..cbfb95f 100644 --- a/backend/community_settings_srv/src/api/get_game.rs +++ b/backend/community_settings_srv/src/api/get_game.rs @@ -88,6 +88,7 @@ pub async fn get_setting_by_app_id_handler( cli: web::Data<&'static Cli>, ) -> std::io::Result { let id: u32 = *id; + #[cfg(debug_assertions)] println!("Accept: {}", accept.to_string()); let preferred = accept.preference(); if super::is_mime_type_ron_capable(&preferred) { diff --git a/backend/community_settings_srv/src/api/save_setting.rs b/backend/community_settings_srv/src/api/save_setting.rs index 173364f..ae41e3e 100644 --- a/backend/community_settings_srv/src/api/save_setting.rs +++ b/backend/community_settings_srv/src/api/save_setting.rs @@ -73,13 +73,13 @@ pub async fn save_setting_handler( } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { - std::os::windows::fs::symlink_file(&path_ron, app_id_folder.join(&filename_ron))?; - std::os::windows::fs::symlink_file(&path_json, app_id_folder.join(&filename_json))?; + std::os::windows::fs::symlink_file(&path_ron, app_id_folder.join(&filename_ron).canonicalize()?)?; + std::os::windows::fs::symlink_file(&path_json, app_id_folder.join(&filename_json).canonicalize()?)?; } #[cfg(target_family = "unix")] { - std::os::unix::fs::symlink(&path_ron, app_id_folder.join(&filename_ron))?; - std::os::unix::fs::symlink(&path_json, app_id_folder.join(&filename_json))?; + std::os::unix::fs::symlink(&path_ron, app_id_folder.join(&filename_ron).canonicalize()?)?; + std::os::unix::fs::symlink(&path_json, app_id_folder.join(&filename_json).canonicalize()?)?; } // create symlinks for user id folder @@ -89,13 +89,13 @@ pub async fn save_setting_handler( } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { - std::os::windows::fs::symlink_file(&path_ron, user_id_folder.join(&filename_ron))?; - std::os::windows::fs::symlink_file(&path_json, user_id_folder.join(&filename_json))?; + std::os::windows::fs::symlink_file(&path_ron, user_id_folder.join(&filename_ron).canonicalize()?)?; + std::os::windows::fs::symlink_file(&path_json, user_id_folder.join(&filename_json).canonicalize()?)?; } #[cfg(target_family = "unix")] { - std::os::unix::fs::symlink(&path_ron, user_id_folder.join(&filename_ron))?; - std::os::unix::fs::symlink(&path_json, user_id_folder.join(&filename_json))?; + std::os::unix::fs::symlink(&path_ron, user_id_folder.join(&filename_ron).canonicalize()?)?; + std::os::unix::fs::symlink(&path_json, user_id_folder.join(&filename_json).canonicalize()?)?; } // create symlinks for each tag @@ -106,13 +106,13 @@ pub async fn save_setting_handler( } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { - std::os::windows::fs::symlink_file(&path_ron, tag_folder.join(&filename_ron))?; - std::os::windows::fs::symlink_file(&path_json, tag_folder.join(&filename_json))?; + std::os::windows::fs::symlink_file(&path_ron, tag_folder.join(&filename_ron).canonicalize()?)?; + std::os::windows::fs::symlink_file(&path_json, tag_folder.join(&filename_json).canonicalize()?)?; } #[cfg(target_family = "unix")] { - std::os::unix::fs::symlink(&path_ron, tag_folder.join(&filename_ron))?; - std::os::unix::fs::symlink(&path_json, tag_folder.join(&filename_json))?; + std::os::unix::fs::symlink(&path_ron, tag_folder.join(&filename_ron).canonicalize()?)?; + std::os::unix::fs::symlink(&path_json, tag_folder.join(&filename_json).canonicalize()?)?; } } diff --git a/backend/community_settings_srv/src/cli.rs b/backend/community_settings_srv/src/cli.rs index b787355..5a05e14 100644 --- a/backend/community_settings_srv/src/cli.rs +++ b/backend/community_settings_srv/src/cli.rs @@ -14,6 +14,10 @@ pub struct Cli { /// Log file location #[arg(short, long, default_value = "/tmp/powertools_community_settings_srv.log")] pub log: std::path::PathBuf, + + /// Perform maintenance tasks + #[arg(short, long)] + pub fix: bool, } impl Cli { diff --git a/backend/community_settings_srv/src/file_util.rs b/backend/community_settings_srv/src/file_util.rs index 670c608..1272a0c 100644 --- a/backend/community_settings_srv/src/file_util.rs +++ b/backend/community_settings_srv/src/file_util.rs @@ -36,6 +36,71 @@ pub fn build_folder_layout(root: impl AsRef) -> std::io::Result<()> { Ok(()) } +pub fn fix_symlinks(root: impl AsRef) -> std::io::Result<()> { + log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display()); + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(TAG_FOLDER).read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + + Ok(()) +} + +fn make_symlinks_absolute_in_dir(root: impl AsRef, dir: impl AsRef) -> std::io::Result<()> { + let abs_root = root.as_ref().canonicalize()?; + assert!(abs_root.is_absolute()); + for dir_entry in dir.as_ref() + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_symlink() { + let path = dir_entry.path(); + let link_path = path.read_link()?; + if !link_path.is_absolute() { + let new_link = abs_root.join( + link_path.strip_prefix(&root).expect("Symlinked path does not begin with root settings folder") + ); + log::info!("Fixing {} -> {} to -> {}", path.display(), link_path.display(), new_link.display()); + std::fs::remove_file(&path)?; + #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained + { + std::os::windows::fs::symlink_file(new_link, &path)?; + } + #[cfg(target_family = "unix")] + { + std::os::unix::fs::symlink(new_link, &path)?; + } + }else { + log::info!("Found already-absolute symlink {} -> {}", path.display(), link_path.display()); + } + } else { + log::info!("Found non-symlink {}: {:?}", dir_entry.path().display(), dir_entry.file_type()?); + } + } + Ok(()) +} + pub fn filename(id: u128, ext: &str) -> String { format!("{}.{}", id, ext) } diff --git a/backend/community_settings_srv/src/main.rs b/backend/community_settings_srv/src/main.rs index 2753bfa..a2dd153 100644 --- a/backend/community_settings_srv/src/main.rs +++ b/backend/community_settings_srv/src/main.rs @@ -29,6 +29,13 @@ async fn main() -> std::io::Result<()> { log::debug!("Building folder layout (if not exists) at: {}", &args.folder.display()); file_util::build_folder_layout(&args.folder)?; + // fix things + if args.fix { + log::debug!("Fixing old symlinks"); + file_util::fix_symlinks(&args.folder)?; + return Ok(()) + } + let leaked_args: &'static cli::Cli = Box::leak::<'static>(Box::new(args)); HttpServer::new(move || { App::new() From be60be71ac28c76d59fcee6e85a8b246ecf235e5 Mon Sep 17 00:00:00 2001 From: "NGnius (Graham)" Date: Fri, 12 Apr 2024 21:35:10 -0400 Subject: [PATCH 2/7] Fix symlink creation again... fix reading file creation->access time --- .../src/api/get_game.rs | 7 ++-- .../src/api/save_setting.rs | 33 ++++++++++++------- backend/community_settings_srv/src/cli.rs | 2 +- .../community_settings_srv/src/file_util.rs | 5 +-- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/backend/community_settings_srv/src/api/get_game.rs b/backend/community_settings_srv/src/api/get_game.rs index cbfb95f..0b8a650 100644 --- a/backend/community_settings_srv/src/api/get_game.rs +++ b/backend/community_settings_srv/src/api/get_game.rs @@ -62,9 +62,10 @@ fn get_some_settings_by_app_id(steam_app_id: u32, cli: &'static Cli) -> std::io: .filter_map(|res| res.ok()) .filter(|f| f.path().extension().map(|ext| ext == file_util::RON_EXTENSION).unwrap_or(false)) .filter_map(|f| f.metadata().ok().map(|meta| (f, meta))) - .filter_map(|(f, meta)| meta.created().ok().map(|time| (f, meta, time))) + .filter_map(|(f, meta)| meta.modified().ok().map(|time| (f, meta, time))) .collect(); files.sort_by(|(_, _, a_created), (_, _, b_created)| a_created.cmp(b_created)); + let files_len = files.len(); let mut results = Vec::with_capacity(MAX_RESULTS); for (_, (f, _, _)) in files.into_iter().enumerate().take_while(|(i, _)| *i < MAX_RESULTS) { @@ -72,12 +73,14 @@ fn get_some_settings_by_app_id(steam_app_id: u32, cli: &'static Cli) -> std::io: let setting = match ron::de::from_reader(reader) { Ok(x) => x, Err(e) => { + log::debug!("Error while reading {}: {}", f.path().display(), e); let e_msg = format!("{}", e); return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); } }; results.push(setting); } + log::debug!("Got {} results (from {} files) for {}", results.len(), files_len, app_id_folder.display()); Ok(results) } @@ -89,7 +92,7 @@ pub async fn get_setting_by_app_id_handler( ) -> std::io::Result { let id: u32 = *id; #[cfg(debug_assertions)] - println!("Accept: {}", accept.to_string()); + log::debug!("Accept: {}", accept.to_string()); let preferred = accept.preference(); if super::is_mime_type_ron_capable(&preferred) { // Send RON diff --git a/backend/community_settings_srv/src/api/save_setting.rs b/backend/community_settings_srv/src/api/save_setting.rs index ae41e3e..1b2bdca 100644 --- a/backend/community_settings_srv/src/api/save_setting.rs +++ b/backend/community_settings_srv/src/api/save_setting.rs @@ -62,24 +62,29 @@ pub async fn save_setting_handler( } } + log::debug!("Saved to {}, building symlinks", path_ron.display()); + // create symlinks for other ways of looking up these settings files let filename_ron = file_util::filename(next_id, file_util::RON_EXTENSION); let filename_json = file_util::filename(next_id, file_util::JSON_EXTENSION); // create symlinks to app id folder let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, parsed_data.steam_app_id); + log::debug!("App id folder {}", app_id_folder.display()); if !app_id_folder.exists() { std::fs::create_dir(&app_id_folder)?; } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { - std::os::windows::fs::symlink_file(&path_ron, app_id_folder.join(&filename_ron).canonicalize()?)?; - std::os::windows::fs::symlink_file(&path_json, app_id_folder.join(&filename_json).canonicalize()?)?; + log::debug!("Symlinking {} -> {}", app_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); + std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, app_id_folder.join(&filename_ron))?; + std::os::windows::fs::symlink_file(&path_json.canonicalize()?, app_id_folder.join(&filename_json))?; } #[cfg(target_family = "unix")] { - std::os::unix::fs::symlink(&path_ron, app_id_folder.join(&filename_ron).canonicalize()?)?; - std::os::unix::fs::symlink(&path_json, app_id_folder.join(&filename_json).canonicalize()?)?; + log::debug!("Symlinking {} -> {}", app_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); + std::os::unix::fs::symlink(&path_ron.canonicalize()?, app_id_folder.join(&filename_ron))?; + std::os::unix::fs::symlink(&path_json.canonicalize()?, app_id_folder.join(&filename_json))?; } // create symlinks for user id folder @@ -89,13 +94,15 @@ pub async fn save_setting_handler( } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { - std::os::windows::fs::symlink_file(&path_ron, user_id_folder.join(&filename_ron).canonicalize()?)?; - std::os::windows::fs::symlink_file(&path_json, user_id_folder.join(&filename_json).canonicalize()?)?; + log::debug!("Symlinking {} -> {}", user_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); + std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, user_id_folder.join(&filename_ron))?; + std::os::windows::fs::symlink_file(&path_json.canonicalize()?, user_id_folder.join(&filename_json))?; } #[cfg(target_family = "unix")] { - std::os::unix::fs::symlink(&path_ron, user_id_folder.join(&filename_ron).canonicalize()?)?; - std::os::unix::fs::symlink(&path_json, user_id_folder.join(&filename_json).canonicalize()?)?; + log::debug!("Symlinking {} -> {}", user_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); + std::os::unix::fs::symlink(&path_ron.canonicalize()?, user_id_folder.join(&filename_ron))?; + std::os::unix::fs::symlink(&path_json.canonicalize()?, user_id_folder.join(&filename_json))?; } // create symlinks for each tag @@ -106,13 +113,15 @@ pub async fn save_setting_handler( } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { - std::os::windows::fs::symlink_file(&path_ron, tag_folder.join(&filename_ron).canonicalize()?)?; - std::os::windows::fs::symlink_file(&path_json, tag_folder.join(&filename_json).canonicalize()?)?; + log::debug!("Symlinking {} -> {}", tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); + std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, tag_folder.join(&filename_ron))?; + std::os::windows::fs::symlink_file(&path_json.canonicalize()?, tag_folder.join(&filename_json))?; } #[cfg(target_family = "unix")] { - std::os::unix::fs::symlink(&path_ron, tag_folder.join(&filename_ron).canonicalize()?)?; - std::os::unix::fs::symlink(&path_json, tag_folder.join(&filename_json).canonicalize()?)?; + log::debug!("Symlinking {} -> {}", tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); + std::os::unix::fs::symlink(&path_ron.canonicalize()?, tag_folder.join(&filename_ron))?; + std::os::unix::fs::symlink(&path_json.canonicalize()?, tag_folder.join(&filename_json))?; } } diff --git a/backend/community_settings_srv/src/cli.rs b/backend/community_settings_srv/src/cli.rs index 5a05e14..b6feb6b 100644 --- a/backend/community_settings_srv/src/cli.rs +++ b/backend/community_settings_srv/src/cli.rs @@ -16,7 +16,7 @@ pub struct Cli { pub log: std::path::PathBuf, /// Perform maintenance tasks - #[arg(short, long)] + #[arg(long)] pub fix: bool, } diff --git a/backend/community_settings_srv/src/file_util.rs b/backend/community_settings_srv/src/file_util.rs index 1272a0c..3050739 100644 --- a/backend/community_settings_srv/src/file_util.rs +++ b/backend/community_settings_srv/src/file_util.rs @@ -138,13 +138,14 @@ pub fn next_setting_id(root: impl AsRef) -> u128 { let mut last_id = *lock; if last_id == 0 { // needs init + last_id = 1; let mut path = setting_path_by_id(root.as_ref(), last_id, RON_EXTENSION); while path.exists() { last_id += 1; path = setting_path_by_id(root.as_ref(), last_id, RON_EXTENSION); } - *lock = last_id; - println!("setting id initialized to {}", last_id); + *lock = last_id - 1; + log::info!("setting id initialized to {}", last_id); } *lock += 1; *lock From 9d452acc28f3ca329912b5479f78e28683f6ce5c Mon Sep 17 00:00:00 2001 From: "NGnius (Graham)" Date: Fri, 12 Apr 2024 21:55:29 -0400 Subject: [PATCH 3/7] Fix ID population --- backend/community_settings_srv/Cargo.lock | 2 +- backend/community_settings_srv/Cargo.toml | 2 +- .../src/api/save_setting.rs | 4 +- .../community_settings_srv/src/file_util.rs | 40 +++++++++++++++++++ backend/community_settings_srv/src/main.rs | 4 +- 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/backend/community_settings_srv/Cargo.lock b/backend/community_settings_srv/Cargo.lock index 1cf7fd3..af16b79 100644 --- a/backend/community_settings_srv/Cargo.lock +++ b/backend/community_settings_srv/Cargo.lock @@ -438,7 +438,7 @@ dependencies = [ [[package]] name = "community_settings_srv" -version = "0.1.0" +version = "0.1.1" dependencies = [ "actix-web", "clap", diff --git a/backend/community_settings_srv/Cargo.toml b/backend/community_settings_srv/Cargo.toml index 37cb95c..01baf76 100644 --- a/backend/community_settings_srv/Cargo.toml +++ b/backend/community_settings_srv/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "community_settings_srv" -version = "0.1.0" +version = "0.1.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/backend/community_settings_srv/src/api/save_setting.rs b/backend/community_settings_srv/src/api/save_setting.rs index 1b2bdca..6b3d96e 100644 --- a/backend/community_settings_srv/src/api/save_setting.rs +++ b/backend/community_settings_srv/src/api/save_setting.rs @@ -18,7 +18,7 @@ pub async fn save_setting_handler( Err(_e) => return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "too many bytes in payload")), }; let next_id = file_util::next_setting_id(&cli.folder); - let parsed_data: community_settings_core::v1::Metadata = if super::is_mime_type_ron_capable(&content_type) { + let mut parsed_data: community_settings_core::v1::Metadata = if super::is_mime_type_ron_capable(&content_type) { // Parse as RON match ron::de::from_reader(bytes.as_ref()) { Ok(x) => x, @@ -42,6 +42,8 @@ pub async fn save_setting_handler( } } }; + // Override the ID with the one used by this server + parsed_data.id = next_id.to_string(); // TODO validate user and app id // Reject blocked users and apps let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, file_util::RON_EXTENSION); diff --git a/backend/community_settings_srv/src/file_util.rs b/backend/community_settings_srv/src/file_util.rs index 3050739..627a9fc 100644 --- a/backend/community_settings_srv/src/file_util.rs +++ b/backend/community_settings_srv/src/file_util.rs @@ -101,6 +101,46 @@ fn make_symlinks_absolute_in_dir(root: impl AsRef, dir: impl AsRef) Ok(()) } +pub fn sync_ids(root: impl AsRef) -> std::io::Result<()> { + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + let f_path = dir_entry.path(); + if let Some(ext) = f_path.extension() { + let id = f_path.file_stem().map(|os| os.to_string_lossy().to_string()).unwrap(); + if ext == RON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + let mut setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) { + Ok(x) => x, + Err(e) => { + log::debug!("Error while reading {}: {}", f_path.display(), e); + let e_msg = format!("{}", e); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); + } + }; + setting.id = id; + ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + } else if ext == JSON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) { + Ok(x) => x, + Err(e) => { + log::debug!("Error while reading {}: {}", f_path.display(), e); + let e_msg = format!("{}", e); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); + } + }; + setting.id = id; + serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + } + } + + } + Ok(()) +} + pub fn filename(id: u128, ext: &str) -> String { format!("{}.{}", id, ext) } diff --git a/backend/community_settings_srv/src/main.rs b/backend/community_settings_srv/src/main.rs index a2dd153..0f4e40c 100644 --- a/backend/community_settings_srv/src/main.rs +++ b/backend/community_settings_srv/src/main.rs @@ -31,8 +31,10 @@ async fn main() -> std::io::Result<()> { // fix things if args.fix { - log::debug!("Fixing old symlinks"); + log::info!("Fixing old symlinks"); file_util::fix_symlinks(&args.folder)?; + log::info!("Resynchronizing file IDs with file name IDs"); + file_util::sync_ids(&args.folder)?; return Ok(()) } From 81b9107bf57a47da14e0d1041167909be4d22c9c Mon Sep 17 00:00:00 2001 From: "NGnius (Graham)" Date: Sat, 13 Apr 2024 10:14:02 -0400 Subject: [PATCH 4/7] Add some double-criteria symlinks (by app id and tag, by user id and tag) --- .../src/api/save_setting.rs | 48 +++++++++++++++++++ .../community_settings_srv/src/file_util.rs | 30 ++++++++++-- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/backend/community_settings_srv/src/api/save_setting.rs b/backend/community_settings_srv/src/api/save_setting.rs index 6b3d96e..bd4960e 100644 --- a/backend/community_settings_srv/src/api/save_setting.rs +++ b/backend/community_settings_srv/src/api/save_setting.rs @@ -109,6 +109,10 @@ pub async fn save_setting_handler( // create symlinks for each tag for tag in parsed_data.tags.iter() { + if !str_is_alphanumeric_or_space(&tag){ + continue; + } + // create symlinks for general tag folder let tag_folder = file_util::setting_folder_by_tag(&cli.folder, tag); if !tag_folder.exists() { std::fs::create_dir(&tag_folder)?; @@ -125,7 +129,51 @@ pub async fn save_setting_handler( std::os::unix::fs::symlink(&path_ron.canonicalize()?, tag_folder.join(&filename_ron))?; std::os::unix::fs::symlink(&path_json.canonicalize()?, tag_folder.join(&filename_json))?; } + + // create symlinks for app id tag folder + let app_tag_folder = file_util::setting_folder_by_app_id_tag(&cli.folder, parsed_data.steam_app_id, tag); + if !app_tag_folder.exists() { + std::fs::create_dir_all(&app_tag_folder)?; + } + #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained + { + log::debug!("Symlinking {} -> {}", app_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); + std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, app_tag_folder.join(&filename_ron))?; + std::os::windows::fs::symlink_file(&path_json.canonicalize()?, app_tag_folder.join(&filename_json))?; + } + #[cfg(target_family = "unix")] + { + log::debug!("Symlinking {} -> {}", app_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); + std::os::unix::fs::symlink(&path_ron.canonicalize()?, app_tag_folder.join(&filename_ron))?; + std::os::unix::fs::symlink(&path_json.canonicalize()?, app_tag_folder.join(&filename_json))?; + } + + // create symlinks for user id tag folder + let user_tag_folder = file_util::setting_folder_by_user_id_tag(&cli.folder, parsed_data.steam_user_id, tag); + if !user_tag_folder.exists() { + std::fs::create_dir_all(&user_tag_folder)?; + } + #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained + { + log::debug!("Symlinking {} -> {}", user_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); + std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, user_tag_folder.join(&filename_ron))?; + std::os::windows::fs::symlink_file(&path_json.canonicalize()?, user_tag_folder.join(&filename_json))?; + } + #[cfg(target_family = "unix")] + { + log::debug!("Symlinking {} -> {}", user_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); + std::os::unix::fs::symlink(&path_ron.canonicalize()?, user_tag_folder.join(&filename_ron))?; + std::os::unix::fs::symlink(&path_json.canonicalize()?, user_tag_folder.join(&filename_json))?; + } } Ok(actix_web::HttpResponse::NoContent()) } + +fn str_is_alphanumeric_or_space(s: &str) -> bool { + let mut result = true; + for ch in s.chars() { + result &= ch.is_ascii_alphanumeric() || ch == ' '; + } + result +} diff --git a/backend/community_settings_srv/src/file_util.rs b/backend/community_settings_srv/src/file_util.rs index 627a9fc..76f511b 100644 --- a/backend/community_settings_srv/src/file_util.rs +++ b/backend/community_settings_srv/src/file_util.rs @@ -120,8 +120,10 @@ pub fn sync_ids(root: impl AsRef) -> std::io::Result<()> { return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); } }; - setting.id = id; - ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + if setting.id != id { + setting.id = id; + ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + } } else if ext == JSON_EXTENSION { let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) { @@ -132,8 +134,10 @@ pub fn sync_ids(root: impl AsRef) -> std::io::Result<()> { return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); } }; - setting.id = id; - serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + if setting.id != id { + setting.id = id; + serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + } } } @@ -159,6 +163,15 @@ pub fn setting_folder_by_app_id(root: impl AsRef, steam_app_id: u32) -> Pa .join(steam_app_id.to_string()) } +pub fn setting_folder_by_app_id_tag(root: impl AsRef, steam_app_id: u32, tag: &str) -> PathBuf { + root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .join(steam_app_id.to_string()) + .join(TAG_FOLDER) + .join(tag) +} + pub fn setting_folder_by_user_id(root: impl AsRef, steam_user_id: u64) -> PathBuf { root.as_ref() .join(SETTING_FOLDER) @@ -166,6 +179,15 @@ pub fn setting_folder_by_user_id(root: impl AsRef, steam_user_id: u64) -> .join(steam_user_id.to_string()) } +pub fn setting_folder_by_user_id_tag(root: impl AsRef, steam_user_id: u64, tag: &str) -> PathBuf { + root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .join(steam_user_id.to_string()) + .join(TAG_FOLDER) + .join(tag) +} + pub fn setting_folder_by_tag(root: impl AsRef, tag: &str) -> PathBuf { root.as_ref() .join(SETTING_FOLDER) From c1064fdf9207a8552e1bb2129626a82b090d1416 Mon Sep 17 00:00:00 2001 From: "NGnius (Graham)" Date: Fri, 19 Apr 2024 22:18:06 -0400 Subject: [PATCH 5/7] Move filesystem fixes and improvements to separate file --- .../src/api/get_game.rs | 2 +- .../src/api/get_setting.rs | 4 +- .../src/api/save_setting.rs | 14 +- backend/community_settings_srv/src/consts.rs | 8 + .../community_settings_srv/src/file_util.rs | 158 ++--------------- backend/community_settings_srv/src/main.rs | 10 +- backend/community_settings_srv/src/upgrade.rs | 164 ++++++++++++++++++ 7 files changed, 207 insertions(+), 153 deletions(-) create mode 100644 backend/community_settings_srv/src/consts.rs create mode 100644 backend/community_settings_srv/src/upgrade.rs diff --git a/backend/community_settings_srv/src/api/get_game.rs b/backend/community_settings_srv/src/api/get_game.rs index 0b8a650..8e42556 100644 --- a/backend/community_settings_srv/src/api/get_game.rs +++ b/backend/community_settings_srv/src/api/get_game.rs @@ -60,7 +60,7 @@ fn get_some_settings_by_app_id(steam_app_id: u32, cli: &'static Cli) -> std::io: let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, steam_app_id); let mut files: Vec<_> = app_id_folder.read_dir()? .filter_map(|res| res.ok()) - .filter(|f| f.path().extension().map(|ext| ext == file_util::RON_EXTENSION).unwrap_or(false)) + .filter(|f| f.path().extension().map(|ext| ext == crate::consts::RON_EXTENSION).unwrap_or(false)) .filter_map(|f| f.metadata().ok().map(|meta| (f, meta))) .filter_map(|(f, meta)| meta.modified().ok().map(|time| (f, meta, time))) .collect(); diff --git a/backend/community_settings_srv/src/api/get_setting.rs b/backend/community_settings_srv/src/api/get_setting.rs index 3121192..9d3a05b 100644 --- a/backend/community_settings_srv/src/api/get_setting.rs +++ b/backend/community_settings_srv/src/api/get_setting.rs @@ -67,7 +67,7 @@ pub async fn get_setting_handler( if super::is_mime_type_ron_capable(&preferred) { // Send RON let ron = if id != 0 { - let path = file_util::setting_path_by_id(&cli.folder, id, file_util::RON_EXTENSION); + let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::RON_EXTENSION); if !path.exists() { return Err(std::io::Error::new(std::io::ErrorKind::NotFound, format!("setting id {} does not exist", id))); } @@ -93,7 +93,7 @@ pub async fn get_setting_handler( } else { // Send JSON (fallback) let json = if id != 0 { - let path = file_util::setting_path_by_id(&cli.folder, id, file_util::JSON_EXTENSION); + let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::JSON_EXTENSION); // TODO? cache this instead of always loading it from file let reader = std::io::BufReader::new(std::fs::File::open(path)?); match serde_json::from_reader(reader) { diff --git a/backend/community_settings_srv/src/api/save_setting.rs b/backend/community_settings_srv/src/api/save_setting.rs index bd4960e..a77ba58 100644 --- a/backend/community_settings_srv/src/api/save_setting.rs +++ b/backend/community_settings_srv/src/api/save_setting.rs @@ -46,14 +46,14 @@ pub async fn save_setting_handler( parsed_data.id = next_id.to_string(); // TODO validate user and app id // Reject blocked users and apps - let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, file_util::RON_EXTENSION); + let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::RON_EXTENSION); let writer = std::io::BufWriter::new(std::fs::File::create(&path_ron)?); if let Err(e) = ron::ser::to_writer(writer, &parsed_data) { let e_msg = format!("{}", e); return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); } - let path_json = file_util::setting_path_by_id(&cli.folder, next_id, file_util::JSON_EXTENSION); + let path_json = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::JSON_EXTENSION); let writer = std::io::BufWriter::new(std::fs::File::create(&path_json)?); if let Err(e) = serde_json::to_writer(writer, &parsed_data) { let e_msg = format!("{}", e); @@ -67,14 +67,15 @@ pub async fn save_setting_handler( log::debug!("Saved to {}, building symlinks", path_ron.display()); // create symlinks for other ways of looking up these settings files - let filename_ron = file_util::filename(next_id, file_util::RON_EXTENSION); - let filename_json = file_util::filename(next_id, file_util::JSON_EXTENSION); + let filename_ron = file_util::filename(next_id, crate::consts::RON_EXTENSION); + let filename_json = file_util::filename(next_id, crate::consts::JSON_EXTENSION); // create symlinks to app id folder let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, parsed_data.steam_app_id); log::debug!("App id folder {}", app_id_folder.display()); if !app_id_folder.exists() { std::fs::create_dir(&app_id_folder)?; + std::fs::create_dir(file_util::setting_tag_folder_by_app_id(&cli.folder, parsed_data.steam_app_id))?; } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { @@ -93,6 +94,7 @@ pub async fn save_setting_handler( let user_id_folder = file_util::setting_folder_by_user_id(&cli.folder, parsed_data.steam_user_id); if !user_id_folder.exists() { std::fs::create_dir(&user_id_folder)?; + std::fs::create_dir(file_util::setting_tag_folder_by_user_id(&cli.folder, parsed_data.steam_user_id))?; } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { @@ -133,7 +135,7 @@ pub async fn save_setting_handler( // create symlinks for app id tag folder let app_tag_folder = file_util::setting_folder_by_app_id_tag(&cli.folder, parsed_data.steam_app_id, tag); if !app_tag_folder.exists() { - std::fs::create_dir_all(&app_tag_folder)?; + std::fs::create_dir(&app_tag_folder)?; } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { @@ -151,7 +153,7 @@ pub async fn save_setting_handler( // create symlinks for user id tag folder let user_tag_folder = file_util::setting_folder_by_user_id_tag(&cli.folder, parsed_data.steam_user_id, tag); if !user_tag_folder.exists() { - std::fs::create_dir_all(&user_tag_folder)?; + std::fs::create_dir(&user_tag_folder)?; } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { diff --git a/backend/community_settings_srv/src/consts.rs b/backend/community_settings_srv/src/consts.rs new file mode 100644 index 0000000..833fd90 --- /dev/null +++ b/backend/community_settings_srv/src/consts.rs @@ -0,0 +1,8 @@ +pub const RON_EXTENSION: &'static str = "ron"; +pub const JSON_EXTENSION: &'static str = "json"; + +pub const SETTING_FOLDER: &'static str = "settings"; +pub const ID_FOLDER: &'static str = "by_id"; +pub const APP_ID_FOLDER: &'static str = "by_app_id"; +pub const USER_ID_FOLDER: &'static str = "by_user_id"; +pub const TAG_FOLDER: &'static str = "by_tag"; diff --git a/backend/community_settings_srv/src/file_util.rs b/backend/community_settings_srv/src/file_util.rs index 76f511b..5a7d4bd 100644 --- a/backend/community_settings_srv/src/file_util.rs +++ b/backend/community_settings_srv/src/file_util.rs @@ -1,150 +1,10 @@ use std::path::{Path, PathBuf}; use std::sync::Mutex; -pub const RON_EXTENSION: &'static str = "ron"; -pub const JSON_EXTENSION: &'static str = "json"; - -const SETTING_FOLDER: &'static str = "settings"; -const ID_FOLDER: &'static str = "by_id"; -const APP_ID_FOLDER: &'static str = "by_app_id"; -const USER_ID_FOLDER: &'static str = "by_user_id"; -const TAG_FOLDER: &'static str = "by_tag"; +use crate::consts::*; static LAST_SETTING_ID: Mutex = Mutex::new(0); -pub fn build_folder_layout(root: impl AsRef) -> std::io::Result<()> { - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(ID_FOLDER) - )?; - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(APP_ID_FOLDER) - )?; - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(USER_ID_FOLDER) - )?; - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(TAG_FOLDER) - )?; - Ok(()) -} - -pub fn fix_symlinks(root: impl AsRef) -> std::io::Result<()> { - log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display()); - for dir_entry in root.as_ref() - .join(SETTING_FOLDER) - .join(APP_ID_FOLDER) - .read_dir()? { - let dir_entry = dir_entry?; - if dir_entry.file_type()?.is_dir() { - make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; - } - } - for dir_entry in root.as_ref() - .join(SETTING_FOLDER) - .join(USER_ID_FOLDER) - .read_dir()? { - let dir_entry = dir_entry?; - if dir_entry.file_type()?.is_dir() { - make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; - } - } - for dir_entry in root.as_ref() - .join(SETTING_FOLDER) - .join(TAG_FOLDER).read_dir()? { - let dir_entry = dir_entry?; - if dir_entry.file_type()?.is_dir() { - make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; - } - } - - Ok(()) -} - -fn make_symlinks_absolute_in_dir(root: impl AsRef, dir: impl AsRef) -> std::io::Result<()> { - let abs_root = root.as_ref().canonicalize()?; - assert!(abs_root.is_absolute()); - for dir_entry in dir.as_ref() - .read_dir()? { - let dir_entry = dir_entry?; - if dir_entry.file_type()?.is_symlink() { - let path = dir_entry.path(); - let link_path = path.read_link()?; - if !link_path.is_absolute() { - let new_link = abs_root.join( - link_path.strip_prefix(&root).expect("Symlinked path does not begin with root settings folder") - ); - log::info!("Fixing {} -> {} to -> {}", path.display(), link_path.display(), new_link.display()); - std::fs::remove_file(&path)?; - #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained - { - std::os::windows::fs::symlink_file(new_link, &path)?; - } - #[cfg(target_family = "unix")] - { - std::os::unix::fs::symlink(new_link, &path)?; - } - }else { - log::info!("Found already-absolute symlink {} -> {}", path.display(), link_path.display()); - } - } else { - log::info!("Found non-symlink {}: {:?}", dir_entry.path().display(), dir_entry.file_type()?); - } - } - Ok(()) -} - -pub fn sync_ids(root: impl AsRef) -> std::io::Result<()> { - for dir_entry in root.as_ref() - .join(SETTING_FOLDER) - .join(ID_FOLDER) - .read_dir()? { - let dir_entry = dir_entry?; - let f_path = dir_entry.path(); - if let Some(ext) = f_path.extension() { - let id = f_path.file_stem().map(|os| os.to_string_lossy().to_string()).unwrap(); - if ext == RON_EXTENSION { - let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); - let mut setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) { - Ok(x) => x, - Err(e) => { - log::debug!("Error while reading {}: {}", f_path.display(), e); - let e_msg = format!("{}", e); - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); - } - }; - if setting.id != id { - setting.id = id; - ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); - } - } else if ext == JSON_EXTENSION { - let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); - let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) { - Ok(x) => x, - Err(e) => { - log::debug!("Error while reading {}: {}", f_path.display(), e); - let e_msg = format!("{}", e); - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); - } - }; - if setting.id != id { - setting.id = id; - serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); - } - } - } - - } - Ok(()) -} - pub fn filename(id: u128, ext: &str) -> String { format!("{}.{}", id, ext) } @@ -163,6 +23,14 @@ pub fn setting_folder_by_app_id(root: impl AsRef, steam_app_id: u32) -> Pa .join(steam_app_id.to_string()) } +pub fn setting_tag_folder_by_app_id(root: impl AsRef, steam_app_id: u32) -> PathBuf { + root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .join(steam_app_id.to_string()) + .join(TAG_FOLDER) +} + pub fn setting_folder_by_app_id_tag(root: impl AsRef, steam_app_id: u32, tag: &str) -> PathBuf { root.as_ref() .join(SETTING_FOLDER) @@ -179,6 +47,14 @@ pub fn setting_folder_by_user_id(root: impl AsRef, steam_user_id: u64) -> .join(steam_user_id.to_string()) } +pub fn setting_tag_folder_by_user_id(root: impl AsRef, steam_user_id: u64) -> PathBuf { + root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .join(steam_user_id.to_string()) + .join(TAG_FOLDER) +} + pub fn setting_folder_by_user_id_tag(root: impl AsRef, steam_user_id: u64, tag: &str) -> PathBuf { root.as_ref() .join(SETTING_FOLDER) diff --git a/backend/community_settings_srv/src/main.rs b/backend/community_settings_srv/src/main.rs index 0f4e40c..58f132a 100644 --- a/backend/community_settings_srv/src/main.rs +++ b/backend/community_settings_srv/src/main.rs @@ -1,6 +1,8 @@ mod api; mod cli; +mod consts; mod file_util; +mod upgrade; use actix_web::{web, App, HttpServer}; @@ -27,14 +29,16 @@ async fn main() -> std::io::Result<()> { // setup log::debug!("Building folder layout (if not exists) at: {}", &args.folder.display()); - file_util::build_folder_layout(&args.folder)?; + upgrade::build_folder_layout(&args.folder)?; // fix things if args.fix { log::info!("Fixing old symlinks"); - file_util::fix_symlinks(&args.folder)?; + upgrade::fix_symlinks(&args.folder)?; + log::info!("Creating missing by_tag folders"); + upgrade::make_tag_subfolders(&args.folder)?; log::info!("Resynchronizing file IDs with file name IDs"); - file_util::sync_ids(&args.folder)?; + upgrade::sync_ids(&args.folder)?; return Ok(()) } diff --git a/backend/community_settings_srv/src/upgrade.rs b/backend/community_settings_srv/src/upgrade.rs new file mode 100644 index 0000000..c48f9b3 --- /dev/null +++ b/backend/community_settings_srv/src/upgrade.rs @@ -0,0 +1,164 @@ +use std::path::Path; + +use crate::consts::*; + +pub fn build_folder_layout(root: impl AsRef) -> std::io::Result<()> { + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(ID_FOLDER) + )?; + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + )?; + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + )?; + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(TAG_FOLDER) + )?; + Ok(()) +} + +pub fn fix_symlinks(root: impl AsRef) -> std::io::Result<()> { + log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display()); + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(TAG_FOLDER).read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + + Ok(()) +} + +fn make_symlinks_absolute_in_dir(root: impl AsRef, dir: impl AsRef) -> std::io::Result<()> { + let abs_root = root.as_ref().canonicalize()?; + assert!(abs_root.is_absolute()); + for dir_entry in dir.as_ref() + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_symlink() { + let path = dir_entry.path(); + let link_path = path.read_link()?; + if !link_path.is_absolute() { + let new_link = abs_root.join( + link_path.strip_prefix(&root).expect("Symlinked path does not begin with root settings folder") + ); + log::info!("Fixing {} -> {} to -> {}", path.display(), link_path.display(), new_link.display()); + std::fs::remove_file(&path)?; + #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained + { + std::os::windows::fs::symlink_file(new_link, &path)?; + } + #[cfg(target_family = "unix")] + { + std::os::unix::fs::symlink(new_link, &path)?; + } + }else { + log::info!("Found already-absolute symlink {} -> {}", path.display(), link_path.display()); + } + } else { + log::info!("Found non-symlink {}: {:?}", dir_entry.path().display(), dir_entry.file_type()?); + } + } + Ok(()) +} + +pub fn sync_ids(root: impl AsRef) -> std::io::Result<()> { + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + let f_path = dir_entry.path(); + if let Some(ext) = f_path.extension() { + let id = f_path.file_stem().map(|os| os.to_string_lossy().to_string()).unwrap(); + if ext == RON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + let mut setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) { + Ok(x) => x, + Err(e) => { + log::debug!("Error while reading {}: {}", f_path.display(), e); + let e_msg = format!("{}", e); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); + } + }; + if setting.id != id { + setting.id = id; + ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + } + } else if ext == JSON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) { + Ok(x) => x, + Err(e) => { + log::debug!("Error while reading {}: {}", f_path.display(), e); + let e_msg = format!("{}", e); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); + } + }; + if setting.id != id { + setting.id = id; + serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + } + } + } + } + Ok(()) +} + +pub fn make_tag_subfolders(root: impl AsRef) -> std::io::Result<()> { + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.metadata()?.is_dir() { + let tag_folder = dir_entry.path().join(TAG_FOLDER); + if !tag_folder.exists() { + std::fs::create_dir(&tag_folder)?; + } + } + } + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.metadata()?.is_dir() { + let tag_folder = dir_entry.path().join(TAG_FOLDER); + if !tag_folder.exists() { + std::fs::create_dir(&tag_folder)?; + } + } + } + // TODO populate folders + Ok(()) +} From 6c28cef9b377f7e70dcb07c28e9e8a27dbb46d55 Mon Sep 17 00:00:00 2001 From: "NGnius (Graham)" Date: Fri, 19 Apr 2024 22:18:20 -0400 Subject: [PATCH 6/7] Fix store scrolling --- backend/Cargo.lock | 2 +- backend/Cargo.toml | 2 +- backend/build.sh | 8 +++---- backend/src/settings/steam_deck/util.rs | 2 +- package.json | 2 +- src/store/page.tsx | 31 ++++++++++++++----------- 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d76f0c1..a9c0d7f 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1170,7 +1170,7 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "powertools" -version = "2.0.0-beta2" +version = "2.0.1-beta1" dependencies = [ "async-trait", "chrono", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5d5e6a9..01f4f42 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "powertools" -version = "2.0.0-beta2" +version = "2.0.1-beta1" edition = "2021" authors = ["NGnius (Graham) "] description = "Backend (superuser) functionality for PowerTools" diff --git a/backend/build.sh b/backend/build.sh index 689ae88..416eeb7 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -3,11 +3,11 @@ #cargo build --release --target x86_64-unknown-linux-musl #cargo build --target x86_64-unknown-linux-musl #cross build -cargo build --release -#cargo build +#cargo build --release +cargo build mkdir -p ../bin #cp --preserve=mode ./target/x86_64-unknown-linux-musl/release/powertools ../bin/backend #cp --preserve=mode ./target/x86_64-unknown-linux-musl/debug/powertools ../bin/backend -cp --preserve=mode ./target/release/powertools ../bin/backend -#cp --preserve=mode ./target/debug/powertools ../bin/backend +#cp --preserve=mode ./target/release/powertools ../bin/backend +cp --preserve=mode ./target/debug/powertools ../bin/backend diff --git a/backend/src/settings/steam_deck/util.rs b/backend/src/settings/steam_deck/util.rs index 88d940c..3f02be1 100644 --- a/backend/src/settings/steam_deck/util.rs +++ b/backend/src/settings/steam_deck/util.rs @@ -31,7 +31,7 @@ const THINGS: &[u8] = &[ 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, ]; -const TIME_UNIT: std::time::Duration = std::time::Duration::from_millis(200); +const TIME_UNIT: std::time::Duration = std::time::Duration::from_millis(250); pub fn flash_led() { use smokepatio::ec::ControllerSet; diff --git a/package.json b/package.json index be7d9cd..76e72f6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "PowerTools", - "version": "2.0.0-beta2", + "version": "2.0.1-beta1", "description": "Power tweaks for power users", "scripts": { "build": "shx rm -rf dist && rollup -c", diff --git a/src/store/page.tsx b/src/store/page.tsx index da8f36d..a9badb8 100644 --- a/src/store/page.tsx +++ b/src/store/page.tsx @@ -40,18 +40,22 @@ export class StoreResultsPage extends Component<{onNewVariant: () => void}> { { tr("No results") /* TODO translate */ } ); } else { - // TODO - return ( + return ( +
+ { storeItems.map((meta: backend.StoreMetadata) => ( void}> { )) } - ); + +
); } } else { From 22e916fe1222c9d5297a8f2b0af9e1628a2535c1 Mon Sep 17 00:00:00 2001 From: "NGnius (Graham)" Date: Sun, 21 Apr 2024 12:34:11 -0400 Subject: [PATCH 7/7] Add some deduplication and another symlink fixer --- backend/community_settings_srv/Cargo.lock | 20 +++ backend/community_settings_srv/Cargo.toml | 2 + .../src/api/save_setting.rs | 108 ++--------------- backend/community_settings_srv/src/cli.rs | 4 + .../community_settings_srv/src/file_util.rs | 68 +++++++++++ backend/community_settings_srv/src/main.rs | 5 + .../community_settings_srv/src/tasks/mod.rs | 16 +++ .../src/tasks/symlink_cleanup.rs | 39 ++++++ .../src/tasks/task_runner.rs | 47 ++++++++ .../src/tasks/user_antispam.rs | 114 ++++++++++++++++++ backend/community_settings_srv/src/upgrade.rs | 53 ++++++++ 11 files changed, 380 insertions(+), 96 deletions(-) create mode 100644 backend/community_settings_srv/src/tasks/mod.rs create mode 100644 backend/community_settings_srv/src/tasks/symlink_cleanup.rs create mode 100644 backend/community_settings_srv/src/tasks/task_runner.rs create mode 100644 backend/community_settings_srv/src/tasks/user_antispam.rs diff --git a/backend/community_settings_srv/Cargo.lock b/backend/community_settings_srv/Cargo.lock index af16b79..3783d84 100644 --- a/backend/community_settings_srv/Cargo.lock +++ b/backend/community_settings_srv/Cargo.lock @@ -450,6 +450,7 @@ dependencies = [ "serde_json", "simplelog", "tokio", + "walkdir", ] [[package]] @@ -1017,6 +1018,15 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1319,6 +1329,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/backend/community_settings_srv/Cargo.toml b/backend/community_settings_srv/Cargo.toml index 01baf76..aee36ff 100644 --- a/backend/community_settings_srv/Cargo.toml +++ b/backend/community_settings_srv/Cargo.toml @@ -16,6 +16,8 @@ tokio = { version = "1", features = ["full"] } actix-web = { version = "4.4" } mime = { version = "0.3.17" } +walkdir = "2" + # logging log = "0.4" simplelog = "0.12" diff --git a/backend/community_settings_srv/src/api/save_setting.rs b/backend/community_settings_srv/src/api/save_setting.rs index a77ba58..6169d06 100644 --- a/backend/community_settings_srv/src/api/save_setting.rs +++ b/backend/community_settings_srv/src/api/save_setting.rs @@ -66,116 +66,32 @@ pub async fn save_setting_handler( log::debug!("Saved to {}, building symlinks", path_ron.display()); - // create symlinks for other ways of looking up these settings files - let filename_ron = file_util::filename(next_id, crate::consts::RON_EXTENSION); - let filename_json = file_util::filename(next_id, crate::consts::JSON_EXTENSION); + let to_symlink = file_util::symlinks(&cli.folder, &parsed_data)?; - // create symlinks to app id folder - let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, parsed_data.steam_app_id); - log::debug!("App id folder {}", app_id_folder.display()); - if !app_id_folder.exists() { - std::fs::create_dir(&app_id_folder)?; - std::fs::create_dir(file_util::setting_tag_folder_by_app_id(&cli.folder, parsed_data.steam_app_id))?; - } - #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained - { - log::debug!("Symlinking {} -> {}", app_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); - std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, app_id_folder.join(&filename_ron))?; - std::os::windows::fs::symlink_file(&path_json.canonicalize()?, app_id_folder.join(&filename_json))?; - } - #[cfg(target_family = "unix")] - { - log::debug!("Symlinking {} -> {}", app_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); - std::os::unix::fs::symlink(&path_ron.canonicalize()?, app_id_folder.join(&filename_ron))?; - std::os::unix::fs::symlink(&path_json.canonicalize()?, app_id_folder.join(&filename_json))?; - } - - // create symlinks for user id folder - let user_id_folder = file_util::setting_folder_by_user_id(&cli.folder, parsed_data.steam_user_id); - if !user_id_folder.exists() { - std::fs::create_dir(&user_id_folder)?; - std::fs::create_dir(file_util::setting_tag_folder_by_user_id(&cli.folder, parsed_data.steam_user_id))?; - } - #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained - { - log::debug!("Symlinking {} -> {}", user_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); - std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, user_id_folder.join(&filename_ron))?; - std::os::windows::fs::symlink_file(&path_json.canonicalize()?, user_id_folder.join(&filename_json))?; - } - #[cfg(target_family = "unix")] - { - log::debug!("Symlinking {} -> {}", user_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); - std::os::unix::fs::symlink(&path_ron.canonicalize()?, user_id_folder.join(&filename_ron))?; - std::os::unix::fs::symlink(&path_json.canonicalize()?, user_id_folder.join(&filename_json))?; - } - - // create symlinks for each tag - for tag in parsed_data.tags.iter() { - if !str_is_alphanumeric_or_space(&tag){ - continue; - } - // create symlinks for general tag folder - let tag_folder = file_util::setting_folder_by_tag(&cli.folder, tag); - if !tag_folder.exists() { - std::fs::create_dir(&tag_folder)?; - } + let path_ron_canon = path_ron.canonicalize()?; + let path_json_canon = path_json.canonicalize()?; + for ron_link in to_symlink.ron { + log::debug!("Symlinking {} -> {}", ron_link.display(), path_ron_canon.display()); #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { - log::debug!("Symlinking {} -> {}", tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); - std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, tag_folder.join(&filename_ron))?; - std::os::windows::fs::symlink_file(&path_json.canonicalize()?, tag_folder.join(&filename_json))?; + std::os::windows::fs::symlink_file(&path_ron_canon, &ron_link)?; } #[cfg(target_family = "unix")] { - log::debug!("Symlinking {} -> {}", tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); - std::os::unix::fs::symlink(&path_ron.canonicalize()?, tag_folder.join(&filename_ron))?; - std::os::unix::fs::symlink(&path_json.canonicalize()?, tag_folder.join(&filename_json))?; - } - - // create symlinks for app id tag folder - let app_tag_folder = file_util::setting_folder_by_app_id_tag(&cli.folder, parsed_data.steam_app_id, tag); - if !app_tag_folder.exists() { - std::fs::create_dir(&app_tag_folder)?; + std::os::unix::fs::symlink(&path_ron_canon, &ron_link)?; } + } + for json_link in to_symlink.json { + //log::debug!("Symlinking {} -> {}", json_link.display(), path_json_canon.display()); #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { - log::debug!("Symlinking {} -> {}", app_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); - std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, app_tag_folder.join(&filename_ron))?; - std::os::windows::fs::symlink_file(&path_json.canonicalize()?, app_tag_folder.join(&filename_json))?; + std::os::windows::fs::symlink_file(&path_json_canon, json_link)?; } #[cfg(target_family = "unix")] { - log::debug!("Symlinking {} -> {}", app_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); - std::os::unix::fs::symlink(&path_ron.canonicalize()?, app_tag_folder.join(&filename_ron))?; - std::os::unix::fs::symlink(&path_json.canonicalize()?, app_tag_folder.join(&filename_json))?; - } - - // create symlinks for user id tag folder - let user_tag_folder = file_util::setting_folder_by_user_id_tag(&cli.folder, parsed_data.steam_user_id, tag); - if !user_tag_folder.exists() { - std::fs::create_dir(&user_tag_folder)?; - } - #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained - { - log::debug!("Symlinking {} -> {}", user_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); - std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, user_tag_folder.join(&filename_ron))?; - std::os::windows::fs::symlink_file(&path_json.canonicalize()?, user_tag_folder.join(&filename_json))?; - } - #[cfg(target_family = "unix")] - { - log::debug!("Symlinking {} -> {}", user_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display()); - std::os::unix::fs::symlink(&path_ron.canonicalize()?, user_tag_folder.join(&filename_ron))?; - std::os::unix::fs::symlink(&path_json.canonicalize()?, user_tag_folder.join(&filename_json))?; + std::os::unix::fs::symlink(&path_json_canon, json_link)?; } } Ok(actix_web::HttpResponse::NoContent()) } - -fn str_is_alphanumeric_or_space(s: &str) -> bool { - let mut result = true; - for ch in s.chars() { - result &= ch.is_ascii_alphanumeric() || ch == ' '; - } - result -} diff --git a/backend/community_settings_srv/src/cli.rs b/backend/community_settings_srv/src/cli.rs index b6feb6b..f960560 100644 --- a/backend/community_settings_srv/src/cli.rs +++ b/backend/community_settings_srv/src/cli.rs @@ -18,6 +18,10 @@ pub struct Cli { /// Perform maintenance tasks #[arg(long)] pub fix: bool, + + /// Keep up to this many duplicate settings (0 will delete everything!!!) + #[arg(long, default_value_t = 2)] + pub duplicates: usize, } impl Cli { diff --git a/backend/community_settings_srv/src/file_util.rs b/backend/community_settings_srv/src/file_util.rs index 5a7d4bd..f776da2 100644 --- a/backend/community_settings_srv/src/file_util.rs +++ b/backend/community_settings_srv/src/file_util.rs @@ -88,3 +88,71 @@ pub fn next_setting_id(root: impl AsRef) -> u128 { *lock += 1; *lock } + +pub struct ToSymlink { + pub ron: Vec, + pub json: Vec, +} + +pub fn symlinks(root: impl AsRef, meta: &community_settings_core::v1::Metadata) -> std::io::Result { + let mut symlink_locations = ToSymlink { ron: Vec::new(), json: Vec::new() }; + let filename_ron = filename(meta.get_id(), crate::consts::RON_EXTENSION); + let filename_json = filename(meta.get_id(), crate::consts::JSON_EXTENSION); + // build symlinks to app id folder + let app_id_folder = setting_folder_by_app_id(&root, meta.steam_app_id); + log::debug!("App id folder {}", app_id_folder.display()); + if !app_id_folder.exists() { + std::fs::create_dir(&app_id_folder)?; + std::fs::create_dir(setting_tag_folder_by_app_id(&root, meta.steam_app_id))?; + } + symlink_locations.ron.push(app_id_folder.join(&filename_ron)); + symlink_locations.json.push(app_id_folder.join(&filename_json)); + + // create symlinks for user id folder + let user_id_folder = setting_folder_by_user_id(&root, meta.steam_user_id); + if !user_id_folder.exists() { + std::fs::create_dir(&user_id_folder)?; + std::fs::create_dir(setting_tag_folder_by_user_id(&root, meta.steam_user_id))?; + } + symlink_locations.ron.push(user_id_folder.join(&filename_ron)); + symlink_locations.json.push(user_id_folder.join(&filename_json)); + + // create symlinks for each tag + for tag in meta.tags.iter() { + if !str_is_alphanumeric_or_space(&tag){ + continue; + } + // create symlinks for general tag folder + let tag_folder = setting_folder_by_tag(&root, tag); + if !tag_folder.exists() { + std::fs::create_dir(&tag_folder)?; + } + symlink_locations.ron.push(tag_folder.join(&filename_ron)); + symlink_locations.json.push(tag_folder.join(&filename_json)); + + // create symlinks for app id tag folder + let app_tag_folder = setting_folder_by_app_id_tag(&root, meta.steam_app_id, tag); + if !app_tag_folder.exists() { + std::fs::create_dir(&app_tag_folder)?; + } + symlink_locations.ron.push(app_tag_folder.join(&filename_ron)); + symlink_locations.json.push(app_tag_folder.join(&filename_json)); + + // create symlinks for user id tag folder + let user_tag_folder = setting_folder_by_user_id_tag(&root, meta.steam_user_id, tag); + if !user_tag_folder.exists() { + std::fs::create_dir(&user_tag_folder)?; + } + symlink_locations.ron.push(user_tag_folder.join(&filename_ron)); + symlink_locations.json.push(user_tag_folder.join(&filename_json)); + } + Ok(symlink_locations) +} + +fn str_is_alphanumeric_or_space(s: &str) -> bool { + let mut result = true; + for ch in s.chars() { + result &= ch.is_ascii_alphanumeric() || ch == ' '; + } + result +} diff --git a/backend/community_settings_srv/src/main.rs b/backend/community_settings_srv/src/main.rs index 58f132a..b3f26d0 100644 --- a/backend/community_settings_srv/src/main.rs +++ b/backend/community_settings_srv/src/main.rs @@ -2,6 +2,7 @@ mod api; mod cli; mod consts; mod file_util; +mod tasks; mod upgrade; use actix_web::{web, App, HttpServer}; @@ -39,9 +40,13 @@ async fn main() -> std::io::Result<()> { upgrade::make_tag_subfolders(&args.folder)?; log::info!("Resynchronizing file IDs with file name IDs"); upgrade::sync_ids(&args.folder)?; + log::info!("Rebuilding missing symlinks"); + upgrade::rebuild_symlinks(&args.folder)?; return Ok(()) } + tasks::start_tasks(args.clone()); + let leaked_args: &'static cli::Cli = Box::leak::<'static>(Box::new(args)); HttpServer::new(move || { App::new() diff --git a/backend/community_settings_srv/src/tasks/mod.rs b/backend/community_settings_srv/src/tasks/mod.rs new file mode 100644 index 0000000..b922f4b --- /dev/null +++ b/backend/community_settings_srv/src/tasks/mod.rs @@ -0,0 +1,16 @@ +mod task_runner; +mod symlink_cleanup; +mod user_antispam; + +pub fn start_tasks(args: crate::cli::Cli) { + task_runner::TaskRunner::new( + args.clone(), + symlink_cleanup::remove_broken_symlinks, + std::time::Duration::from_secs(1 * 60 /* 1 minute */), + ).run(); + task_runner::TaskRunner::new( + args.clone(), + user_antispam::remove_similar_user_uploads, + std::time::Duration::from_secs(5 * 60 /* 5 minutes */), + ).run(); +} diff --git a/backend/community_settings_srv/src/tasks/symlink_cleanup.rs b/backend/community_settings_srv/src/tasks/symlink_cleanup.rs new file mode 100644 index 0000000..34893e8 --- /dev/null +++ b/backend/community_settings_srv/src/tasks/symlink_cleanup.rs @@ -0,0 +1,39 @@ +//! Realistically this shouldn't be occur unless there's corruption or a settings files is manually deleted + +pub fn remove_broken_symlinks(args: &mut crate::cli::Cli) { + log::info!("Starting broken symlink removal task"); + if let Err(e) = enforce_working_symlinks(&args.folder) { + log::error!("Error in broken symlink task: {}", e); + } + log::info!("Completed broken symlink removal task"); +} + +fn enforce_working_symlinks(root: impl AsRef) -> std::io::Result<()> { + for dir_entry in walkdir::WalkDir::new(root.as_ref().join(crate::consts::SETTING_FOLDER)) { + let dir_entry = dir_entry?; + match check_symlink(dir_entry.path()) { + Ok(true) => {}, + Ok(false) => { + log::info!("Symlink {} seems broken, removing...", dir_entry.path().display()); + if let Err(e) = std::fs::remove_file(dir_entry.path()) { + log::warn!("Failed to delete broken symlink {}: {}", dir_entry.path().display(), e); + } + }, + Err(symlink_e) => { + log::info!("Thing {} seems broken, removing... err: {}", dir_entry.path().display(), symlink_e); + if let Err(e) = std::fs::remove_file(dir_entry.path()) { + log::warn!("Failed to delete broken thing {}: {}", dir_entry.path().display(), e); + } + } + } + } + Ok(()) +} + +fn check_symlink(path: impl AsRef) -> std::io::Result { + if path.as_ref().is_symlink() { + Ok(path.as_ref().read_link().is_ok_and(|link| link.exists())) + } else { + Ok(true) + } +} diff --git a/backend/community_settings_srv/src/tasks/task_runner.rs b/backend/community_settings_srv/src/tasks/task_runner.rs new file mode 100644 index 0000000..ced9533 --- /dev/null +++ b/backend/community_settings_srv/src/tasks/task_runner.rs @@ -0,0 +1,47 @@ +use std::thread::{spawn, JoinHandle, sleep}; +use std::time::{Duration, Instant}; + +pub struct TaskRunner { + task: F, + context: C, + period: Option, +} + +impl TaskRunner { + pub fn new(c: C, f: F, period: Duration) -> Self { + Self { + task: f, + context: c, + period: Some(period), + } + } + + /* + pub fn new_oneshot(c: C, f: F) -> Self { + Self { + task: f, + context: c, + period: None, + } + }*/ + + pub fn run(mut self) -> JoinHandle<()> { + if let Some(period) = self.period { + spawn(move || { + let mut pre_task; + let mut after_task; + loop { + pre_task = Instant::now(); + (self.task)(&mut self.context); + after_task = Instant::now(); + sleep(period - (after_task.duration_since(pre_task))); + } + }) + } else { + spawn(move || { + (self.task)(&mut self.context); + }) + } + + } +} diff --git a/backend/community_settings_srv/src/tasks/user_antispam.rs b/backend/community_settings_srv/src/tasks/user_antispam.rs new file mode 100644 index 0000000..e3379df --- /dev/null +++ b/backend/community_settings_srv/src/tasks/user_antispam.rs @@ -0,0 +1,114 @@ +use crate::file_util; + +pub fn remove_similar_user_uploads(args: &mut crate::cli::Cli) { + log::info!("Starting user antispam task"); + if let Err(e) = enforce_user_dirs(&args.folder, args.duplicates) { + log::error!("Error in user antispam task: {}", e); + } + log::info!("Completed user antispam task"); +} + +fn enforce_user_dirs(root: impl AsRef, keep_duplicates: usize) -> std::io::Result<()> { + let mut to_remove: Vec = Vec::new(); + let mut seen_names = std::collections::HashMap::>::new(); + for dir_entry in root.as_ref() + .join(crate::consts::SETTING_FOLDER) + .join(crate::consts::USER_ID_FOLDER) + .read_dir()? { + match dir_entry { + Ok(dir_entry) => { + log::info!("Scanning {} for user antispam", dir_entry.path().display()); + if dir_entry.file_type()?.is_dir() { + seen_names.clear(); + to_remove.clear(); + for user_entry in dir_entry.path().read_dir()? { + match user_entry { + Ok(user_entry) => { + let f_path = user_entry.path(); + if let Some(ext) = f_path.extension() { + if ext == crate::consts::RON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + match ron::de::from_reader::<_, community_settings_core::v1::Metadata>(reader) { + Ok(x) => { + let sani_name = sanitise_name(&x.name); + if let Some(seen_in_ids) = seen_names.get_mut(&sani_name) { + seen_in_ids.push(x); + } else { + seen_names.insert(sani_name, vec![x]); + } + + }, + Err(e) => log::debug!("Error while reading {}: {}", f_path.display(), e) + } + } + } + }, + Err(e) => log::warn!("Skipping file in {} for user antispam task due to error: {}", dir_entry.path().display(), e), + } + } + for seen in seen_names.values_mut() { + seen.sort_by_key(|meta| meta.get_id()); // sort lowest to highest + // keep the highest id (i.e. latest uploaded) settings + for _ in 0..keep_duplicates { + seen.pop(); + } + to_remove.append(seen); + } + log::info!("Found {} spammy entries in {}", to_remove.len(), dir_entry.path().display()); + // remove settings (and related symlinks) from the filesystem + for meta in to_remove.iter() { + let filename_ron = file_util::filename(meta.get_id(), crate::consts::RON_EXTENSION); + let filename_json = file_util::filename(meta.get_id(), crate::consts::JSON_EXTENSION); + // delete tag symlinks + for tag in meta.tags.iter() { + let app_id_tag_folder = file_util::setting_folder_by_app_id_tag(root.as_ref(), meta.steam_app_id, tag); + let user_id_tag_folder = file_util::setting_folder_by_user_id_tag(root.as_ref(), meta.steam_user_id, tag); + let tag_folder = file_util::setting_folder_by_tag(root.as_ref(), tag); + let paths = [ + app_id_tag_folder.join(&filename_json), + app_id_tag_folder.join(&filename_ron), + user_id_tag_folder.join(&filename_json), + user_id_tag_folder.join(&filename_ron), + tag_folder.join(&filename_json), + tag_folder.join(&filename_ron), + ]; + for path in paths { + if path.exists() { + if let Err(e) = std::fs::remove_file(&path) { + log::warn!("Failed to delete {}: {}", path.display(), e); + } + } + } + } + // delete first-order symlinks and finally the actual files + let app_id_folder = file_util::setting_folder_by_app_id(root.as_ref(), meta.steam_app_id); + let user_id_folder = file_util::setting_folder_by_user_id(root.as_ref(), meta.steam_user_id); + let paths = [ + app_id_folder.join(&filename_json), + app_id_folder.join(&filename_ron), + user_id_folder.join(&filename_json), + user_id_folder.join(&filename_ron), + file_util::setting_path_by_id(root.as_ref(), meta.get_id(), crate::consts::JSON_EXTENSION), + file_util::setting_path_by_id(root.as_ref(), meta.get_id(), crate::consts::RON_EXTENSION), + ]; + for path in paths { + if path.exists() { + if let Err(e) = std::fs::remove_file(&path) { + log::warn!("Failed to delete {}: {}", path.display(), e); + } + } + } + } + } else { + log::info!("Encountered non-folder path in user dir: {}", dir_entry.path().display()); + } + }, + Err(e) => log::warn!("Skipping file for user antispam task due to error: {}", e), + } + } + Ok(()) +} + +fn sanitise_name(name: &str) -> String { + name.trim().to_lowercase() +} diff --git a/backend/community_settings_srv/src/upgrade.rs b/backend/community_settings_srv/src/upgrade.rs index c48f9b3..052cb83 100644 --- a/backend/community_settings_srv/src/upgrade.rs +++ b/backend/community_settings_srv/src/upgrade.rs @@ -26,6 +26,59 @@ pub fn build_folder_layout(root: impl AsRef) -> std::io::Result<()> { Ok(()) } +pub fn rebuild_symlinks(root: impl AsRef) -> std::io::Result<()> { + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_file() { + let f_path = dir_entry.path(); + if let Some(ext) = f_path.extension() { + if ext == RON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + let setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) { + Ok(x) => x, + Err(e) => { + log::debug!("Error while reading {}: {}", f_path.display(), e); + let e_msg = format!("{}", e); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); + } + }; + let to_symlink = crate::file_util::symlinks(&root, &setting)?; + let path_ron_canon = f_path.canonicalize()?; + let path_json_canon = crate::file_util::setting_path_by_id(&root, setting.get_id(), JSON_EXTENSION).canonicalize()?; + for ron_link in to_symlink.ron { + if ron_link.exists() { continue; } + log::info!("Rebuilding symlink {} -> {}", ron_link.display(), path_ron_canon.display()); + #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained + { + std::os::windows::fs::symlink_file(&path_ron_canon, &ron_link)?; + } + #[cfg(target_family = "unix")] + { + std::os::unix::fs::symlink(&path_ron_canon, &ron_link)?; + } + } + for json_link in to_symlink.json { + if json_link.exists() { continue; } + log::info!("Rebuilding symlink {} -> {}", json_link.display(), path_json_canon.display()); + #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained + { + std::os::windows::fs::symlink_file(&path_json_canon, json_link)?; + } + #[cfg(target_family = "unix")] + { + std::os::unix::fs::symlink(&path_json_canon, json_link)?; + } + } + } + } + } + } + Ok(()) +} + pub fn fix_symlinks(root: impl AsRef) -> std::io::Result<()> { log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display()); for dir_entry in root.as_ref()