Merge branch 'dev'

backend/Cargo.lock

@@ -1170,7 +1170,7 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "powertools"
-version = "2.0.0"
+version = "2.0.1"
 dependencies = [
  "async-trait",
  "chrono",

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "powertools"
-version = "2.0.0"
+version = "2.0.1"
 edition = "2021"
 authors = ["NGnius (Graham) <ngniusness@gmail.com>"]
 description = "Backend (superuser) functionality for PowerTools"

backend/build.sh

@@ -3,11 +3,11 @@
 #cargo build --release --target x86_64-unknown-linux-musl
 #cargo build --target x86_64-unknown-linux-musl
 #cross build
-cargo build --release
+#cargo build --release
-#cargo build
+cargo build
 
 mkdir -p ../bin
 #cp --preserve=mode ./target/x86_64-unknown-linux-musl/release/powertools ../bin/backend
 #cp --preserve=mode ./target/x86_64-unknown-linux-musl/debug/powertools ../bin/backend
-cp --preserve=mode ./target/release/powertools ../bin/backend
+#cp --preserve=mode ./target/release/powertools ../bin/backend
-#cp --preserve=mode ./target/debug/powertools ../bin/backend
+cp --preserve=mode ./target/debug/powertools ../bin/backend

backend/community_settings_srv/Cargo.lock

@@ -438,7 +438,7 @@ dependencies = [
 
 [[package]]
 name = "community_settings_srv"
-version = "0.1.0"
+version = "0.1.1"
 dependencies = [
  "actix-web",
  "clap",
@@ -450,6 +450,7 @@ dependencies = [
  "serde_json",
  "simplelog",
  "tokio",
+ "walkdir",
 ]
 
 [[package]]
@@ -1017,6 +1018,15 @@ version = "1.0.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -1319,6 +1329,16 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"

backend/community_settings_srv/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "community_settings_srv"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -16,6 +16,8 @@ tokio = { version = "1", features = ["full"] }
 actix-web = { version = "4.4" }
 mime = { version = "0.3.17" }
 
+walkdir = "2"
+
 # logging
 log = "0.4"
 simplelog = "0.12"

backend/community_settings_srv/build_arm64.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+cargo build --release --target aarch64-unknown-linux-musl

backend/community_settings_srv/src/api/get_game.rs

@@ -60,11 +60,12 @@ fn get_some_settings_by_app_id(steam_app_id: u32, cli: &'static Cli) -> std::io:
     let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, steam_app_id);
     let mut files: Vec<_> = app_id_folder.read_dir()?
                     .filter_map(|res| res.ok())
-                    .filter(|f| f.path().extension().map(|ext| ext == file_util::RON_EXTENSION).unwrap_or(false))
+                    .filter(|f| f.path().extension().map(|ext| ext == crate::consts::RON_EXTENSION).unwrap_or(false))
                     .filter_map(|f| f.metadata().ok().map(|meta| (f, meta)))
-                    .filter_map(|(f, meta)| meta.created().ok().map(|time| (f, meta, time)))
+                    .filter_map(|(f, meta)| meta.modified().ok().map(|time| (f, meta, time)))
                     .collect();
     files.sort_by(|(_, _, a_created), (_, _, b_created)| a_created.cmp(b_created));
+    let files_len = files.len();
 
     let mut results = Vec::with_capacity(MAX_RESULTS);
     for (_, (f, _, _)) in files.into_iter().enumerate().take_while(|(i, _)| *i < MAX_RESULTS) {
@@ -72,12 +73,14 @@ fn get_some_settings_by_app_id(steam_app_id: u32, cli: &'static Cli) -> std::io:
         let setting = match ron::de::from_reader(reader) {
             Ok(x) => x,
             Err(e) => {
+                log::debug!("Error while reading {}: {}", f.path().display(), e);
                 let e_msg = format!("{}", e);
                 return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
             }
         };
         results.push(setting);
     }
+    log::debug!("Got {} results (from {} files) for {}", results.len(), files_len, app_id_folder.display());
     Ok(results)
 }
 
@@ -88,7 +91,8 @@ pub async fn get_setting_by_app_id_handler(
     cli: web::Data<&'static Cli>,
 ) -> std::io::Result<impl Responder> {
     let id: u32 = *id;
-    println!("Accept: {}", accept.to_string());
+    #[cfg(debug_assertions)]
+    log::debug!("Accept: {}", accept.to_string());
     let preferred = accept.preference();
     if super::is_mime_type_ron_capable(&preferred) {
         // Send RON

backend/community_settings_srv/src/api/get_setting.rs

@@ -67,7 +67,7 @@ pub async fn get_setting_handler(
     if super::is_mime_type_ron_capable(&preferred) {
         // Send RON
         let ron = if id != 0 {
-            let path = file_util::setting_path_by_id(&cli.folder, id, file_util::RON_EXTENSION);
+            let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::RON_EXTENSION);
             if !path.exists() {
                 return Err(std::io::Error::new(std::io::ErrorKind::NotFound, format!("setting id {} does not exist", id)));
             }
@@ -93,7 +93,7 @@ pub async fn get_setting_handler(
     } else {
         // Send JSON (fallback)
         let json = if id != 0 {
-            let path = file_util::setting_path_by_id(&cli.folder, id, file_util::JSON_EXTENSION);
+            let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::JSON_EXTENSION);
             // TODO? cache this instead of always loading it from file
             let reader = std::io::BufReader::new(std::fs::File::open(path)?);
             match serde_json::from_reader(reader) {

backend/community_settings_srv/src/api/save_setting.rs

@@ -18,7 +18,7 @@ pub async fn save_setting_handler(
         Err(_e) => return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "too many bytes in payload")),
     };
     let next_id = file_util::next_setting_id(&cli.folder);
-    let parsed_data: community_settings_core::v1::Metadata = if super::is_mime_type_ron_capable(&content_type) {
+    let mut parsed_data: community_settings_core::v1::Metadata = if super::is_mime_type_ron_capable(&content_type) {
         // Parse as RON
         match ron::de::from_reader(bytes.as_ref()) {
             Ok(x) => x,
@@ -42,16 +42,18 @@ pub async fn save_setting_handler(
             }
         }
     };
+    // Override the ID with the one used by this server
+    parsed_data.id = next_id.to_string();
     // TODO validate user and app id
     // Reject blocked users and apps
-    let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, file_util::RON_EXTENSION);
+    let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::RON_EXTENSION);
     let writer = std::io::BufWriter::new(std::fs::File::create(&path_ron)?);
     if let Err(e) = ron::ser::to_writer(writer, &parsed_data) {
         let e_msg = format!("{}", e);
         return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
     }
 
-    let path_json = file_util::setting_path_by_id(&cli.folder, next_id, file_util::JSON_EXTENSION);
+    let path_json = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::JSON_EXTENSION);
     let writer = std::io::BufWriter::new(std::fs::File::create(&path_json)?);
     if let Err(e) = serde_json::to_writer(writer, &parsed_data) {
         let e_msg = format!("{}", e);
@@ -62,57 +64,32 @@ pub async fn save_setting_handler(
         }
     }
 
-    // create symlinks for other ways of looking up these settings files
+    log::debug!("Saved to {}, building symlinks", path_ron.display());
-    let filename_ron = file_util::filename(next_id, file_util::RON_EXTENSION);
-    let filename_json = file_util::filename(next_id, file_util::JSON_EXTENSION);
 
-    // create symlinks to app id folder
+    let to_symlink = file_util::symlinks(&cli.folder, &parsed_data)?;
-    let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, parsed_data.steam_app_id);
-    if !app_id_folder.exists() {
-        std::fs::create_dir(&app_id_folder)?;
-    }
-    #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
-    {
-        std::os::windows::fs::symlink_file(&path_ron, app_id_folder.join(&filename_ron))?;
-        std::os::windows::fs::symlink_file(&path_json, app_id_folder.join(&filename_json))?;
-    }
-    #[cfg(target_family = "unix")]
-    {
-        std::os::unix::fs::symlink(&path_ron, app_id_folder.join(&filename_ron))?;
-        std::os::unix::fs::symlink(&path_json, app_id_folder.join(&filename_json))?;
-    }
 
-    // create symlinks for user id folder
+    let path_ron_canon = path_ron.canonicalize()?;
-    let user_id_folder = file_util::setting_folder_by_user_id(&cli.folder, parsed_data.steam_user_id);
+    let path_json_canon = path_json.canonicalize()?;
-    if !user_id_folder.exists() {
+    for ron_link in to_symlink.ron {
-        std::fs::create_dir(&user_id_folder)?;
+        log::debug!("Symlinking {} -> {}", ron_link.display(), path_ron_canon.display());
-    }
-    #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
-    {
-        std::os::windows::fs::symlink_file(&path_ron, user_id_folder.join(&filename_ron))?;
-        std::os::windows::fs::symlink_file(&path_json, user_id_folder.join(&filename_json))?;
-    }
-    #[cfg(target_family = "unix")]
-    {
-        std::os::unix::fs::symlink(&path_ron, user_id_folder.join(&filename_ron))?;
-        std::os::unix::fs::symlink(&path_json, user_id_folder.join(&filename_json))?;
-    }
-
-    // create symlinks for each tag
-    for tag in parsed_data.tags.iter() {
-        let tag_folder = file_util::setting_folder_by_tag(&cli.folder, tag);
-        if !tag_folder.exists() {
-            std::fs::create_dir(&tag_folder)?;
-        }
         #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
         {
-            std::os::windows::fs::symlink_file(&path_ron, tag_folder.join(&filename_ron))?;
+            std::os::windows::fs::symlink_file(&path_ron_canon, &ron_link)?;
-            std::os::windows::fs::symlink_file(&path_json, tag_folder.join(&filename_json))?;
         }
         #[cfg(target_family = "unix")]
         {
-            std::os::unix::fs::symlink(&path_ron, tag_folder.join(&filename_ron))?;
+            std::os::unix::fs::symlink(&path_ron_canon, &ron_link)?;
-            std::os::unix::fs::symlink(&path_json, tag_folder.join(&filename_json))?;
+        }
+    }
+    for json_link in to_symlink.json {
+        //log::debug!("Symlinking {} -> {}", json_link.display(), path_json_canon.display());
+        #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
+        {
+            std::os::windows::fs::symlink_file(&path_json_canon, json_link)?;
+        }
+        #[cfg(target_family = "unix")]
+        {
+            std::os::unix::fs::symlink(&path_json_canon, json_link)?;
         }
     }
 

backend/community_settings_srv/src/cli.rs

@@ -14,6 +14,14 @@ pub struct Cli {
     /// Log file location
     #[arg(short, long, default_value = "/tmp/powertools_community_settings_srv.log")]
     pub log: std::path::PathBuf,
+
+    /// Perform maintenance tasks
+    #[arg(long)]
+    pub fix: bool,
+
+    /// Keep up to this many duplicate settings (0 will delete everything!!!)
+    #[arg(long, default_value_t = 2)]
+    pub duplicates: usize,
 }
 
 impl Cli {

backend/community_settings_srv/src/consts.rs

@@ -0,0 +1,8 @@
+pub const RON_EXTENSION: &'static str = "ron";
+pub const JSON_EXTENSION: &'static str = "json";
+
+pub const SETTING_FOLDER: &'static str = "settings";
+pub const ID_FOLDER: &'static str = "by_id";
+pub const APP_ID_FOLDER: &'static str = "by_app_id";
+pub const USER_ID_FOLDER: &'static str = "by_user_id";
+pub const TAG_FOLDER: &'static str = "by_tag";

backend/community_settings_srv/src/file_util.rs

@@ -1,41 +1,10 @@
 use std::path::{Path, PathBuf};
 use std::sync::Mutex;
 
-pub const RON_EXTENSION: &'static str = "ron";
+use crate::consts::*;
-pub const JSON_EXTENSION: &'static str = "json";
-
-const SETTING_FOLDER: &'static str = "settings";
-const ID_FOLDER: &'static str = "by_id";
-const APP_ID_FOLDER: &'static str = "by_app_id";
-const USER_ID_FOLDER: &'static str = "by_user_id";
-const TAG_FOLDER: &'static str = "by_tag";
 
 static LAST_SETTING_ID: Mutex<u128> = Mutex::new(0);
 
-pub fn build_folder_layout(root: impl AsRef<Path>) -> std::io::Result<()> {
-    std::fs::create_dir_all(
-        root.as_ref()
-            .join(SETTING_FOLDER)
-            .join(ID_FOLDER)
-    )?;
-    std::fs::create_dir_all(
-        root.as_ref()
-            .join(SETTING_FOLDER)
-            .join(APP_ID_FOLDER)
-    )?;
-    std::fs::create_dir_all(
-        root.as_ref()
-            .join(SETTING_FOLDER)
-            .join(USER_ID_FOLDER)
-    )?;
-    std::fs::create_dir_all(
-        root.as_ref()
-            .join(SETTING_FOLDER)
-            .join(TAG_FOLDER)
-    )?;
-    Ok(())
-}
-
 pub fn filename(id: u128, ext: &str) -> String {
     format!("{}.{}", id, ext)
 }
@@ -54,6 +23,23 @@ pub fn setting_folder_by_app_id(root: impl AsRef<Path>, steam_app_id: u32) -> Pa
         .join(steam_app_id.to_string())
 }
 
+pub fn setting_tag_folder_by_app_id(root: impl AsRef<Path>, steam_app_id: u32) -> PathBuf {
+    root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(APP_ID_FOLDER)
+        .join(steam_app_id.to_string())
+        .join(TAG_FOLDER)
+}
+
+pub fn setting_folder_by_app_id_tag(root: impl AsRef<Path>, steam_app_id: u32, tag: &str) -> PathBuf {
+    root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(APP_ID_FOLDER)
+        .join(steam_app_id.to_string())
+        .join(TAG_FOLDER)
+        .join(tag)
+}
+
 pub fn setting_folder_by_user_id(root: impl AsRef<Path>, steam_user_id: u64) -> PathBuf {
     root.as_ref()
         .join(SETTING_FOLDER)
@@ -61,6 +47,23 @@ pub fn setting_folder_by_user_id(root: impl AsRef<Path>, steam_user_id: u64) ->
         .join(steam_user_id.to_string())
 }
 
+pub fn setting_tag_folder_by_user_id(root: impl AsRef<Path>, steam_user_id: u64) -> PathBuf {
+    root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(USER_ID_FOLDER)
+        .join(steam_user_id.to_string())
+        .join(TAG_FOLDER)
+}
+
+pub fn setting_folder_by_user_id_tag(root: impl AsRef<Path>, steam_user_id: u64, tag: &str) -> PathBuf {
+    root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(USER_ID_FOLDER)
+        .join(steam_user_id.to_string())
+        .join(TAG_FOLDER)
+        .join(tag)
+}
+
 pub fn setting_folder_by_tag(root: impl AsRef<Path>, tag: &str) -> PathBuf {
     root.as_ref()
         .join(SETTING_FOLDER)
@@ -73,14 +76,83 @@ pub fn next_setting_id(root: impl AsRef<Path>) -> u128 {
     let mut last_id = *lock;
     if last_id == 0 {
         // needs init
+        last_id = 1;
         let mut path = setting_path_by_id(root.as_ref(), last_id, RON_EXTENSION);
         while path.exists() {
             last_id += 1;
             path = setting_path_by_id(root.as_ref(), last_id, RON_EXTENSION);
         }
-        *lock = last_id;
+        *lock = last_id - 1;
-        println!("setting id initialized to {}", last_id);
+        log::info!("setting id initialized to {}", last_id);
     }
     *lock += 1;
     *lock
 }
+
+pub struct ToSymlink {
+    pub ron: Vec<PathBuf>,
+    pub json: Vec<PathBuf>,
+}
+
+pub fn symlinks(root: impl AsRef<Path>, meta: &community_settings_core::v1::Metadata) -> std::io::Result<ToSymlink> {
+    let mut symlink_locations = ToSymlink { ron: Vec::new(), json: Vec::new() };
+    let filename_ron = filename(meta.get_id(), crate::consts::RON_EXTENSION);
+    let filename_json = filename(meta.get_id(), crate::consts::JSON_EXTENSION);
+    // build symlinks to app id folder
+    let app_id_folder = setting_folder_by_app_id(&root, meta.steam_app_id);
+    log::debug!("App id folder {}", app_id_folder.display());
+    if !app_id_folder.exists() {
+        std::fs::create_dir(&app_id_folder)?;
+        std::fs::create_dir(setting_tag_folder_by_app_id(&root, meta.steam_app_id))?;
+    }
+    symlink_locations.ron.push(app_id_folder.join(&filename_ron));
+    symlink_locations.json.push(app_id_folder.join(&filename_json));
+
+    // create symlinks for user id folder
+    let user_id_folder = setting_folder_by_user_id(&root, meta.steam_user_id);
+    if !user_id_folder.exists() {
+        std::fs::create_dir(&user_id_folder)?;
+        std::fs::create_dir(setting_tag_folder_by_user_id(&root, meta.steam_user_id))?;
+    }
+    symlink_locations.ron.push(user_id_folder.join(&filename_ron));
+    symlink_locations.json.push(user_id_folder.join(&filename_json));
+
+    // create symlinks for each tag
+    for tag in meta.tags.iter() {
+        if !str_is_alphanumeric_or_space(&tag){
+            continue;
+        }
+        // create symlinks for general tag folder
+        let tag_folder = setting_folder_by_tag(&root, tag);
+        if !tag_folder.exists() {
+            std::fs::create_dir(&tag_folder)?;
+        }
+        symlink_locations.ron.push(tag_folder.join(&filename_ron));
+        symlink_locations.json.push(tag_folder.join(&filename_json));
+
+        // create symlinks for app id tag folder
+        let app_tag_folder = setting_folder_by_app_id_tag(&root, meta.steam_app_id, tag);
+        if !app_tag_folder.exists() {
+            std::fs::create_dir(&app_tag_folder)?;
+        }
+        symlink_locations.ron.push(app_tag_folder.join(&filename_ron));
+        symlink_locations.json.push(app_tag_folder.join(&filename_json));
+
+        // create symlinks for user id tag folder
+        let user_tag_folder = setting_folder_by_user_id_tag(&root, meta.steam_user_id, tag);
+        if !user_tag_folder.exists() {
+            std::fs::create_dir(&user_tag_folder)?;
+        }
+        symlink_locations.ron.push(user_tag_folder.join(&filename_ron));
+        symlink_locations.json.push(user_tag_folder.join(&filename_json));
+    }
+    Ok(symlink_locations)
+}
+
+fn str_is_alphanumeric_or_space(s: &str) -> bool {
+    let mut result = true;
+    for ch in s.chars() {
+        result &= ch.is_ascii_alphanumeric() || ch == ' ';
+    }
+    result
+}

backend/community_settings_srv/src/main.rs

@@ -1,6 +1,9 @@
 mod api;
 mod cli;
+mod consts;
 mod file_util;
+mod tasks;
+mod upgrade;
 
 use actix_web::{web, App, HttpServer};
 
@@ -27,7 +30,22 @@ async fn main() -> std::io::Result<()> {
 
     // setup
     log::debug!("Building folder layout (if not exists) at: {}", &args.folder.display());
-    file_util::build_folder_layout(&args.folder)?;
+    upgrade::build_folder_layout(&args.folder)?;
+
+    // fix things
+    if args.fix {
+        log::info!("Fixing old symlinks");
+        upgrade::fix_symlinks(&args.folder)?;
+        log::info!("Creating missing by_tag folders");
+        upgrade::make_tag_subfolders(&args.folder)?;
+        log::info!("Resynchronizing file IDs with file name IDs");
+        upgrade::sync_ids(&args.folder)?;
+        log::info!("Rebuilding missing symlinks");
+        upgrade::rebuild_symlinks(&args.folder)?;
+        return Ok(())
+    }
+
+    tasks::start_tasks(args.clone());
 
     let leaked_args: &'static cli::Cli = Box::leak::<'static>(Box::new(args));
     HttpServer::new(move || {

backend/community_settings_srv/src/tasks/mod.rs

@@ -0,0 +1,16 @@
+mod task_runner;
+mod symlink_cleanup;
+mod user_antispam;
+
+pub fn start_tasks(args: crate::cli::Cli) {
+    task_runner::TaskRunner::new(
+        args.clone(),
+        symlink_cleanup::remove_broken_symlinks,
+        std::time::Duration::from_secs(1 * 60 /* 1 minute */),
+    ).run();
+    task_runner::TaskRunner::new(
+        args.clone(),
+        user_antispam::remove_similar_user_uploads,
+        std::time::Duration::from_secs(5 * 60 /* 5 minutes */),
+    ).run();
+}

backend/community_settings_srv/src/tasks/symlink_cleanup.rs

@@ -0,0 +1,39 @@
+//! Realistically this shouldn't be occur unless there's corruption or a settings files is manually deleted
+
+pub fn remove_broken_symlinks(args: &mut crate::cli::Cli) {
+    log::info!("Starting broken symlink removal task");
+    if let Err(e) = enforce_working_symlinks(&args.folder) {
+        log::error!("Error in broken symlink task: {}", e);
+    }
+    log::info!("Completed broken symlink removal task");
+}
+
+fn enforce_working_symlinks(root: impl AsRef<std::path::Path>) -> std::io::Result<()> {
+    for dir_entry in walkdir::WalkDir::new(root.as_ref().join(crate::consts::SETTING_FOLDER)) {
+        let dir_entry = dir_entry?;
+        match check_symlink(dir_entry.path()) {
+            Ok(true) => {},
+            Ok(false) => {
+                log::info!("Symlink {} seems broken, removing...", dir_entry.path().display());
+                if let Err(e) = std::fs::remove_file(dir_entry.path()) {
+                    log::warn!("Failed to delete broken symlink {}: {}", dir_entry.path().display(), e);
+                }
+            },
+            Err(symlink_e) => {
+                log::info!("Thing {} seems broken, removing... err: {}", dir_entry.path().display(), symlink_e);
+                if let Err(e) = std::fs::remove_file(dir_entry.path()) {
+                    log::warn!("Failed to delete broken thing {}: {}", dir_entry.path().display(), e);
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+fn check_symlink(path: impl AsRef<std::path::Path>) -> std::io::Result<bool> {
+    if path.as_ref().is_symlink() {
+        Ok(path.as_ref().read_link().is_ok_and(|link| link.exists()))
+    } else {
+        Ok(true)
+    }
+}

backend/community_settings_srv/src/tasks/task_runner.rs

@@ -0,0 +1,47 @@
+use std::thread::{spawn, JoinHandle, sleep};
+use std::time::{Duration, Instant};
+
+pub struct TaskRunner<C: Send + 'static, F: FnMut(&mut C) + Send + 'static> {
+    task: F,
+    context: C,
+    period: Option<Duration>,
+}
+
+impl <C: Send + 'static, F: FnMut(&mut C) + Send + 'static> TaskRunner<C, F> {
+    pub fn new(c: C, f: F, period: Duration) -> Self {
+        Self {
+            task: f,
+            context: c,
+            period: Some(period),
+        }
+    }
+
+    /*
+    pub fn new_oneshot(c: C, f: F) -> Self {
+        Self {
+            task: f,
+            context: c,
+            period: None,
+        }
+    }*/
+
+    pub fn run(mut self) -> JoinHandle<()> {
+        if let Some(period) = self.period {
+            spawn(move || {
+                let mut pre_task;
+                let mut after_task;
+                loop {
+                    pre_task = Instant::now();
+                    (self.task)(&mut self.context);
+                    after_task = Instant::now();
+                    sleep(period - (after_task.duration_since(pre_task)));
+                }
+            })
+        } else {
+            spawn(move || {
+                (self.task)(&mut self.context);
+            })
+        }
+
+    }
+}

backend/community_settings_srv/src/tasks/user_antispam.rs

@@ -0,0 +1,114 @@
+use crate::file_util;
+
+pub fn remove_similar_user_uploads(args: &mut crate::cli::Cli) {
+    log::info!("Starting user antispam task");
+    if let Err(e) = enforce_user_dirs(&args.folder, args.duplicates) {
+        log::error!("Error in user antispam task: {}", e);
+    }
+    log::info!("Completed user antispam task");
+}
+
+fn enforce_user_dirs(root: impl AsRef<std::path::Path>, keep_duplicates: usize) -> std::io::Result<()> {
+    let mut to_remove: Vec<community_settings_core::v1::Metadata> = Vec::new();
+    let mut seen_names = std::collections::HashMap::<String, Vec<community_settings_core::v1::Metadata>>::new();
+    for dir_entry in root.as_ref()
+                .join(crate::consts::SETTING_FOLDER)
+                .join(crate::consts::USER_ID_FOLDER)
+                .read_dir()? {
+        match dir_entry {
+            Ok(dir_entry) => {
+                log::info!("Scanning {} for user antispam", dir_entry.path().display());
+                if dir_entry.file_type()?.is_dir() {
+                    seen_names.clear();
+                    to_remove.clear();
+                    for user_entry in dir_entry.path().read_dir()? {
+                        match user_entry {
+                            Ok(user_entry) => {
+                                let f_path = user_entry.path();
+                                if let Some(ext) = f_path.extension() {
+                                    if ext == crate::consts::RON_EXTENSION {
+                                        let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
+                                        match ron::de::from_reader::<_, community_settings_core::v1::Metadata>(reader) {
+                                            Ok(x) =>  {
+                                                let sani_name = sanitise_name(&x.name);
+                                                if let Some(seen_in_ids) = seen_names.get_mut(&sani_name) {
+                                                    seen_in_ids.push(x);
+                                                } else {
+                                                    seen_names.insert(sani_name, vec![x]);
+                                                }
+
+                                            },
+                                            Err(e) => log::debug!("Error while reading {}: {}", f_path.display(), e)
+                                        }
+                                    }
+                                }
+                            },
+                            Err(e) => log::warn!("Skipping file in {} for user antispam task due to error: {}", dir_entry.path().display(), e),
+                        }
+                    }
+                    for seen in seen_names.values_mut() {
+                        seen.sort_by_key(|meta| meta.get_id()); // sort lowest to highest
+                        // keep the highest id (i.e. latest uploaded) settings
+                        for _ in 0..keep_duplicates {
+                            seen.pop();
+                        }
+                        to_remove.append(seen);
+                    }
+                    log::info!("Found {} spammy entries in {}", to_remove.len(), dir_entry.path().display());
+                    // remove settings (and related symlinks) from the filesystem
+                    for meta in to_remove.iter() {
+                        let filename_ron = file_util::filename(meta.get_id(), crate::consts::RON_EXTENSION);
+                        let filename_json = file_util::filename(meta.get_id(), crate::consts::JSON_EXTENSION);
+                        // delete tag symlinks
+                        for tag in meta.tags.iter() {
+                            let app_id_tag_folder = file_util::setting_folder_by_app_id_tag(root.as_ref(), meta.steam_app_id, tag);
+                            let user_id_tag_folder = file_util::setting_folder_by_user_id_tag(root.as_ref(), meta.steam_user_id, tag);
+                            let tag_folder = file_util::setting_folder_by_tag(root.as_ref(), tag);
+                            let paths = [
+                                app_id_tag_folder.join(&filename_json),
+                                app_id_tag_folder.join(&filename_ron),
+                                user_id_tag_folder.join(&filename_json),
+                                user_id_tag_folder.join(&filename_ron),
+                                tag_folder.join(&filename_json),
+                                tag_folder.join(&filename_ron),
+                            ];
+                            for path in paths {
+                                if path.exists() {
+                                    if let Err(e) = std::fs::remove_file(&path) {
+                                        log::warn!("Failed to delete {}: {}", path.display(), e);
+                                    }
+                                }
+                            }
+                        }
+                        // delete first-order symlinks and finally the actual files
+                        let app_id_folder = file_util::setting_folder_by_app_id(root.as_ref(), meta.steam_app_id);
+                        let user_id_folder = file_util::setting_folder_by_user_id(root.as_ref(), meta.steam_user_id);
+                        let paths = [
+                            app_id_folder.join(&filename_json),
+                            app_id_folder.join(&filename_ron),
+                            user_id_folder.join(&filename_json),
+                            user_id_folder.join(&filename_ron),
+                            file_util::setting_path_by_id(root.as_ref(), meta.get_id(), crate::consts::JSON_EXTENSION),
+                            file_util::setting_path_by_id(root.as_ref(), meta.get_id(), crate::consts::RON_EXTENSION),
+                        ];
+                        for path in paths {
+                            if path.exists() {
+                                if let Err(e) = std::fs::remove_file(&path) {
+                                    log::warn!("Failed to delete {}: {}", path.display(), e);
+                                }
+                            }
+                        }
+                    }
+                } else {
+                    log::info!("Encountered non-folder path in user dir: {}", dir_entry.path().display());
+                }
+            },
+            Err(e) => log::warn!("Skipping file for user antispam task due to error: {}", e),
+        }
+    }
+    Ok(())
+}
+
+fn sanitise_name(name: &str) -> String {
+    name.trim().to_lowercase()
+}

backend/community_settings_srv/src/upgrade.rs

@@ -0,0 +1,217 @@
+use std::path::Path;
+
+use crate::consts::*;
+
+pub fn build_folder_layout(root: impl AsRef<Path>) -> std::io::Result<()> {
+    std::fs::create_dir_all(
+        root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(ID_FOLDER)
+    )?;
+    std::fs::create_dir_all(
+        root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(APP_ID_FOLDER)
+    )?;
+    std::fs::create_dir_all(
+        root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(USER_ID_FOLDER)
+    )?;
+    std::fs::create_dir_all(
+        root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(TAG_FOLDER)
+    )?;
+    Ok(())
+}
+
+pub fn rebuild_symlinks(root: impl AsRef<Path>) -> std::io::Result<()> {
+    for dir_entry in root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(ID_FOLDER)
+            .read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.file_type()?.is_file() {
+            let f_path = dir_entry.path();
+            if let Some(ext) = f_path.extension() {
+                if ext == RON_EXTENSION {
+                    let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
+                    let setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) {
+                        Ok(x) => x,
+                        Err(e) => {
+                            log::debug!("Error while reading {}: {}", f_path.display(), e);
+                            let e_msg = format!("{}", e);
+                            return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
+                        }
+                    };
+                    let to_symlink = crate::file_util::symlinks(&root, &setting)?;
+                    let path_ron_canon = f_path.canonicalize()?;
+                    let path_json_canon = crate::file_util::setting_path_by_id(&root, setting.get_id(), JSON_EXTENSION).canonicalize()?;
+                    for ron_link in to_symlink.ron {
+                        if ron_link.exists() { continue; }
+                        log::info!("Rebuilding symlink {} -> {}", ron_link.display(), path_ron_canon.display());
+                        #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
+                        {
+                            std::os::windows::fs::symlink_file(&path_ron_canon, &ron_link)?;
+                        }
+                        #[cfg(target_family = "unix")]
+                        {
+                            std::os::unix::fs::symlink(&path_ron_canon, &ron_link)?;
+                        }
+                    }
+                    for json_link in to_symlink.json {
+                        if json_link.exists() { continue; }
+                        log::info!("Rebuilding symlink {} -> {}", json_link.display(), path_json_canon.display());
+                        #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
+                        {
+                            std::os::windows::fs::symlink_file(&path_json_canon, json_link)?;
+                        }
+                        #[cfg(target_family = "unix")]
+                        {
+                            std::os::unix::fs::symlink(&path_json_canon, json_link)?;
+                        }
+                    }
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+pub fn fix_symlinks(root: impl AsRef<Path>) -> std::io::Result<()> {
+    log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display());
+    for dir_entry in root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(APP_ID_FOLDER)
+            .read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.file_type()?.is_dir() {
+            make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
+        }
+    }
+    for dir_entry in root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(USER_ID_FOLDER)
+            .read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.file_type()?.is_dir() {
+            make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
+        }
+    }
+    for dir_entry in root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(TAG_FOLDER).read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.file_type()?.is_dir() {
+            make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
+        }
+    }
+
+    Ok(())
+}
+
+fn make_symlinks_absolute_in_dir(root: impl AsRef<Path>, dir: impl AsRef<Path>) -> std::io::Result<()> {
+    let abs_root = root.as_ref().canonicalize()?;
+    assert!(abs_root.is_absolute());
+    for dir_entry in dir.as_ref()
+            .read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.file_type()?.is_symlink() {
+            let path = dir_entry.path();
+            let link_path = path.read_link()?;
+            if !link_path.is_absolute() {
+                let new_link = abs_root.join(
+                    link_path.strip_prefix(&root).expect("Symlinked path does not begin with root settings folder")
+                );
+                log::info!("Fixing {} -> {} to -> {}", path.display(), link_path.display(), new_link.display());
+                std::fs::remove_file(&path)?;
+                #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
+                {
+                    std::os::windows::fs::symlink_file(new_link, &path)?;
+                }
+                #[cfg(target_family = "unix")]
+                {
+                    std::os::unix::fs::symlink(new_link, &path)?;
+                }
+            }else {
+                log::info!("Found already-absolute symlink {} -> {}", path.display(), link_path.display());
+            }
+        } else {
+            log::info!("Found non-symlink {}: {:?}", dir_entry.path().display(), dir_entry.file_type()?);
+        }
+    }
+    Ok(())
+}
+
+pub fn sync_ids(root: impl AsRef<Path>) -> std::io::Result<()> {
+    for dir_entry in root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(ID_FOLDER)
+        .read_dir()? {
+        let dir_entry = dir_entry?;
+        let f_path = dir_entry.path();
+        if let Some(ext) = f_path.extension() {
+            let id = f_path.file_stem().map(|os| os.to_string_lossy().to_string()).unwrap();
+            if ext == RON_EXTENSION {
+                let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
+                let mut setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) {
+                    Ok(x) => x,
+                    Err(e) => {
+                        log::debug!("Error while reading {}: {}", f_path.display(), e);
+                        let e_msg = format!("{}", e);
+                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
+                    }
+                };
+                if setting.id != id {
+                    setting.id = id;
+                    ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap();
+                }
+            } else if ext == JSON_EXTENSION {
+                let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
+                let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) {
+                    Ok(x) => x,
+                    Err(e) => {
+                        log::debug!("Error while reading {}: {}", f_path.display(), e);
+                        let e_msg = format!("{}", e);
+                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
+                    }
+                };
+                if setting.id != id {
+                    setting.id = id;
+                    serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap();
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+pub fn make_tag_subfolders(root: impl AsRef<Path>) -> std::io::Result<()> {
+    for dir_entry in root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(USER_ID_FOLDER)
+        .read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.metadata()?.is_dir() {
+            let tag_folder = dir_entry.path().join(TAG_FOLDER);
+            if !tag_folder.exists() {
+                std::fs::create_dir(&tag_folder)?;
+            }
+        }
+    }
+    for dir_entry in root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(APP_ID_FOLDER)
+        .read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.metadata()?.is_dir() {
+            let tag_folder = dir_entry.path().join(TAG_FOLDER);
+            if !tag_folder.exists() {
+                std::fs::create_dir(&tag_folder)?;
+            }
+        }
+    }
+    // TODO populate folders
+    Ok(())
+}

backend/src/settings/steam_deck/util.rs

@@ -31,7 +31,7 @@ const THINGS: &[u8] = &[
     0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0,
 ];
 
-const TIME_UNIT: std::time::Duration = std::time::Duration::from_millis(200);
+const TIME_UNIT: std::time::Duration = std::time::Duration::from_millis(250);
 
 pub fn flash_led() {
     use smokepatio::ec::ControllerSet;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "PowerTools",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "Power tweaks for power users",
   "scripts": {
     "build": "shx rm -rf dist && rollup -c",

src/store/page.tsx

@@ -40,18 +40,22 @@ export class StoreResultsPage extends Component<{onNewVariant: () => void}> {
                     { tr("No results") /* TODO translate */ }
                 </Focusable>);
             } else {
-                // TODO
+                return (
-                return (<Focusable
+                <div style={{
-                    style={{
+                    marginTop: "40px",
-                        display: "flex",
+                    marginBottom: "40px",
-                        flexWrap: "wrap",
+                    overflowY: "scroll",
-                        justifyContent: "center",
+                }}>
-                        rowGap: "5px",
+                    <Focusable
-                        columnGap: "5px",
+                        style={{
-                        maxWidth: "100%",
+                            display: "flex",
-                        margin: "2em 0.5em",
+                            flexWrap: "wrap",
-                    }}
+                            justifyContent: "center",
-                >
+                            rowGap: "0.5em",
+                            columnGap: "0.5em",
+                            maxWidth: "100%",
+                        }}
+                    >
                     {
                         storeItems.map((meta: backend.StoreMetadata) => (<PanelSectionRow>
                                 <Focusable style={{
@@ -149,7 +153,8 @@ export class StoreResultsPage extends Component<{onNewVariant: () => void}> {
                                 </Focusable>
                             </PanelSectionRow>))
                     }
-                    </Focusable>);
+                    </Focusable>
+                </div>);
             }
 
         } else {