Move filesystem fixes and improvements to separate file

backend/community_settings_srv/src/api/get_game.rs

@@ -60,7 +60,7 @@ fn get_some_settings_by_app_id(steam_app_id: u32, cli: &'static Cli) -> std::io:
     let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, steam_app_id);
     let mut files: Vec<_> = app_id_folder.read_dir()?
                     .filter_map(|res| res.ok())
-                    .filter(|f| f.path().extension().map(|ext| ext == file_util::RON_EXTENSION).unwrap_or(false))
+                    .filter(|f| f.path().extension().map(|ext| ext == crate::consts::RON_EXTENSION).unwrap_or(false))
                     .filter_map(|f| f.metadata().ok().map(|meta| (f, meta)))
                     .filter_map(|(f, meta)| meta.modified().ok().map(|time| (f, meta, time)))
                     .collect();

backend/community_settings_srv/src/api/get_setting.rs

@@ -67,7 +67,7 @@ pub async fn get_setting_handler(
     if super::is_mime_type_ron_capable(&preferred) {
         // Send RON
         let ron = if id != 0 {
-            let path = file_util::setting_path_by_id(&cli.folder, id, file_util::RON_EXTENSION);
+            let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::RON_EXTENSION);
             if !path.exists() {
                 return Err(std::io::Error::new(std::io::ErrorKind::NotFound, format!("setting id {} does not exist", id)));
             }
@@ -93,7 +93,7 @@ pub async fn get_setting_handler(
     } else {
         // Send JSON (fallback)
         let json = if id != 0 {
-            let path = file_util::setting_path_by_id(&cli.folder, id, file_util::JSON_EXTENSION);
+            let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::JSON_EXTENSION);
             // TODO? cache this instead of always loading it from file
             let reader = std::io::BufReader::new(std::fs::File::open(path)?);
             match serde_json::from_reader(reader) {

backend/community_settings_srv/src/api/save_setting.rs

@@ -46,14 +46,14 @@ pub async fn save_setting_handler(
     parsed_data.id = next_id.to_string();
     // TODO validate user and app id
     // Reject blocked users and apps
-    let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, file_util::RON_EXTENSION);
+    let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::RON_EXTENSION);
     let writer = std::io::BufWriter::new(std::fs::File::create(&path_ron)?);
     if let Err(e) = ron::ser::to_writer(writer, &parsed_data) {
         let e_msg = format!("{}", e);
         return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
     }
 
-    let path_json = file_util::setting_path_by_id(&cli.folder, next_id, file_util::JSON_EXTENSION);
+    let path_json = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::JSON_EXTENSION);
     let writer = std::io::BufWriter::new(std::fs::File::create(&path_json)?);
     if let Err(e) = serde_json::to_writer(writer, &parsed_data) {
         let e_msg = format!("{}", e);
@@ -67,14 +67,15 @@ pub async fn save_setting_handler(
     log::debug!("Saved to {}, building symlinks", path_ron.display());
 
     // create symlinks for other ways of looking up these settings files
-    let filename_ron = file_util::filename(next_id, file_util::RON_EXTENSION);
+    let filename_ron = file_util::filename(next_id, crate::consts::RON_EXTENSION);
-    let filename_json = file_util::filename(next_id, file_util::JSON_EXTENSION);
+    let filename_json = file_util::filename(next_id, crate::consts::JSON_EXTENSION);
 
     // create symlinks to app id folder
     let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, parsed_data.steam_app_id);
     log::debug!("App id folder {}", app_id_folder.display());
     if !app_id_folder.exists() {
         std::fs::create_dir(&app_id_folder)?;
+        std::fs::create_dir(file_util::setting_tag_folder_by_app_id(&cli.folder, parsed_data.steam_app_id))?;
     }
     #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
     {
@@ -93,6 +94,7 @@ pub async fn save_setting_handler(
     let user_id_folder = file_util::setting_folder_by_user_id(&cli.folder, parsed_data.steam_user_id);
     if !user_id_folder.exists() {
         std::fs::create_dir(&user_id_folder)?;
+        std::fs::create_dir(file_util::setting_tag_folder_by_user_id(&cli.folder, parsed_data.steam_user_id))?;
     }
     #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
     {
@@ -133,7 +135,7 @@ pub async fn save_setting_handler(
         // create symlinks for app id tag folder
         let app_tag_folder = file_util::setting_folder_by_app_id_tag(&cli.folder, parsed_data.steam_app_id, tag);
         if !app_tag_folder.exists() {
-            std::fs::create_dir_all(&app_tag_folder)?;
+            std::fs::create_dir(&app_tag_folder)?;
         }
         #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
         {
@@ -151,7 +153,7 @@ pub async fn save_setting_handler(
         // create symlinks for user id tag folder
         let user_tag_folder = file_util::setting_folder_by_user_id_tag(&cli.folder, parsed_data.steam_user_id, tag);
         if !user_tag_folder.exists() {
-            std::fs::create_dir_all(&user_tag_folder)?;
+            std::fs::create_dir(&user_tag_folder)?;
         }
         #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
         {

backend/community_settings_srv/src/consts.rs

@@ -0,0 +1,8 @@
+pub const RON_EXTENSION: &'static str = "ron";
+pub const JSON_EXTENSION: &'static str = "json";
+
+pub const SETTING_FOLDER: &'static str = "settings";
+pub const ID_FOLDER: &'static str = "by_id";
+pub const APP_ID_FOLDER: &'static str = "by_app_id";
+pub const USER_ID_FOLDER: &'static str = "by_user_id";
+pub const TAG_FOLDER: &'static str = "by_tag";

backend/community_settings_srv/src/file_util.rs

@@ -1,150 +1,10 @@
 use std::path::{Path, PathBuf};
 use std::sync::Mutex;
 
-pub const RON_EXTENSION: &'static str = "ron";
+use crate::consts::*;
-pub const JSON_EXTENSION: &'static str = "json";
-
-const SETTING_FOLDER: &'static str = "settings";
-const ID_FOLDER: &'static str = "by_id";
-const APP_ID_FOLDER: &'static str = "by_app_id";
-const USER_ID_FOLDER: &'static str = "by_user_id";
-const TAG_FOLDER: &'static str = "by_tag";
 
 static LAST_SETTING_ID: Mutex<u128> = Mutex::new(0);
 
-pub fn build_folder_layout(root: impl AsRef<Path>) -> std::io::Result<()> {
-    std::fs::create_dir_all(
-        root.as_ref()
-            .join(SETTING_FOLDER)
-            .join(ID_FOLDER)
-    )?;
-    std::fs::create_dir_all(
-        root.as_ref()
-            .join(SETTING_FOLDER)
-            .join(APP_ID_FOLDER)
-    )?;
-    std::fs::create_dir_all(
-        root.as_ref()
-            .join(SETTING_FOLDER)
-            .join(USER_ID_FOLDER)
-    )?;
-    std::fs::create_dir_all(
-        root.as_ref()
-            .join(SETTING_FOLDER)
-            .join(TAG_FOLDER)
-    )?;
-    Ok(())
-}
-
-pub fn fix_symlinks(root: impl AsRef<Path>) -> std::io::Result<()> {
-    log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display());
-    for dir_entry in root.as_ref()
-            .join(SETTING_FOLDER)
-            .join(APP_ID_FOLDER)
-            .read_dir()? {
-        let dir_entry = dir_entry?;
-        if dir_entry.file_type()?.is_dir() {
-            make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
-        }
-    }
-    for dir_entry in root.as_ref()
-            .join(SETTING_FOLDER)
-            .join(USER_ID_FOLDER)
-            .read_dir()? {
-        let dir_entry = dir_entry?;
-        if dir_entry.file_type()?.is_dir() {
-            make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
-        }
-    }
-    for dir_entry in root.as_ref()
-            .join(SETTING_FOLDER)
-            .join(TAG_FOLDER).read_dir()? {
-        let dir_entry = dir_entry?;
-        if dir_entry.file_type()?.is_dir() {
-            make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
-        }
-    }
-
-    Ok(())
-}
-
-fn make_symlinks_absolute_in_dir(root: impl AsRef<Path>, dir: impl AsRef<Path>) -> std::io::Result<()> {
-    let abs_root = root.as_ref().canonicalize()?;
-    assert!(abs_root.is_absolute());
-    for dir_entry in dir.as_ref()
-            .read_dir()? {
-        let dir_entry = dir_entry?;
-        if dir_entry.file_type()?.is_symlink() {
-            let path = dir_entry.path();
-            let link_path = path.read_link()?;
-            if !link_path.is_absolute() {
-                let new_link = abs_root.join(
-                    link_path.strip_prefix(&root).expect("Symlinked path does not begin with root settings folder")
-                );
-                log::info!("Fixing {} -> {} to -> {}", path.display(), link_path.display(), new_link.display());
-                std::fs::remove_file(&path)?;
-                #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
-                {
-                    std::os::windows::fs::symlink_file(new_link, &path)?;
-                }
-                #[cfg(target_family = "unix")]
-                {
-                    std::os::unix::fs::symlink(new_link, &path)?;
-                }
-            }else {
-                log::info!("Found already-absolute symlink {} -> {}", path.display(), link_path.display());
-            }
-        } else {
-            log::info!("Found non-symlink {}: {:?}", dir_entry.path().display(), dir_entry.file_type()?);
-        }
-    }
-    Ok(())
-}
-
-pub fn sync_ids(root: impl AsRef<Path>) -> std::io::Result<()> {
-    for dir_entry in root.as_ref()
-        .join(SETTING_FOLDER)
-        .join(ID_FOLDER)
-        .read_dir()? {
-        let dir_entry = dir_entry?;
-        let f_path = dir_entry.path();
-        if let Some(ext) = f_path.extension() {
-            let id = f_path.file_stem().map(|os| os.to_string_lossy().to_string()).unwrap();
-            if ext == RON_EXTENSION {
-                let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
-                let mut setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) {
-                    Ok(x) => x,
-                    Err(e) => {
-                        log::debug!("Error while reading {}: {}", f_path.display(), e);
-                        let e_msg = format!("{}", e);
-                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
-                    }
-                };
-                if setting.id != id {
-                    setting.id = id;
-                    ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap();
-                }
-            } else if ext == JSON_EXTENSION {
-                let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
-                let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) {
-                    Ok(x) => x,
-                    Err(e) => {
-                        log::debug!("Error while reading {}: {}", f_path.display(), e);
-                        let e_msg = format!("{}", e);
-                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
-                    }
-                };
-                if setting.id != id {
-                    setting.id = id;
-                    serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap();
-                }
-            }
-        }
-
-    }
-    Ok(())
-}
-
 pub fn filename(id: u128, ext: &str) -> String {
     format!("{}.{}", id, ext)
 }
@@ -163,6 +23,14 @@ pub fn setting_folder_by_app_id(root: impl AsRef<Path>, steam_app_id: u32) -> Pa
         .join(steam_app_id.to_string())
 }
 
+pub fn setting_tag_folder_by_app_id(root: impl AsRef<Path>, steam_app_id: u32) -> PathBuf {
+    root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(APP_ID_FOLDER)
+        .join(steam_app_id.to_string())
+        .join(TAG_FOLDER)
+}
+
 pub fn setting_folder_by_app_id_tag(root: impl AsRef<Path>, steam_app_id: u32, tag: &str) -> PathBuf {
     root.as_ref()
         .join(SETTING_FOLDER)
@@ -179,6 +47,14 @@ pub fn setting_folder_by_user_id(root: impl AsRef<Path>, steam_user_id: u64) ->
         .join(steam_user_id.to_string())
 }
 
+pub fn setting_tag_folder_by_user_id(root: impl AsRef<Path>, steam_user_id: u64) -> PathBuf {
+    root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(USER_ID_FOLDER)
+        .join(steam_user_id.to_string())
+        .join(TAG_FOLDER)
+}
+
 pub fn setting_folder_by_user_id_tag(root: impl AsRef<Path>, steam_user_id: u64, tag: &str) -> PathBuf {
     root.as_ref()
         .join(SETTING_FOLDER)

backend/community_settings_srv/src/main.rs

@@ -1,6 +1,8 @@
 mod api;
 mod cli;
+mod consts;
 mod file_util;
+mod upgrade;
 
 use actix_web::{web, App, HttpServer};
 
@@ -27,14 +29,16 @@ async fn main() -> std::io::Result<()> {
 
     // setup
     log::debug!("Building folder layout (if not exists) at: {}", &args.folder.display());
-    file_util::build_folder_layout(&args.folder)?;
+    upgrade::build_folder_layout(&args.folder)?;
 
     // fix things
     if args.fix {
         log::info!("Fixing old symlinks");
-        file_util::fix_symlinks(&args.folder)?;
+        upgrade::fix_symlinks(&args.folder)?;
+        log::info!("Creating missing by_tag folders");
+        upgrade::make_tag_subfolders(&args.folder)?;
         log::info!("Resynchronizing file IDs with file name IDs");
-        file_util::sync_ids(&args.folder)?;
+        upgrade::sync_ids(&args.folder)?;
         return Ok(())
     }
 

backend/community_settings_srv/src/upgrade.rs

@@ -0,0 +1,164 @@
+use std::path::Path;
+
+use crate::consts::*;
+
+pub fn build_folder_layout(root: impl AsRef<Path>) -> std::io::Result<()> {
+    std::fs::create_dir_all(
+        root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(ID_FOLDER)
+    )?;
+    std::fs::create_dir_all(
+        root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(APP_ID_FOLDER)
+    )?;
+    std::fs::create_dir_all(
+        root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(USER_ID_FOLDER)
+    )?;
+    std::fs::create_dir_all(
+        root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(TAG_FOLDER)
+    )?;
+    Ok(())
+}
+
+pub fn fix_symlinks(root: impl AsRef<Path>) -> std::io::Result<()> {
+    log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display());
+    for dir_entry in root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(APP_ID_FOLDER)
+            .read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.file_type()?.is_dir() {
+            make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
+        }
+    }
+    for dir_entry in root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(USER_ID_FOLDER)
+            .read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.file_type()?.is_dir() {
+            make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
+        }
+    }
+    for dir_entry in root.as_ref()
+            .join(SETTING_FOLDER)
+            .join(TAG_FOLDER).read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.file_type()?.is_dir() {
+            make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
+        }
+    }
+
+    Ok(())
+}
+
+fn make_symlinks_absolute_in_dir(root: impl AsRef<Path>, dir: impl AsRef<Path>) -> std::io::Result<()> {
+    let abs_root = root.as_ref().canonicalize()?;
+    assert!(abs_root.is_absolute());
+    for dir_entry in dir.as_ref()
+            .read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.file_type()?.is_symlink() {
+            let path = dir_entry.path();
+            let link_path = path.read_link()?;
+            if !link_path.is_absolute() {
+                let new_link = abs_root.join(
+                    link_path.strip_prefix(&root).expect("Symlinked path does not begin with root settings folder")
+                );
+                log::info!("Fixing {} -> {} to -> {}", path.display(), link_path.display(), new_link.display());
+                std::fs::remove_file(&path)?;
+                #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
+                {
+                    std::os::windows::fs::symlink_file(new_link, &path)?;
+                }
+                #[cfg(target_family = "unix")]
+                {
+                    std::os::unix::fs::symlink(new_link, &path)?;
+                }
+            }else {
+                log::info!("Found already-absolute symlink {} -> {}", path.display(), link_path.display());
+            }
+        } else {
+            log::info!("Found non-symlink {}: {:?}", dir_entry.path().display(), dir_entry.file_type()?);
+        }
+    }
+    Ok(())
+}
+
+pub fn sync_ids(root: impl AsRef<Path>) -> std::io::Result<()> {
+    for dir_entry in root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(ID_FOLDER)
+        .read_dir()? {
+        let dir_entry = dir_entry?;
+        let f_path = dir_entry.path();
+        if let Some(ext) = f_path.extension() {
+            let id = f_path.file_stem().map(|os| os.to_string_lossy().to_string()).unwrap();
+            if ext == RON_EXTENSION {
+                let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
+                let mut setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) {
+                    Ok(x) => x,
+                    Err(e) => {
+                        log::debug!("Error while reading {}: {}", f_path.display(), e);
+                        let e_msg = format!("{}", e);
+                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
+                    }
+                };
+                if setting.id != id {
+                    setting.id = id;
+                    ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap();
+                }
+            } else if ext == JSON_EXTENSION {
+                let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
+                let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) {
+                    Ok(x) => x,
+                    Err(e) => {
+                        log::debug!("Error while reading {}: {}", f_path.display(), e);
+                        let e_msg = format!("{}", e);
+                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
+                    }
+                };
+                if setting.id != id {
+                    setting.id = id;
+                    serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap();
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+pub fn make_tag_subfolders(root: impl AsRef<Path>) -> std::io::Result<()> {
+    for dir_entry in root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(USER_ID_FOLDER)
+        .read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.metadata()?.is_dir() {
+            let tag_folder = dir_entry.path().join(TAG_FOLDER);
+            if !tag_folder.exists() {
+                std::fs::create_dir(&tag_folder)?;
+            }
+        }
+    }
+    for dir_entry in root.as_ref()
+        .join(SETTING_FOLDER)
+        .join(APP_ID_FOLDER)
+        .read_dir()? {
+        let dir_entry = dir_entry?;
+        if dir_entry.metadata()?.is_dir() {
+            let tag_folder = dir_entry.path().join(TAG_FOLDER);
+            if !tag_folder.exists() {
+                std::fs::create_dir(&tag_folder)?;
+            }
+        }
+    }
+    // TODO populate folders
+    Ok(())
+}