From c1064fdf9207a8552e1bb2129626a82b090d1416 Mon Sep 17 00:00:00 2001 From: "NGnius (Graham)" Date: Fri, 19 Apr 2024 22:18:06 -0400 Subject: [PATCH] Move filesystem fixes and improvements to separate file --- .../src/api/get_game.rs | 2 +- .../src/api/get_setting.rs | 4 +- .../src/api/save_setting.rs | 14 +- backend/community_settings_srv/src/consts.rs | 8 + .../community_settings_srv/src/file_util.rs | 158 ++--------------- backend/community_settings_srv/src/main.rs | 10 +- backend/community_settings_srv/src/upgrade.rs | 164 ++++++++++++++++++ 7 files changed, 207 insertions(+), 153 deletions(-) create mode 100644 backend/community_settings_srv/src/consts.rs create mode 100644 backend/community_settings_srv/src/upgrade.rs diff --git a/backend/community_settings_srv/src/api/get_game.rs b/backend/community_settings_srv/src/api/get_game.rs index 0b8a650..8e42556 100644 --- a/backend/community_settings_srv/src/api/get_game.rs +++ b/backend/community_settings_srv/src/api/get_game.rs @@ -60,7 +60,7 @@ fn get_some_settings_by_app_id(steam_app_id: u32, cli: &'static Cli) -> std::io: let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, steam_app_id); let mut files: Vec<_> = app_id_folder.read_dir()? .filter_map(|res| res.ok()) - .filter(|f| f.path().extension().map(|ext| ext == file_util::RON_EXTENSION).unwrap_or(false)) + .filter(|f| f.path().extension().map(|ext| ext == crate::consts::RON_EXTENSION).unwrap_or(false)) .filter_map(|f| f.metadata().ok().map(|meta| (f, meta))) .filter_map(|(f, meta)| meta.modified().ok().map(|time| (f, meta, time))) .collect(); diff --git a/backend/community_settings_srv/src/api/get_setting.rs b/backend/community_settings_srv/src/api/get_setting.rs index 3121192..9d3a05b 100644 --- a/backend/community_settings_srv/src/api/get_setting.rs +++ b/backend/community_settings_srv/src/api/get_setting.rs @@ -67,7 +67,7 @@ pub async fn get_setting_handler( if super::is_mime_type_ron_capable(&preferred) { // Send RON let ron = if id != 0 { - let path = file_util::setting_path_by_id(&cli.folder, id, file_util::RON_EXTENSION); + let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::RON_EXTENSION); if !path.exists() { return Err(std::io::Error::new(std::io::ErrorKind::NotFound, format!("setting id {} does not exist", id))); } @@ -93,7 +93,7 @@ pub async fn get_setting_handler( } else { // Send JSON (fallback) let json = if id != 0 { - let path = file_util::setting_path_by_id(&cli.folder, id, file_util::JSON_EXTENSION); + let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::JSON_EXTENSION); // TODO? cache this instead of always loading it from file let reader = std::io::BufReader::new(std::fs::File::open(path)?); match serde_json::from_reader(reader) { diff --git a/backend/community_settings_srv/src/api/save_setting.rs b/backend/community_settings_srv/src/api/save_setting.rs index bd4960e..a77ba58 100644 --- a/backend/community_settings_srv/src/api/save_setting.rs +++ b/backend/community_settings_srv/src/api/save_setting.rs @@ -46,14 +46,14 @@ pub async fn save_setting_handler( parsed_data.id = next_id.to_string(); // TODO validate user and app id // Reject blocked users and apps - let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, file_util::RON_EXTENSION); + let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::RON_EXTENSION); let writer = std::io::BufWriter::new(std::fs::File::create(&path_ron)?); if let Err(e) = ron::ser::to_writer(writer, &parsed_data) { let e_msg = format!("{}", e); return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); } - let path_json = file_util::setting_path_by_id(&cli.folder, next_id, file_util::JSON_EXTENSION); + let path_json = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::JSON_EXTENSION); let writer = std::io::BufWriter::new(std::fs::File::create(&path_json)?); if let Err(e) = serde_json::to_writer(writer, &parsed_data) { let e_msg = format!("{}", e); @@ -67,14 +67,15 @@ pub async fn save_setting_handler( log::debug!("Saved to {}, building symlinks", path_ron.display()); // create symlinks for other ways of looking up these settings files - let filename_ron = file_util::filename(next_id, file_util::RON_EXTENSION); - let filename_json = file_util::filename(next_id, file_util::JSON_EXTENSION); + let filename_ron = file_util::filename(next_id, crate::consts::RON_EXTENSION); + let filename_json = file_util::filename(next_id, crate::consts::JSON_EXTENSION); // create symlinks to app id folder let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, parsed_data.steam_app_id); log::debug!("App id folder {}", app_id_folder.display()); if !app_id_folder.exists() { std::fs::create_dir(&app_id_folder)?; + std::fs::create_dir(file_util::setting_tag_folder_by_app_id(&cli.folder, parsed_data.steam_app_id))?; } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { @@ -93,6 +94,7 @@ pub async fn save_setting_handler( let user_id_folder = file_util::setting_folder_by_user_id(&cli.folder, parsed_data.steam_user_id); if !user_id_folder.exists() { std::fs::create_dir(&user_id_folder)?; + std::fs::create_dir(file_util::setting_tag_folder_by_user_id(&cli.folder, parsed_data.steam_user_id))?; } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { @@ -133,7 +135,7 @@ pub async fn save_setting_handler( // create symlinks for app id tag folder let app_tag_folder = file_util::setting_folder_by_app_id_tag(&cli.folder, parsed_data.steam_app_id, tag); if !app_tag_folder.exists() { - std::fs::create_dir_all(&app_tag_folder)?; + std::fs::create_dir(&app_tag_folder)?; } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { @@ -151,7 +153,7 @@ pub async fn save_setting_handler( // create symlinks for user id tag folder let user_tag_folder = file_util::setting_folder_by_user_id_tag(&cli.folder, parsed_data.steam_user_id, tag); if !user_tag_folder.exists() { - std::fs::create_dir_all(&user_tag_folder)?; + std::fs::create_dir(&user_tag_folder)?; } #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained { diff --git a/backend/community_settings_srv/src/consts.rs b/backend/community_settings_srv/src/consts.rs new file mode 100644 index 0000000..833fd90 --- /dev/null +++ b/backend/community_settings_srv/src/consts.rs @@ -0,0 +1,8 @@ +pub const RON_EXTENSION: &'static str = "ron"; +pub const JSON_EXTENSION: &'static str = "json"; + +pub const SETTING_FOLDER: &'static str = "settings"; +pub const ID_FOLDER: &'static str = "by_id"; +pub const APP_ID_FOLDER: &'static str = "by_app_id"; +pub const USER_ID_FOLDER: &'static str = "by_user_id"; +pub const TAG_FOLDER: &'static str = "by_tag"; diff --git a/backend/community_settings_srv/src/file_util.rs b/backend/community_settings_srv/src/file_util.rs index 76f511b..5a7d4bd 100644 --- a/backend/community_settings_srv/src/file_util.rs +++ b/backend/community_settings_srv/src/file_util.rs @@ -1,150 +1,10 @@ use std::path::{Path, PathBuf}; use std::sync::Mutex; -pub const RON_EXTENSION: &'static str = "ron"; -pub const JSON_EXTENSION: &'static str = "json"; - -const SETTING_FOLDER: &'static str = "settings"; -const ID_FOLDER: &'static str = "by_id"; -const APP_ID_FOLDER: &'static str = "by_app_id"; -const USER_ID_FOLDER: &'static str = "by_user_id"; -const TAG_FOLDER: &'static str = "by_tag"; +use crate::consts::*; static LAST_SETTING_ID: Mutex = Mutex::new(0); -pub fn build_folder_layout(root: impl AsRef) -> std::io::Result<()> { - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(ID_FOLDER) - )?; - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(APP_ID_FOLDER) - )?; - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(USER_ID_FOLDER) - )?; - std::fs::create_dir_all( - root.as_ref() - .join(SETTING_FOLDER) - .join(TAG_FOLDER) - )?; - Ok(()) -} - -pub fn fix_symlinks(root: impl AsRef) -> std::io::Result<()> { - log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display()); - for dir_entry in root.as_ref() - .join(SETTING_FOLDER) - .join(APP_ID_FOLDER) - .read_dir()? { - let dir_entry = dir_entry?; - if dir_entry.file_type()?.is_dir() { - make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; - } - } - for dir_entry in root.as_ref() - .join(SETTING_FOLDER) - .join(USER_ID_FOLDER) - .read_dir()? { - let dir_entry = dir_entry?; - if dir_entry.file_type()?.is_dir() { - make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; - } - } - for dir_entry in root.as_ref() - .join(SETTING_FOLDER) - .join(TAG_FOLDER).read_dir()? { - let dir_entry = dir_entry?; - if dir_entry.file_type()?.is_dir() { - make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; - } - } - - Ok(()) -} - -fn make_symlinks_absolute_in_dir(root: impl AsRef, dir: impl AsRef) -> std::io::Result<()> { - let abs_root = root.as_ref().canonicalize()?; - assert!(abs_root.is_absolute()); - for dir_entry in dir.as_ref() - .read_dir()? { - let dir_entry = dir_entry?; - if dir_entry.file_type()?.is_symlink() { - let path = dir_entry.path(); - let link_path = path.read_link()?; - if !link_path.is_absolute() { - let new_link = abs_root.join( - link_path.strip_prefix(&root).expect("Symlinked path does not begin with root settings folder") - ); - log::info!("Fixing {} -> {} to -> {}", path.display(), link_path.display(), new_link.display()); - std::fs::remove_file(&path)?; - #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained - { - std::os::windows::fs::symlink_file(new_link, &path)?; - } - #[cfg(target_family = "unix")] - { - std::os::unix::fs::symlink(new_link, &path)?; - } - }else { - log::info!("Found already-absolute symlink {} -> {}", path.display(), link_path.display()); - } - } else { - log::info!("Found non-symlink {}: {:?}", dir_entry.path().display(), dir_entry.file_type()?); - } - } - Ok(()) -} - -pub fn sync_ids(root: impl AsRef) -> std::io::Result<()> { - for dir_entry in root.as_ref() - .join(SETTING_FOLDER) - .join(ID_FOLDER) - .read_dir()? { - let dir_entry = dir_entry?; - let f_path = dir_entry.path(); - if let Some(ext) = f_path.extension() { - let id = f_path.file_stem().map(|os| os.to_string_lossy().to_string()).unwrap(); - if ext == RON_EXTENSION { - let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); - let mut setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) { - Ok(x) => x, - Err(e) => { - log::debug!("Error while reading {}: {}", f_path.display(), e); - let e_msg = format!("{}", e); - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); - } - }; - if setting.id != id { - setting.id = id; - ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); - } - } else if ext == JSON_EXTENSION { - let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); - let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) { - Ok(x) => x, - Err(e) => { - log::debug!("Error while reading {}: {}", f_path.display(), e); - let e_msg = format!("{}", e); - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); - } - }; - if setting.id != id { - setting.id = id; - serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); - } - } - } - - } - Ok(()) -} - pub fn filename(id: u128, ext: &str) -> String { format!("{}.{}", id, ext) } @@ -163,6 +23,14 @@ pub fn setting_folder_by_app_id(root: impl AsRef, steam_app_id: u32) -> Pa .join(steam_app_id.to_string()) } +pub fn setting_tag_folder_by_app_id(root: impl AsRef, steam_app_id: u32) -> PathBuf { + root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .join(steam_app_id.to_string()) + .join(TAG_FOLDER) +} + pub fn setting_folder_by_app_id_tag(root: impl AsRef, steam_app_id: u32, tag: &str) -> PathBuf { root.as_ref() .join(SETTING_FOLDER) @@ -179,6 +47,14 @@ pub fn setting_folder_by_user_id(root: impl AsRef, steam_user_id: u64) -> .join(steam_user_id.to_string()) } +pub fn setting_tag_folder_by_user_id(root: impl AsRef, steam_user_id: u64) -> PathBuf { + root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .join(steam_user_id.to_string()) + .join(TAG_FOLDER) +} + pub fn setting_folder_by_user_id_tag(root: impl AsRef, steam_user_id: u64, tag: &str) -> PathBuf { root.as_ref() .join(SETTING_FOLDER) diff --git a/backend/community_settings_srv/src/main.rs b/backend/community_settings_srv/src/main.rs index 0f4e40c..58f132a 100644 --- a/backend/community_settings_srv/src/main.rs +++ b/backend/community_settings_srv/src/main.rs @@ -1,6 +1,8 @@ mod api; mod cli; +mod consts; mod file_util; +mod upgrade; use actix_web::{web, App, HttpServer}; @@ -27,14 +29,16 @@ async fn main() -> std::io::Result<()> { // setup log::debug!("Building folder layout (if not exists) at: {}", &args.folder.display()); - file_util::build_folder_layout(&args.folder)?; + upgrade::build_folder_layout(&args.folder)?; // fix things if args.fix { log::info!("Fixing old symlinks"); - file_util::fix_symlinks(&args.folder)?; + upgrade::fix_symlinks(&args.folder)?; + log::info!("Creating missing by_tag folders"); + upgrade::make_tag_subfolders(&args.folder)?; log::info!("Resynchronizing file IDs with file name IDs"); - file_util::sync_ids(&args.folder)?; + upgrade::sync_ids(&args.folder)?; return Ok(()) } diff --git a/backend/community_settings_srv/src/upgrade.rs b/backend/community_settings_srv/src/upgrade.rs new file mode 100644 index 0000000..c48f9b3 --- /dev/null +++ b/backend/community_settings_srv/src/upgrade.rs @@ -0,0 +1,164 @@ +use std::path::Path; + +use crate::consts::*; + +pub fn build_folder_layout(root: impl AsRef) -> std::io::Result<()> { + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(ID_FOLDER) + )?; + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + )?; + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + )?; + std::fs::create_dir_all( + root.as_ref() + .join(SETTING_FOLDER) + .join(TAG_FOLDER) + )?; + Ok(()) +} + +pub fn fix_symlinks(root: impl AsRef) -> std::io::Result<()> { + log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display()); + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(TAG_FOLDER).read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_dir() { + make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?; + } + } + + Ok(()) +} + +fn make_symlinks_absolute_in_dir(root: impl AsRef, dir: impl AsRef) -> std::io::Result<()> { + let abs_root = root.as_ref().canonicalize()?; + assert!(abs_root.is_absolute()); + for dir_entry in dir.as_ref() + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.file_type()?.is_symlink() { + let path = dir_entry.path(); + let link_path = path.read_link()?; + if !link_path.is_absolute() { + let new_link = abs_root.join( + link_path.strip_prefix(&root).expect("Symlinked path does not begin with root settings folder") + ); + log::info!("Fixing {} -> {} to -> {}", path.display(), link_path.display(), new_link.display()); + std::fs::remove_file(&path)?; + #[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained + { + std::os::windows::fs::symlink_file(new_link, &path)?; + } + #[cfg(target_family = "unix")] + { + std::os::unix::fs::symlink(new_link, &path)?; + } + }else { + log::info!("Found already-absolute symlink {} -> {}", path.display(), link_path.display()); + } + } else { + log::info!("Found non-symlink {}: {:?}", dir_entry.path().display(), dir_entry.file_type()?); + } + } + Ok(()) +} + +pub fn sync_ids(root: impl AsRef) -> std::io::Result<()> { + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + let f_path = dir_entry.path(); + if let Some(ext) = f_path.extension() { + let id = f_path.file_stem().map(|os| os.to_string_lossy().to_string()).unwrap(); + if ext == RON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + let mut setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) { + Ok(x) => x, + Err(e) => { + log::debug!("Error while reading {}: {}", f_path.display(), e); + let e_msg = format!("{}", e); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); + } + }; + if setting.id != id { + setting.id = id; + ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + } + } else if ext == JSON_EXTENSION { + let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?); + let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) { + Ok(x) => x, + Err(e) => { + log::debug!("Error while reading {}: {}", f_path.display(), e); + let e_msg = format!("{}", e); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg)); + } + }; + if setting.id != id { + setting.id = id; + serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap(); + } + } + } + } + Ok(()) +} + +pub fn make_tag_subfolders(root: impl AsRef) -> std::io::Result<()> { + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(USER_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.metadata()?.is_dir() { + let tag_folder = dir_entry.path().join(TAG_FOLDER); + if !tag_folder.exists() { + std::fs::create_dir(&tag_folder)?; + } + } + } + for dir_entry in root.as_ref() + .join(SETTING_FOLDER) + .join(APP_ID_FOLDER) + .read_dir()? { + let dir_entry = dir_entry?; + if dir_entry.metadata()?.is_dir() { + let tag_folder = dir_entry.path().join(TAG_FOLDER); + if !tag_folder.exists() { + std::fs::create_dir(&tag_folder)?; + } + } + } + // TODO populate folders + Ok(()) +}