not_a_segfault/PowerTools
1
0
Fork 0
forked from NG-SD-Plugins/PowerTools
Code Pull requests Activity

Move filesystem fixes and improvements to separate file

Browse source
This commit is contained in:
NGnius (Graham) 2024-04-19 22:18:06 -04:00
parent 81b9107bf5
commit c1064fdf92
7 changed files with 207 additions and 153 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/community_settings_srv/src/api/get_game.rs
View file

@ -60,7 +60,7 @@ fn get_some_settings_by_app_id(steam_app_id: u32, cli: &'static Cli) -> std::io:
let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, steam_app_id);
let mut files: Vec<_> = app_id_folder.read_dir()?
.filter_map(|res| res.ok())
.filter(|f| f.path().extension().map(|ext| ext == file_util::RON_EXTENSION).unwrap_or(false))
.filter(|f| f.path().extension().map(|ext| ext == crate::consts::RON_EXTENSION).unwrap_or(false))
.filter_map(|f| f.metadata().ok().map(|meta| (f, meta)))
.filter_map(|(f, meta)| meta.modified().ok().map(|time| (f, meta, time)))
.collect();

4
backend/community_settings_srv/src/api/get_setting.rs
View file

@ -67,7 +67,7 @@ pub async fn get_setting_handler(
if super::is_mime_type_ron_capable(&preferred) {
// Send RON
let ron = if id != 0 {
let path = file_util::setting_path_by_id(&cli.folder, id, file_util::RON_EXTENSION);
let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::RON_EXTENSION);
if !path.exists() {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, format!("setting id {} does not exist", id)));
}
@ -93,7 +93,7 @@ pub async fn get_setting_handler(
} else {
// Send JSON (fallback)
let json = if id != 0 {
let path = file_util::setting_path_by_id(&cli.folder, id, file_util::JSON_EXTENSION);
let path = file_util::setting_path_by_id(&cli.folder, id, crate::consts::JSON_EXTENSION);
// TODO? cache this instead of always loading it from file
let reader = std::io::BufReader::new(std::fs::File::open(path)?);
match serde_json::from_reader(reader) {

14
backend/community_settings_srv/src/api/save_setting.rs
View file

@ -46,14 +46,14 @@ pub async fn save_setting_handler(
parsed_data.id = next_id.to_string();
// TODO validate user and app id
// Reject blocked users and apps
let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, file_util::RON_EXTENSION);
let path_ron = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::RON_EXTENSION);
let writer = std::io::BufWriter::new(std::fs::File::create(&path_ron)?);
if let Err(e) = ron::ser::to_writer(writer, &parsed_data) {
let e_msg = format!("{}", e);
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
}
let path_json = file_util::setting_path_by_id(&cli.folder, next_id, file_util::JSON_EXTENSION);
let path_json = file_util::setting_path_by_id(&cli.folder, next_id, crate::consts::JSON_EXTENSION);
let writer = std::io::BufWriter::new(std::fs::File::create(&path_json)?);
if let Err(e) = serde_json::to_writer(writer, &parsed_data) {
let e_msg = format!("{}", e);
@ -67,14 +67,15 @@ pub async fn save_setting_handler(
log::debug!("Saved to {}, building symlinks", path_ron.display());
// create symlinks for other ways of looking up these settings files
let filename_ron = file_util::filename(next_id, file_util::RON_EXTENSION);
let filename_json = file_util::filename(next_id, file_util::JSON_EXTENSION);
let filename_ron = file_util::filename(next_id, crate::consts::RON_EXTENSION);
let filename_json = file_util::filename(next_id, crate::consts::JSON_EXTENSION);
// create symlinks to app id folder
let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, parsed_data.steam_app_id);
log::debug!("App id folder {}", app_id_folder.display());
if !app_id_folder.exists() {
std::fs::create_dir(&app_id_folder)?;
std::fs::create_dir(file_util::setting_tag_folder_by_app_id(&cli.folder, parsed_data.steam_app_id))?;
}
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
@ -93,6 +94,7 @@ pub async fn save_setting_handler(
let user_id_folder = file_util::setting_folder_by_user_id(&cli.folder, parsed_data.steam_user_id);
if !user_id_folder.exists() {
std::fs::create_dir(&user_id_folder)?;
std::fs::create_dir(file_util::setting_tag_folder_by_user_id(&cli.folder, parsed_data.steam_user_id))?;
}
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
@ -133,7 +135,7 @@ pub async fn save_setting_handler(
// create symlinks for app id tag folder
let app_tag_folder = file_util::setting_folder_by_app_id_tag(&cli.folder, parsed_data.steam_app_id, tag);
if !app_tag_folder.exists() {
std::fs::create_dir_all(&app_tag_folder)?;
std::fs::create_dir(&app_tag_folder)?;
}
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
@ -151,7 +153,7 @@ pub async fn save_setting_handler(
// create symlinks for user id tag folder
let user_tag_folder = file_util::setting_folder_by_user_id_tag(&cli.folder, parsed_data.steam_user_id, tag);
if !user_tag_folder.exists() {
std::fs::create_dir_all(&user_tag_folder)?;
std::fs::create_dir(&user_tag_folder)?;
}
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{

8
backend/community_settings_srv/src/consts.rs Normal file
View file

@ -0,0 +1,8 @@
pub const RON_EXTENSION: &'static str = "ron";
pub const JSON_EXTENSION: &'static str = "json";
pub const SETTING_FOLDER: &'static str = "settings";
pub const ID_FOLDER: &'static str = "by_id";
pub const APP_ID_FOLDER: &'static str = "by_app_id";
pub const USER_ID_FOLDER: &'static str = "by_user_id";
pub const TAG_FOLDER: &'static str = "by_tag";

158
backend/community_settings_srv/src/file_util.rs
View file

@ -1,150 +1,10 @@
use std::path::{Path, PathBuf};
use std::sync::Mutex;
pub const RON_EXTENSION: &'static str = "ron";
pub const JSON_EXTENSION: &'static str = "json";
const SETTING_FOLDER: &'static str = "settings";
const ID_FOLDER: &'static str = "by_id";
const APP_ID_FOLDER: &'static str = "by_app_id";
const USER_ID_FOLDER: &'static str = "by_user_id";
const TAG_FOLDER: &'static str = "by_tag";
use crate::consts::*;
static LAST_SETTING_ID: Mutex<u128> = Mutex::new(0);
pub fn build_folder_layout(root: impl AsRef<Path>) -> std::io::Result<()> {
std::fs::create_dir_all(
root.as_ref()
.join(SETTING_FOLDER)
.join(ID_FOLDER)
)?;
std::fs::create_dir_all(
root.as_ref()
.join(SETTING_FOLDER)
.join(APP_ID_FOLDER)
)?;
std::fs::create_dir_all(
root.as_ref()
.join(SETTING_FOLDER)
.join(USER_ID_FOLDER)
)?;
std::fs::create_dir_all(
root.as_ref()
.join(SETTING_FOLDER)
.join(TAG_FOLDER)
)?;
Ok(())
}
pub fn fix_symlinks(root: impl AsRef<Path>) -> std::io::Result<()> {
log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display());
for dir_entry in root.as_ref()
.join(SETTING_FOLDER)
.join(APP_ID_FOLDER)
.read_dir()? {
let dir_entry = dir_entry?;
if dir_entry.file_type()?.is_dir() {
make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
}
}
for dir_entry in root.as_ref()
.join(SETTING_FOLDER)
.join(USER_ID_FOLDER)
.read_dir()? {
let dir_entry = dir_entry?;
if dir_entry.file_type()?.is_dir() {
make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
}
}
for dir_entry in root.as_ref()
.join(SETTING_FOLDER)
.join(TAG_FOLDER).read_dir()? {
let dir_entry = dir_entry?;
if dir_entry.file_type()?.is_dir() {
make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
}
}
Ok(())
}
fn make_symlinks_absolute_in_dir(root: impl AsRef<Path>, dir: impl AsRef<Path>) -> std::io::Result<()> {
let abs_root = root.as_ref().canonicalize()?;
assert!(abs_root.is_absolute());
for dir_entry in dir.as_ref()
.read_dir()? {
let dir_entry = dir_entry?;
if dir_entry.file_type()?.is_symlink() {
let path = dir_entry.path();
let link_path = path.read_link()?;
if !link_path.is_absolute() {
let new_link = abs_root.join(
link_path.strip_prefix(&root).expect("Symlinked path does not begin with root settings folder")
);
log::info!("Fixing {} -> {} to -> {}", path.display(), link_path.display(), new_link.display());
std::fs::remove_file(&path)?;
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
std::os::windows::fs::symlink_file(new_link, &path)?;
}
#[cfg(target_family = "unix")]
{
std::os::unix::fs::symlink(new_link, &path)?;
}
}else {
log::info!("Found already-absolute symlink {} -> {}", path.display(), link_path.display());
}
} else {
log::info!("Found non-symlink {}: {:?}", dir_entry.path().display(), dir_entry.file_type()?);
}
}
Ok(())
}
pub fn sync_ids(root: impl AsRef<Path>) -> std::io::Result<()> {
for dir_entry in root.as_ref()
.join(SETTING_FOLDER)
.join(ID_FOLDER)
.read_dir()? {
let dir_entry = dir_entry?;
let f_path = dir_entry.path();
if let Some(ext) = f_path.extension() {
let id = f_path.file_stem().map(|os| os.to_string_lossy().to_string()).unwrap();
if ext == RON_EXTENSION {
let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
let mut setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) {
Ok(x) => x,
Err(e) => {
log::debug!("Error while reading {}: {}", f_path.display(), e);
let e_msg = format!("{}", e);
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
}
};
if setting.id != id {
setting.id = id;
ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap();
}
} else if ext == JSON_EXTENSION {
let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) {
Ok(x) => x,
Err(e) => {
log::debug!("Error while reading {}: {}", f_path.display(), e);
let e_msg = format!("{}", e);
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
}
};
if setting.id != id {
setting.id = id;
serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap();
}
}
}
}
Ok(())
}
pub fn filename(id: u128, ext: &str) -> String {
format!("{}.{}", id, ext)
}
@ -163,6 +23,14 @@ pub fn setting_folder_by_app_id(root: impl AsRef<Path>, steam_app_id: u32) -> Pa
.join(steam_app_id.to_string())
}
pub fn setting_tag_folder_by_app_id(root: impl AsRef<Path>, steam_app_id: u32) -> PathBuf {
root.as_ref()
.join(SETTING_FOLDER)
.join(APP_ID_FOLDER)
.join(steam_app_id.to_string())
.join(TAG_FOLDER)
}
pub fn setting_folder_by_app_id_tag(root: impl AsRef<Path>, steam_app_id: u32, tag: &str) -> PathBuf {
root.as_ref()
.join(SETTING_FOLDER)
@ -179,6 +47,14 @@ pub fn setting_folder_by_user_id(root: impl AsRef<Path>, steam_user_id: u64) ->
.join(steam_user_id.to_string())
}
pub fn setting_tag_folder_by_user_id(root: impl AsRef<Path>, steam_user_id: u64) -> PathBuf {
root.as_ref()
.join(SETTING_FOLDER)
.join(USER_ID_FOLDER)
.join(steam_user_id.to_string())
.join(TAG_FOLDER)
}
pub fn setting_folder_by_user_id_tag(root: impl AsRef<Path>, steam_user_id: u64, tag: &str) -> PathBuf {
root.as_ref()
.join(SETTING_FOLDER)

10
backend/community_settings_srv/src/main.rs
View file

@ -1,6 +1,8 @@
mod api;
mod cli;
mod consts;
mod file_util;
mod upgrade;
use actix_web::{web, App, HttpServer};
@ -27,14 +29,16 @@ async fn main() -> std::io::Result<()> {
// setup
log::debug!("Building folder layout (if not exists) at: {}", &args.folder.display());
file_util::build_folder_layout(&args.folder)?;
upgrade::build_folder_layout(&args.folder)?;
// fix things
if args.fix {
log::info!("Fixing old symlinks");
file_util::fix_symlinks(&args.folder)?;
upgrade::fix_symlinks(&args.folder)?;
log::info!("Creating missing by_tag folders");
upgrade::make_tag_subfolders(&args.folder)?;
log::info!("Resynchronizing file IDs with file name IDs");
file_util::sync_ids(&args.folder)?;
upgrade::sync_ids(&args.folder)?;
return Ok(())
}

164
backend/community_settings_srv/src/upgrade.rs Normal file
View file

@ -0,0 +1,164 @@
use std::path::Path;
use crate::consts::*;
pub fn build_folder_layout(root: impl AsRef<Path>) -> std::io::Result<()> {
std::fs::create_dir_all(
root.as_ref()
.join(SETTING_FOLDER)
.join(ID_FOLDER)
)?;
std::fs::create_dir_all(
root.as_ref()
.join(SETTING_FOLDER)
.join(APP_ID_FOLDER)
)?;
std::fs::create_dir_all(
root.as_ref()
.join(SETTING_FOLDER)
.join(USER_ID_FOLDER)
)?;
std::fs::create_dir_all(
root.as_ref()
.join(SETTING_FOLDER)
.join(TAG_FOLDER)
)?;
Ok(())
}
pub fn fix_symlinks(root: impl AsRef<Path>) -> std::io::Result<()> {
log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display());
for dir_entry in root.as_ref()
.join(SETTING_FOLDER)
.join(APP_ID_FOLDER)
.read_dir()? {
let dir_entry = dir_entry?;
if dir_entry.file_type()?.is_dir() {
make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
}
}
for dir_entry in root.as_ref()
.join(SETTING_FOLDER)
.join(USER_ID_FOLDER)
.read_dir()? {
let dir_entry = dir_entry?;
if dir_entry.file_type()?.is_dir() {
make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
}
}
for dir_entry in root.as_ref()
.join(SETTING_FOLDER)
.join(TAG_FOLDER).read_dir()? {
let dir_entry = dir_entry?;
if dir_entry.file_type()?.is_dir() {
make_symlinks_absolute_in_dir(root.as_ref(), dir_entry.path())?;
}
}
Ok(())
}
fn make_symlinks_absolute_in_dir(root: impl AsRef<Path>, dir: impl AsRef<Path>) -> std::io::Result<()> {
let abs_root = root.as_ref().canonicalize()?;
assert!(abs_root.is_absolute());
for dir_entry in dir.as_ref()
.read_dir()? {
let dir_entry = dir_entry?;
if dir_entry.file_type()?.is_symlink() {
let path = dir_entry.path();
let link_path = path.read_link()?;
if !link_path.is_absolute() {
let new_link = abs_root.join(
link_path.strip_prefix(&root).expect("Symlinked path does not begin with root settings folder")
);
log::info!("Fixing {} -> {} to -> {}", path.display(), link_path.display(), new_link.display());
std::fs::remove_file(&path)?;
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
std::os::windows::fs::symlink_file(new_link, &path)?;
}
#[cfg(target_family = "unix")]
{
std::os::unix::fs::symlink(new_link, &path)?;
}
}else {
log::info!("Found already-absolute symlink {} -> {}", path.display(), link_path.display());
}
} else {
log::info!("Found non-symlink {}: {:?}", dir_entry.path().display(), dir_entry.file_type()?);
}
}
Ok(())
}
pub fn sync_ids(root: impl AsRef<Path>) -> std::io::Result<()> {
for dir_entry in root.as_ref()
.join(SETTING_FOLDER)
.join(ID_FOLDER)
.read_dir()? {
let dir_entry = dir_entry?;
let f_path = dir_entry.path();
if let Some(ext) = f_path.extension() {
let id = f_path.file_stem().map(|os| os.to_string_lossy().to_string()).unwrap();
if ext == RON_EXTENSION {
let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
let mut setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) {
Ok(x) => x,
Err(e) => {
log::debug!("Error while reading {}: {}", f_path.display(), e);
let e_msg = format!("{}", e);
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
}
};
if setting.id != id {
setting.id = id;
ron::ser::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap();
}
} else if ext == JSON_EXTENSION {
let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
let mut setting: community_settings_core::v1::Metadata = match serde_json::from_reader(reader) {
Ok(x) => x,
Err(e) => {
log::debug!("Error while reading {}: {}", f_path.display(), e);
let e_msg = format!("{}", e);
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
}
};
if setting.id != id {
setting.id = id;
serde_json::to_writer(std::fs::File::create(&f_path)?, &setting).unwrap();
}
}
}
}
Ok(())
}
pub fn make_tag_subfolders(root: impl AsRef<Path>) -> std::io::Result<()> {
for dir_entry in root.as_ref()
.join(SETTING_FOLDER)
.join(USER_ID_FOLDER)
.read_dir()? {
let dir_entry = dir_entry?;
if dir_entry.metadata()?.is_dir() {
let tag_folder = dir_entry.path().join(TAG_FOLDER);
if !tag_folder.exists() {
std::fs::create_dir(&tag_folder)?;
}
}
}
for dir_entry in root.as_ref()
.join(SETTING_FOLDER)
.join(APP_ID_FOLDER)
.read_dir()? {
let dir_entry = dir_entry?;
if dir_entry.metadata()?.is_dir() {
let tag_folder = dir_entry.path().join(TAG_FOLDER);
if !tag_folder.exists() {
std::fs::create_dir(&tag_folder)?;
}
}
}
// TODO populate folders
Ok(())
}