NG-SD-Plugins/PowerTools
NG-SD-Plugins/PowerTools
4
4
Fork 3
Code Issues 18 Pull requests 1 Projects Releases 24 Wiki Activity

Add some deduplication and another symlink fixer

Browse source
This commit is contained in:
NGnius (Graham) 2024-04-21 12:34:11 -04:00
parent 6c28cef9b3
commit 22e916fe12
11 changed files with 380 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
backend/community_settings_srv/Cargo.lock generated
View file

@ -450,6 +450,7 @@ dependencies = [
"serde_json",
"simplelog",
"tokio",
"walkdir",
]
[[package]]
@ -1017,6 +1018,15 @@ version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "scopeguard"
version = "1.2.0"
@ -1319,6 +1329,16 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"

2
backend/community_settings_srv/Cargo.toml
View file

@ -16,6 +16,8 @@ tokio = { version = "1", features = ["full"] }
actix-web = { version = "4.4" }
mime = { version = "0.3.17" }
walkdir = "2"
# logging
log = "0.4"
simplelog = "0.12"

108
backend/community_settings_srv/src/api/save_setting.rs
View file

@ -66,116 +66,32 @@ pub async fn save_setting_handler(
log::debug!("Saved to {}, building symlinks", path_ron.display());
// create symlinks for other ways of looking up these settings files
let filename_ron = file_util::filename(next_id, crate::consts::RON_EXTENSION);
let filename_json = file_util::filename(next_id, crate::consts::JSON_EXTENSION);
let to_symlink = file_util::symlinks(&cli.folder, &parsed_data)?;
// create symlinks to app id folder
let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, parsed_data.steam_app_id);
log::debug!("App id folder {}", app_id_folder.display());
if !app_id_folder.exists() {
std::fs::create_dir(&app_id_folder)?;
std::fs::create_dir(file_util::setting_tag_folder_by_app_id(&cli.folder, parsed_data.steam_app_id))?;
}
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
log::debug!("Symlinking {} -> {}", app_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, app_id_folder.join(&filename_ron))?;
std::os::windows::fs::symlink_file(&path_json.canonicalize()?, app_id_folder.join(&filename_json))?;
}
#[cfg(target_family = "unix")]
{
log::debug!("Symlinking {} -> {}", app_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
std::os::unix::fs::symlink(&path_ron.canonicalize()?, app_id_folder.join(&filename_ron))?;
std::os::unix::fs::symlink(&path_json.canonicalize()?, app_id_folder.join(&filename_json))?;
}
// create symlinks for user id folder
let user_id_folder = file_util::setting_folder_by_user_id(&cli.folder, parsed_data.steam_user_id);
if !user_id_folder.exists() {
std::fs::create_dir(&user_id_folder)?;
std::fs::create_dir(file_util::setting_tag_folder_by_user_id(&cli.folder, parsed_data.steam_user_id))?;
}
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
log::debug!("Symlinking {} -> {}", user_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, user_id_folder.join(&filename_ron))?;
std::os::windows::fs::symlink_file(&path_json.canonicalize()?, user_id_folder.join(&filename_json))?;
}
#[cfg(target_family = "unix")]
{
log::debug!("Symlinking {} -> {}", user_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
std::os::unix::fs::symlink(&path_ron.canonicalize()?, user_id_folder.join(&filename_ron))?;
std::os::unix::fs::symlink(&path_json.canonicalize()?, user_id_folder.join(&filename_json))?;
}
// create symlinks for each tag
for tag in parsed_data.tags.iter() {
if !str_is_alphanumeric_or_space(&tag){
continue;
}
// create symlinks for general tag folder
let tag_folder = file_util::setting_folder_by_tag(&cli.folder, tag);
if !tag_folder.exists() {
std::fs::create_dir(&tag_folder)?;
}
let path_ron_canon = path_ron.canonicalize()?;
let path_json_canon = path_json.canonicalize()?;
for ron_link in to_symlink.ron {
log::debug!("Symlinking {} -> {}", ron_link.display(), path_ron_canon.display());
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
log::debug!("Symlinking {} -> {}", tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, tag_folder.join(&filename_ron))?;
std::os::windows::fs::symlink_file(&path_json.canonicalize()?, tag_folder.join(&filename_json))?;
std::os::windows::fs::symlink_file(&path_ron_canon, &ron_link)?;
}
#[cfg(target_family = "unix")]
{
log::debug!("Symlinking {} -> {}", tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
std::os::unix::fs::symlink(&path_ron.canonicalize()?, tag_folder.join(&filename_ron))?;
std::os::unix::fs::symlink(&path_json.canonicalize()?, tag_folder.join(&filename_json))?;
}
// create symlinks for app id tag folder
let app_tag_folder = file_util::setting_folder_by_app_id_tag(&cli.folder, parsed_data.steam_app_id, tag);
if !app_tag_folder.exists() {
std::fs::create_dir(&app_tag_folder)?;
std::os::unix::fs::symlink(&path_ron_canon, &ron_link)?;
}
}
for json_link in to_symlink.json {
//log::debug!("Symlinking {} -> {}", json_link.display(), path_json_canon.display());
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
log::debug!("Symlinking {} -> {}", app_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, app_tag_folder.join(&filename_ron))?;
std::os::windows::fs::symlink_file(&path_json.canonicalize()?, app_tag_folder.join(&filename_json))?;
std::os::windows::fs::symlink_file(&path_json_canon, json_link)?;
}
#[cfg(target_family = "unix")]
{
log::debug!("Symlinking {} -> {}", app_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
std::os::unix::fs::symlink(&path_ron.canonicalize()?, app_tag_folder.join(&filename_ron))?;
std::os::unix::fs::symlink(&path_json.canonicalize()?, app_tag_folder.join(&filename_json))?;
}
// create symlinks for user id tag folder
let user_tag_folder = file_util::setting_folder_by_user_id_tag(&cli.folder, parsed_data.steam_user_id, tag);
if !user_tag_folder.exists() {
std::fs::create_dir(&user_tag_folder)?;
}
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
log::debug!("Symlinking {} -> {}", user_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, user_tag_folder.join(&filename_ron))?;
std::os::windows::fs::symlink_file(&path_json.canonicalize()?, user_tag_folder.join(&filename_json))?;
}
#[cfg(target_family = "unix")]
{
log::debug!("Symlinking {} -> {}", user_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
std::os::unix::fs::symlink(&path_ron.canonicalize()?, user_tag_folder.join(&filename_ron))?;
std::os::unix::fs::symlink(&path_json.canonicalize()?, user_tag_folder.join(&filename_json))?;
std::os::unix::fs::symlink(&path_json_canon, json_link)?;
}
}
Ok(actix_web::HttpResponse::NoContent())
}
fn str_is_alphanumeric_or_space(s: &str) -> bool {
let mut result = true;
for ch in s.chars() {
result &= ch.is_ascii_alphanumeric() || ch == ' ';
}
result
}

4
backend/community_settings_srv/src/cli.rs
View file

@ -18,6 +18,10 @@ pub struct Cli {
/// Perform maintenance tasks
#[arg(long)]
pub fix: bool,
/// Keep up to this many duplicate settings (0 will delete everything!!!)
#[arg(long, default_value_t = 2)]
pub duplicates: usize,
}
impl Cli {

68
backend/community_settings_srv/src/file_util.rs
View file

@ -88,3 +88,71 @@ pub fn next_setting_id(root: impl AsRef<Path>) -> u128 {
*lock += 1;
*lock
}
pub struct ToSymlink {
pub ron: Vec<PathBuf>,
pub json: Vec<PathBuf>,
}
pub fn symlinks(root: impl AsRef<Path>, meta: &community_settings_core::v1::Metadata) -> std::io::Result<ToSymlink> {
let mut symlink_locations = ToSymlink { ron: Vec::new(), json: Vec::new() };
let filename_ron = filename(meta.get_id(), crate::consts::RON_EXTENSION);
let filename_json = filename(meta.get_id(), crate::consts::JSON_EXTENSION);
// build symlinks to app id folder
let app_id_folder = setting_folder_by_app_id(&root, meta.steam_app_id);
log::debug!("App id folder {}", app_id_folder.display());
if !app_id_folder.exists() {
std::fs::create_dir(&app_id_folder)?;
std::fs::create_dir(setting_tag_folder_by_app_id(&root, meta.steam_app_id))?;
}
symlink_locations.ron.push(app_id_folder.join(&filename_ron));
symlink_locations.json.push(app_id_folder.join(&filename_json));
// create symlinks for user id folder
let user_id_folder = setting_folder_by_user_id(&root, meta.steam_user_id);
if !user_id_folder.exists() {
std::fs::create_dir(&user_id_folder)?;
std::fs::create_dir(setting_tag_folder_by_user_id(&root, meta.steam_user_id))?;
}
symlink_locations.ron.push(user_id_folder.join(&filename_ron));
symlink_locations.json.push(user_id_folder.join(&filename_json));
// create symlinks for each tag
for tag in meta.tags.iter() {
if !str_is_alphanumeric_or_space(&tag){
continue;
}
// create symlinks for general tag folder
let tag_folder = setting_folder_by_tag(&root, tag);
if !tag_folder.exists() {
std::fs::create_dir(&tag_folder)?;
}
symlink_locations.ron.push(tag_folder.join(&filename_ron));
symlink_locations.json.push(tag_folder.join(&filename_json));
// create symlinks for app id tag folder
let app_tag_folder = setting_folder_by_app_id_tag(&root, meta.steam_app_id, tag);
if !app_tag_folder.exists() {
std::fs::create_dir(&app_tag_folder)?;
}
symlink_locations.ron.push(app_tag_folder.join(&filename_ron));
symlink_locations.json.push(app_tag_folder.join(&filename_json));
// create symlinks for user id tag folder
let user_tag_folder = setting_folder_by_user_id_tag(&root, meta.steam_user_id, tag);
if !user_tag_folder.exists() {
std::fs::create_dir(&user_tag_folder)?;
}
symlink_locations.ron.push(user_tag_folder.join(&filename_ron));
symlink_locations.json.push(user_tag_folder.join(&filename_json));
}
Ok(symlink_locations)
}
fn str_is_alphanumeric_or_space(s: &str) -> bool {
let mut result = true;
for ch in s.chars() {
result &= ch.is_ascii_alphanumeric() || ch == ' ';
}
result
}

5
backend/community_settings_srv/src/main.rs
View file

@ -2,6 +2,7 @@ mod api;
mod cli;
mod consts;
mod file_util;
mod tasks;
mod upgrade;
use actix_web::{web, App, HttpServer};
@ -39,9 +40,13 @@ async fn main() -> std::io::Result<()> {
upgrade::make_tag_subfolders(&args.folder)?;
log::info!("Resynchronizing file IDs with file name IDs");
upgrade::sync_ids(&args.folder)?;
log::info!("Rebuilding missing symlinks");
upgrade::rebuild_symlinks(&args.folder)?;
return Ok(())
}
tasks::start_tasks(args.clone());
let leaked_args: &'static cli::Cli = Box::leak::<'static>(Box::new(args));
HttpServer::new(move || {
App::new()

16
backend/community_settings_srv/src/tasks/mod.rs Normal file
View file

@ -0,0 +1,16 @@
mod task_runner;
mod symlink_cleanup;
mod user_antispam;
pub fn start_tasks(args: crate::cli::Cli) {
task_runner::TaskRunner::new(
args.clone(),
symlink_cleanup::remove_broken_symlinks,
std::time::Duration::from_secs(1 * 60 /* 1 minute */),
).run();
task_runner::TaskRunner::new(
args.clone(),
user_antispam::remove_similar_user_uploads,
std::time::Duration::from_secs(5 * 60 /* 5 minutes */),
).run();
}

39
backend/community_settings_srv/src/tasks/symlink_cleanup.rs Normal file
View file

@ -0,0 +1,39 @@
//! Realistically this shouldn't be occur unless there's corruption or a settings files is manually deleted
pub fn remove_broken_symlinks(args: &mut crate::cli::Cli) {
log::info!("Starting broken symlink removal task");
if let Err(e) = enforce_working_symlinks(&args.folder) {
log::error!("Error in broken symlink task: {}", e);
}
log::info!("Completed broken symlink removal task");
}
fn enforce_working_symlinks(root: impl AsRef<std::path::Path>) -> std::io::Result<()> {
for dir_entry in walkdir::WalkDir::new(root.as_ref().join(crate::consts::SETTING_FOLDER)) {
let dir_entry = dir_entry?;
match check_symlink(dir_entry.path()) {
Ok(true) => {},
Ok(false) => {
log::info!("Symlink {} seems broken, removing...", dir_entry.path().display());
if let Err(e) = std::fs::remove_file(dir_entry.path()) {
log::warn!("Failed to delete broken symlink {}: {}", dir_entry.path().display(), e);
}
},
Err(symlink_e) => {
log::info!("Thing {} seems broken, removing... err: {}", dir_entry.path().display(), symlink_e);
if let Err(e) = std::fs::remove_file(dir_entry.path()) {
log::warn!("Failed to delete broken thing {}: {}", dir_entry.path().display(), e);
}
}
}
}
Ok(())
}
fn check_symlink(path: impl AsRef<std::path::Path>) -> std::io::Result<bool> {
if path.as_ref().is_symlink() {
Ok(path.as_ref().read_link().is_ok_and(|link| link.exists()))
} else {
Ok(true)
}
}

47
backend/community_settings_srv/src/tasks/task_runner.rs Normal file
View file

@ -0,0 +1,47 @@
use std::thread::{spawn, JoinHandle, sleep};
use std::time::{Duration, Instant};
pub struct TaskRunner<C: Send + 'static, F: FnMut(&mut C) + Send + 'static> {
task: F,
context: C,
period: Option<Duration>,
}
impl <C: Send + 'static, F: FnMut(&mut C) + Send + 'static> TaskRunner<C, F> {
pub fn new(c: C, f: F, period: Duration) -> Self {
Self {
task: f,
context: c,
period: Some(period),
}
}
/*
pub fn new_oneshot(c: C, f: F) -> Self {
Self {
task: f,
context: c,
period: None,
}
}*/
pub fn run(mut self) -> JoinHandle<()> {
if let Some(period) = self.period {
spawn(move || {
let mut pre_task;
let mut after_task;
loop {
pre_task = Instant::now();
(self.task)(&mut self.context);
after_task = Instant::now();
sleep(period - (after_task.duration_since(pre_task)));
}
})
} else {
spawn(move || {
(self.task)(&mut self.context);
})
}
}
}

114
backend/community_settings_srv/src/tasks/user_antispam.rs Normal file
View file

@ -0,0 +1,114 @@
use crate::file_util;
pub fn remove_similar_user_uploads(args: &mut crate::cli::Cli) {
log::info!("Starting user antispam task");
if let Err(e) = enforce_user_dirs(&args.folder, args.duplicates) {
log::error!("Error in user antispam task: {}", e);
}
log::info!("Completed user antispam task");
}
fn enforce_user_dirs(root: impl AsRef<std::path::Path>, keep_duplicates: usize) -> std::io::Result<()> {
let mut to_remove: Vec<community_settings_core::v1::Metadata> = Vec::new();
let mut seen_names = std::collections::HashMap::<String, Vec<community_settings_core::v1::Metadata>>::new();
for dir_entry in root.as_ref()
.join(crate::consts::SETTING_FOLDER)
.join(crate::consts::USER_ID_FOLDER)
.read_dir()? {
match dir_entry {
Ok(dir_entry) => {
log::info!("Scanning {} for user antispam", dir_entry.path().display());
if dir_entry.file_type()?.is_dir() {
seen_names.clear();
to_remove.clear();
for user_entry in dir_entry.path().read_dir()? {
match user_entry {
Ok(user_entry) => {
let f_path = user_entry.path();
if let Some(ext) = f_path.extension() {
if ext == crate::consts::RON_EXTENSION {
let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
match ron::de::from_reader::<_, community_settings_core::v1::Metadata>(reader) {
Ok(x) => {
let sani_name = sanitise_name(&x.name);
if let Some(seen_in_ids) = seen_names.get_mut(&sani_name) {
seen_in_ids.push(x);
} else {
seen_names.insert(sani_name, vec![x]);
}
},
Err(e) => log::debug!("Error while reading {}: {}", f_path.display(), e)
}
}
}
},
Err(e) => log::warn!("Skipping file in {} for user antispam task due to error: {}", dir_entry.path().display(), e),
}
}
for seen in seen_names.values_mut() {
seen.sort_by_key(|meta| meta.get_id()); // sort lowest to highest
// keep the highest id (i.e. latest uploaded) settings
for _ in 0..keep_duplicates {
seen.pop();
}
to_remove.append(seen);
}
log::info!("Found {} spammy entries in {}", to_remove.len(), dir_entry.path().display());
// remove settings (and related symlinks) from the filesystem
for meta in to_remove.iter() {
let filename_ron = file_util::filename(meta.get_id(), crate::consts::RON_EXTENSION);
let filename_json = file_util::filename(meta.get_id(), crate::consts::JSON_EXTENSION);
// delete tag symlinks
for tag in meta.tags.iter() {
let app_id_tag_folder = file_util::setting_folder_by_app_id_tag(root.as_ref(), meta.steam_app_id, tag);
let user_id_tag_folder = file_util::setting_folder_by_user_id_tag(root.as_ref(), meta.steam_user_id, tag);
let tag_folder = file_util::setting_folder_by_tag(root.as_ref(), tag);
let paths = [
app_id_tag_folder.join(&filename_json),
app_id_tag_folder.join(&filename_ron),
user_id_tag_folder.join(&filename_json),
user_id_tag_folder.join(&filename_ron),
tag_folder.join(&filename_json),
tag_folder.join(&filename_ron),
];
for path in paths {
if path.exists() {
if let Err(e) = std::fs::remove_file(&path) {
log::warn!("Failed to delete {}: {}", path.display(), e);
}
}
}
}
// delete first-order symlinks and finally the actual files
let app_id_folder = file_util::setting_folder_by_app_id(root.as_ref(), meta.steam_app_id);
let user_id_folder = file_util::setting_folder_by_user_id(root.as_ref(), meta.steam_user_id);
let paths = [
app_id_folder.join(&filename_json),
app_id_folder.join(&filename_ron),
user_id_folder.join(&filename_json),
user_id_folder.join(&filename_ron),
file_util::setting_path_by_id(root.as_ref(), meta.get_id(), crate::consts::JSON_EXTENSION),
file_util::setting_path_by_id(root.as_ref(), meta.get_id(), crate::consts::RON_EXTENSION),
];
for path in paths {
if path.exists() {
if let Err(e) = std::fs::remove_file(&path) {
log::warn!("Failed to delete {}: {}", path.display(), e);
}
}
}
}
} else {
log::info!("Encountered non-folder path in user dir: {}", dir_entry.path().display());
}
},
Err(e) => log::warn!("Skipping file for user antispam task due to error: {}", e),
}
}
Ok(())
}
fn sanitise_name(name: &str) -> String {
name.trim().to_lowercase()
}

53
backend/community_settings_srv/src/upgrade.rs
View file

@ -26,6 +26,59 @@ pub fn build_folder_layout(root: impl AsRef<Path>) -> std::io::Result<()> {
Ok(())
}
pub fn rebuild_symlinks(root: impl AsRef<Path>) -> std::io::Result<()> {
for dir_entry in root.as_ref()
.join(SETTING_FOLDER)
.join(ID_FOLDER)
.read_dir()? {
let dir_entry = dir_entry?;
if dir_entry.file_type()?.is_file() {
let f_path = dir_entry.path();
if let Some(ext) = f_path.extension() {
if ext == RON_EXTENSION {
let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
let setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) {
Ok(x) => x,
Err(e) => {
log::debug!("Error while reading {}: {}", f_path.display(), e);
let e_msg = format!("{}", e);
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
}
};
let to_symlink = crate::file_util::symlinks(&root, &setting)?;
let path_ron_canon = f_path.canonicalize()?;
let path_json_canon = crate::file_util::setting_path_by_id(&root, setting.get_id(), JSON_EXTENSION).canonicalize()?;
for ron_link in to_symlink.ron {
if ron_link.exists() { continue; }
log::info!("Rebuilding symlink {} -> {}", ron_link.display(), path_ron_canon.display());
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
std::os::windows::fs::symlink_file(&path_ron_canon, &ron_link)?;
}
#[cfg(target_family = "unix")]
{
std::os::unix::fs::symlink(&path_ron_canon, &ron_link)?;
}
}
for json_link in to_symlink.json {
if json_link.exists() { continue; }
log::info!("Rebuilding symlink {} -> {}", json_link.display(), path_json_canon.display());
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
{
std::os::windows::fs::symlink_file(&path_json_canon, json_link)?;
}
#[cfg(target_family = "unix")]
{
std::os::unix::fs::symlink(&path_json_canon, json_link)?;
}
}
}
}
}
}
Ok(())
}
pub fn fix_symlinks(root: impl AsRef<Path>) -> std::io::Result<()> {
log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display());
for dir_entry in root.as_ref()