Add some deduplication and another symlink fixer
This commit is contained in:
parent
6c28cef9b3
commit
22e916fe12
11 changed files with 380 additions and 96 deletions
20
backend/community_settings_srv/Cargo.lock
generated
20
backend/community_settings_srv/Cargo.lock
generated
|
@ -450,6 +450,7 @@ dependencies = [
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"simplelog",
|
"simplelog",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"walkdir",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -1017,6 +1018,15 @@ version = "1.0.16"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
|
checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "same-file"
|
||||||
|
version = "1.0.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
||||||
|
dependencies = [
|
||||||
|
"winapi-util",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "scopeguard"
|
name = "scopeguard"
|
||||||
version = "1.2.0"
|
version = "1.2.0"
|
||||||
|
@ -1319,6 +1329,16 @@ version = "0.9.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
|
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "walkdir"
|
||||||
|
version = "2.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
||||||
|
dependencies = [
|
||||||
|
"same-file",
|
||||||
|
"winapi-util",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasi"
|
name = "wasi"
|
||||||
version = "0.11.0+wasi-snapshot-preview1"
|
version = "0.11.0+wasi-snapshot-preview1"
|
||||||
|
|
|
@ -16,6 +16,8 @@ tokio = { version = "1", features = ["full"] }
|
||||||
actix-web = { version = "4.4" }
|
actix-web = { version = "4.4" }
|
||||||
mime = { version = "0.3.17" }
|
mime = { version = "0.3.17" }
|
||||||
|
|
||||||
|
walkdir = "2"
|
||||||
|
|
||||||
# logging
|
# logging
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
simplelog = "0.12"
|
simplelog = "0.12"
|
||||||
|
|
|
@ -66,116 +66,32 @@ pub async fn save_setting_handler(
|
||||||
|
|
||||||
log::debug!("Saved to {}, building symlinks", path_ron.display());
|
log::debug!("Saved to {}, building symlinks", path_ron.display());
|
||||||
|
|
||||||
// create symlinks for other ways of looking up these settings files
|
let to_symlink = file_util::symlinks(&cli.folder, &parsed_data)?;
|
||||||
let filename_ron = file_util::filename(next_id, crate::consts::RON_EXTENSION);
|
|
||||||
let filename_json = file_util::filename(next_id, crate::consts::JSON_EXTENSION);
|
|
||||||
|
|
||||||
// create symlinks to app id folder
|
let path_ron_canon = path_ron.canonicalize()?;
|
||||||
let app_id_folder = file_util::setting_folder_by_app_id(&cli.folder, parsed_data.steam_app_id);
|
let path_json_canon = path_json.canonicalize()?;
|
||||||
log::debug!("App id folder {}", app_id_folder.display());
|
for ron_link in to_symlink.ron {
|
||||||
if !app_id_folder.exists() {
|
log::debug!("Symlinking {} -> {}", ron_link.display(), path_ron_canon.display());
|
||||||
std::fs::create_dir(&app_id_folder)?;
|
|
||||||
std::fs::create_dir(file_util::setting_tag_folder_by_app_id(&cli.folder, parsed_data.steam_app_id))?;
|
|
||||||
}
|
|
||||||
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
|
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
|
||||||
{
|
{
|
||||||
log::debug!("Symlinking {} -> {}", app_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
|
std::os::windows::fs::symlink_file(&path_ron_canon, &ron_link)?;
|
||||||
std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, app_id_folder.join(&filename_ron))?;
|
|
||||||
std::os::windows::fs::symlink_file(&path_json.canonicalize()?, app_id_folder.join(&filename_json))?;
|
|
||||||
}
|
}
|
||||||
#[cfg(target_family = "unix")]
|
#[cfg(target_family = "unix")]
|
||||||
{
|
{
|
||||||
log::debug!("Symlinking {} -> {}", app_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
|
std::os::unix::fs::symlink(&path_ron_canon, &ron_link)?;
|
||||||
std::os::unix::fs::symlink(&path_ron.canonicalize()?, app_id_folder.join(&filename_ron))?;
|
|
||||||
std::os::unix::fs::symlink(&path_json.canonicalize()?, app_id_folder.join(&filename_json))?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// create symlinks for user id folder
|
|
||||||
let user_id_folder = file_util::setting_folder_by_user_id(&cli.folder, parsed_data.steam_user_id);
|
|
||||||
if !user_id_folder.exists() {
|
|
||||||
std::fs::create_dir(&user_id_folder)?;
|
|
||||||
std::fs::create_dir(file_util::setting_tag_folder_by_user_id(&cli.folder, parsed_data.steam_user_id))?;
|
|
||||||
}
|
}
|
||||||
|
for json_link in to_symlink.json {
|
||||||
|
//log::debug!("Symlinking {} -> {}", json_link.display(), path_json_canon.display());
|
||||||
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
|
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
|
||||||
{
|
{
|
||||||
log::debug!("Symlinking {} -> {}", user_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
|
std::os::windows::fs::symlink_file(&path_json_canon, json_link)?;
|
||||||
std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, user_id_folder.join(&filename_ron))?;
|
|
||||||
std::os::windows::fs::symlink_file(&path_json.canonicalize()?, user_id_folder.join(&filename_json))?;
|
|
||||||
}
|
}
|
||||||
#[cfg(target_family = "unix")]
|
#[cfg(target_family = "unix")]
|
||||||
{
|
{
|
||||||
log::debug!("Symlinking {} -> {}", user_id_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
|
std::os::unix::fs::symlink(&path_json_canon, json_link)?;
|
||||||
std::os::unix::fs::symlink(&path_ron.canonicalize()?, user_id_folder.join(&filename_ron))?;
|
|
||||||
std::os::unix::fs::symlink(&path_json.canonicalize()?, user_id_folder.join(&filename_json))?;
|
|
||||||
}
|
|
||||||
|
|
||||||
// create symlinks for each tag
|
|
||||||
for tag in parsed_data.tags.iter() {
|
|
||||||
if !str_is_alphanumeric_or_space(&tag){
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// create symlinks for general tag folder
|
|
||||||
let tag_folder = file_util::setting_folder_by_tag(&cli.folder, tag);
|
|
||||||
if !tag_folder.exists() {
|
|
||||||
std::fs::create_dir(&tag_folder)?;
|
|
||||||
}
|
|
||||||
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
|
|
||||||
{
|
|
||||||
log::debug!("Symlinking {} -> {}", tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
|
|
||||||
std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, tag_folder.join(&filename_ron))?;
|
|
||||||
std::os::windows::fs::symlink_file(&path_json.canonicalize()?, tag_folder.join(&filename_json))?;
|
|
||||||
}
|
|
||||||
#[cfg(target_family = "unix")]
|
|
||||||
{
|
|
||||||
log::debug!("Symlinking {} -> {}", tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
|
|
||||||
std::os::unix::fs::symlink(&path_ron.canonicalize()?, tag_folder.join(&filename_ron))?;
|
|
||||||
std::os::unix::fs::symlink(&path_json.canonicalize()?, tag_folder.join(&filename_json))?;
|
|
||||||
}
|
|
||||||
|
|
||||||
// create symlinks for app id tag folder
|
|
||||||
let app_tag_folder = file_util::setting_folder_by_app_id_tag(&cli.folder, parsed_data.steam_app_id, tag);
|
|
||||||
if !app_tag_folder.exists() {
|
|
||||||
std::fs::create_dir(&app_tag_folder)?;
|
|
||||||
}
|
|
||||||
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
|
|
||||||
{
|
|
||||||
log::debug!("Symlinking {} -> {}", app_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
|
|
||||||
std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, app_tag_folder.join(&filename_ron))?;
|
|
||||||
std::os::windows::fs::symlink_file(&path_json.canonicalize()?, app_tag_folder.join(&filename_json))?;
|
|
||||||
}
|
|
||||||
#[cfg(target_family = "unix")]
|
|
||||||
{
|
|
||||||
log::debug!("Symlinking {} -> {}", app_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
|
|
||||||
std::os::unix::fs::symlink(&path_ron.canonicalize()?, app_tag_folder.join(&filename_ron))?;
|
|
||||||
std::os::unix::fs::symlink(&path_json.canonicalize()?, app_tag_folder.join(&filename_json))?;
|
|
||||||
}
|
|
||||||
|
|
||||||
// create symlinks for user id tag folder
|
|
||||||
let user_tag_folder = file_util::setting_folder_by_user_id_tag(&cli.folder, parsed_data.steam_user_id, tag);
|
|
||||||
if !user_tag_folder.exists() {
|
|
||||||
std::fs::create_dir(&user_tag_folder)?;
|
|
||||||
}
|
|
||||||
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
|
|
||||||
{
|
|
||||||
log::debug!("Symlinking {} -> {}", user_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
|
|
||||||
std::os::windows::fs::symlink_file(&path_ron.canonicalize()?, user_tag_folder.join(&filename_ron))?;
|
|
||||||
std::os::windows::fs::symlink_file(&path_json.canonicalize()?, user_tag_folder.join(&filename_json))?;
|
|
||||||
}
|
|
||||||
#[cfg(target_family = "unix")]
|
|
||||||
{
|
|
||||||
log::debug!("Symlinking {} -> {}", user_tag_folder.join(&filename_ron).display(), path_ron.canonicalize()?.display());
|
|
||||||
std::os::unix::fs::symlink(&path_ron.canonicalize()?, user_tag_folder.join(&filename_ron))?;
|
|
||||||
std::os::unix::fs::symlink(&path_json.canonicalize()?, user_tag_folder.join(&filename_json))?;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(actix_web::HttpResponse::NoContent())
|
Ok(actix_web::HttpResponse::NoContent())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn str_is_alphanumeric_or_space(s: &str) -> bool {
|
|
||||||
let mut result = true;
|
|
||||||
for ch in s.chars() {
|
|
||||||
result &= ch.is_ascii_alphanumeric() || ch == ' ';
|
|
||||||
}
|
|
||||||
result
|
|
||||||
}
|
|
||||||
|
|
|
@ -18,6 +18,10 @@ pub struct Cli {
|
||||||
/// Perform maintenance tasks
|
/// Perform maintenance tasks
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
pub fix: bool,
|
pub fix: bool,
|
||||||
|
|
||||||
|
/// Keep up to this many duplicate settings (0 will delete everything!!!)
|
||||||
|
#[arg(long, default_value_t = 2)]
|
||||||
|
pub duplicates: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Cli {
|
impl Cli {
|
||||||
|
|
|
@ -88,3 +88,71 @@ pub fn next_setting_id(root: impl AsRef<Path>) -> u128 {
|
||||||
*lock += 1;
|
*lock += 1;
|
||||||
*lock
|
*lock
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct ToSymlink {
|
||||||
|
pub ron: Vec<PathBuf>,
|
||||||
|
pub json: Vec<PathBuf>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn symlinks(root: impl AsRef<Path>, meta: &community_settings_core::v1::Metadata) -> std::io::Result<ToSymlink> {
|
||||||
|
let mut symlink_locations = ToSymlink { ron: Vec::new(), json: Vec::new() };
|
||||||
|
let filename_ron = filename(meta.get_id(), crate::consts::RON_EXTENSION);
|
||||||
|
let filename_json = filename(meta.get_id(), crate::consts::JSON_EXTENSION);
|
||||||
|
// build symlinks to app id folder
|
||||||
|
let app_id_folder = setting_folder_by_app_id(&root, meta.steam_app_id);
|
||||||
|
log::debug!("App id folder {}", app_id_folder.display());
|
||||||
|
if !app_id_folder.exists() {
|
||||||
|
std::fs::create_dir(&app_id_folder)?;
|
||||||
|
std::fs::create_dir(setting_tag_folder_by_app_id(&root, meta.steam_app_id))?;
|
||||||
|
}
|
||||||
|
symlink_locations.ron.push(app_id_folder.join(&filename_ron));
|
||||||
|
symlink_locations.json.push(app_id_folder.join(&filename_json));
|
||||||
|
|
||||||
|
// create symlinks for user id folder
|
||||||
|
let user_id_folder = setting_folder_by_user_id(&root, meta.steam_user_id);
|
||||||
|
if !user_id_folder.exists() {
|
||||||
|
std::fs::create_dir(&user_id_folder)?;
|
||||||
|
std::fs::create_dir(setting_tag_folder_by_user_id(&root, meta.steam_user_id))?;
|
||||||
|
}
|
||||||
|
symlink_locations.ron.push(user_id_folder.join(&filename_ron));
|
||||||
|
symlink_locations.json.push(user_id_folder.join(&filename_json));
|
||||||
|
|
||||||
|
// create symlinks for each tag
|
||||||
|
for tag in meta.tags.iter() {
|
||||||
|
if !str_is_alphanumeric_or_space(&tag){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// create symlinks for general tag folder
|
||||||
|
let tag_folder = setting_folder_by_tag(&root, tag);
|
||||||
|
if !tag_folder.exists() {
|
||||||
|
std::fs::create_dir(&tag_folder)?;
|
||||||
|
}
|
||||||
|
symlink_locations.ron.push(tag_folder.join(&filename_ron));
|
||||||
|
symlink_locations.json.push(tag_folder.join(&filename_json));
|
||||||
|
|
||||||
|
// create symlinks for app id tag folder
|
||||||
|
let app_tag_folder = setting_folder_by_app_id_tag(&root, meta.steam_app_id, tag);
|
||||||
|
if !app_tag_folder.exists() {
|
||||||
|
std::fs::create_dir(&app_tag_folder)?;
|
||||||
|
}
|
||||||
|
symlink_locations.ron.push(app_tag_folder.join(&filename_ron));
|
||||||
|
symlink_locations.json.push(app_tag_folder.join(&filename_json));
|
||||||
|
|
||||||
|
// create symlinks for user id tag folder
|
||||||
|
let user_tag_folder = setting_folder_by_user_id_tag(&root, meta.steam_user_id, tag);
|
||||||
|
if !user_tag_folder.exists() {
|
||||||
|
std::fs::create_dir(&user_tag_folder)?;
|
||||||
|
}
|
||||||
|
symlink_locations.ron.push(user_tag_folder.join(&filename_ron));
|
||||||
|
symlink_locations.json.push(user_tag_folder.join(&filename_json));
|
||||||
|
}
|
||||||
|
Ok(symlink_locations)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn str_is_alphanumeric_or_space(s: &str) -> bool {
|
||||||
|
let mut result = true;
|
||||||
|
for ch in s.chars() {
|
||||||
|
result &= ch.is_ascii_alphanumeric() || ch == ' ';
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
|
@ -2,6 +2,7 @@ mod api;
|
||||||
mod cli;
|
mod cli;
|
||||||
mod consts;
|
mod consts;
|
||||||
mod file_util;
|
mod file_util;
|
||||||
|
mod tasks;
|
||||||
mod upgrade;
|
mod upgrade;
|
||||||
|
|
||||||
use actix_web::{web, App, HttpServer};
|
use actix_web::{web, App, HttpServer};
|
||||||
|
@ -39,9 +40,13 @@ async fn main() -> std::io::Result<()> {
|
||||||
upgrade::make_tag_subfolders(&args.folder)?;
|
upgrade::make_tag_subfolders(&args.folder)?;
|
||||||
log::info!("Resynchronizing file IDs with file name IDs");
|
log::info!("Resynchronizing file IDs with file name IDs");
|
||||||
upgrade::sync_ids(&args.folder)?;
|
upgrade::sync_ids(&args.folder)?;
|
||||||
|
log::info!("Rebuilding missing symlinks");
|
||||||
|
upgrade::rebuild_symlinks(&args.folder)?;
|
||||||
return Ok(())
|
return Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
tasks::start_tasks(args.clone());
|
||||||
|
|
||||||
let leaked_args: &'static cli::Cli = Box::leak::<'static>(Box::new(args));
|
let leaked_args: &'static cli::Cli = Box::leak::<'static>(Box::new(args));
|
||||||
HttpServer::new(move || {
|
HttpServer::new(move || {
|
||||||
App::new()
|
App::new()
|
||||||
|
|
16
backend/community_settings_srv/src/tasks/mod.rs
Normal file
16
backend/community_settings_srv/src/tasks/mod.rs
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
mod task_runner;
|
||||||
|
mod symlink_cleanup;
|
||||||
|
mod user_antispam;
|
||||||
|
|
||||||
|
pub fn start_tasks(args: crate::cli::Cli) {
|
||||||
|
task_runner::TaskRunner::new(
|
||||||
|
args.clone(),
|
||||||
|
symlink_cleanup::remove_broken_symlinks,
|
||||||
|
std::time::Duration::from_secs(1 * 60 /* 1 minute */),
|
||||||
|
).run();
|
||||||
|
task_runner::TaskRunner::new(
|
||||||
|
args.clone(),
|
||||||
|
user_antispam::remove_similar_user_uploads,
|
||||||
|
std::time::Duration::from_secs(5 * 60 /* 5 minutes */),
|
||||||
|
).run();
|
||||||
|
}
|
39
backend/community_settings_srv/src/tasks/symlink_cleanup.rs
Normal file
39
backend/community_settings_srv/src/tasks/symlink_cleanup.rs
Normal file
|
@ -0,0 +1,39 @@
|
||||||
|
//! Realistically this shouldn't be occur unless there's corruption or a settings files is manually deleted
|
||||||
|
|
||||||
|
pub fn remove_broken_symlinks(args: &mut crate::cli::Cli) {
|
||||||
|
log::info!("Starting broken symlink removal task");
|
||||||
|
if let Err(e) = enforce_working_symlinks(&args.folder) {
|
||||||
|
log::error!("Error in broken symlink task: {}", e);
|
||||||
|
}
|
||||||
|
log::info!("Completed broken symlink removal task");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn enforce_working_symlinks(root: impl AsRef<std::path::Path>) -> std::io::Result<()> {
|
||||||
|
for dir_entry in walkdir::WalkDir::new(root.as_ref().join(crate::consts::SETTING_FOLDER)) {
|
||||||
|
let dir_entry = dir_entry?;
|
||||||
|
match check_symlink(dir_entry.path()) {
|
||||||
|
Ok(true) => {},
|
||||||
|
Ok(false) => {
|
||||||
|
log::info!("Symlink {} seems broken, removing...", dir_entry.path().display());
|
||||||
|
if let Err(e) = std::fs::remove_file(dir_entry.path()) {
|
||||||
|
log::warn!("Failed to delete broken symlink {}: {}", dir_entry.path().display(), e);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(symlink_e) => {
|
||||||
|
log::info!("Thing {} seems broken, removing... err: {}", dir_entry.path().display(), symlink_e);
|
||||||
|
if let Err(e) = std::fs::remove_file(dir_entry.path()) {
|
||||||
|
log::warn!("Failed to delete broken thing {}: {}", dir_entry.path().display(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_symlink(path: impl AsRef<std::path::Path>) -> std::io::Result<bool> {
|
||||||
|
if path.as_ref().is_symlink() {
|
||||||
|
Ok(path.as_ref().read_link().is_ok_and(|link| link.exists()))
|
||||||
|
} else {
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
}
|
47
backend/community_settings_srv/src/tasks/task_runner.rs
Normal file
47
backend/community_settings_srv/src/tasks/task_runner.rs
Normal file
|
@ -0,0 +1,47 @@
|
||||||
|
use std::thread::{spawn, JoinHandle, sleep};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
pub struct TaskRunner<C: Send + 'static, F: FnMut(&mut C) + Send + 'static> {
|
||||||
|
task: F,
|
||||||
|
context: C,
|
||||||
|
period: Option<Duration>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl <C: Send + 'static, F: FnMut(&mut C) + Send + 'static> TaskRunner<C, F> {
|
||||||
|
pub fn new(c: C, f: F, period: Duration) -> Self {
|
||||||
|
Self {
|
||||||
|
task: f,
|
||||||
|
context: c,
|
||||||
|
period: Some(period),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
pub fn new_oneshot(c: C, f: F) -> Self {
|
||||||
|
Self {
|
||||||
|
task: f,
|
||||||
|
context: c,
|
||||||
|
period: None,
|
||||||
|
}
|
||||||
|
}*/
|
||||||
|
|
||||||
|
pub fn run(mut self) -> JoinHandle<()> {
|
||||||
|
if let Some(period) = self.period {
|
||||||
|
spawn(move || {
|
||||||
|
let mut pre_task;
|
||||||
|
let mut after_task;
|
||||||
|
loop {
|
||||||
|
pre_task = Instant::now();
|
||||||
|
(self.task)(&mut self.context);
|
||||||
|
after_task = Instant::now();
|
||||||
|
sleep(period - (after_task.duration_since(pre_task)));
|
||||||
|
}
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
spawn(move || {
|
||||||
|
(self.task)(&mut self.context);
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
114
backend/community_settings_srv/src/tasks/user_antispam.rs
Normal file
114
backend/community_settings_srv/src/tasks/user_antispam.rs
Normal file
|
@ -0,0 +1,114 @@
|
||||||
|
use crate::file_util;
|
||||||
|
|
||||||
|
pub fn remove_similar_user_uploads(args: &mut crate::cli::Cli) {
|
||||||
|
log::info!("Starting user antispam task");
|
||||||
|
if let Err(e) = enforce_user_dirs(&args.folder, args.duplicates) {
|
||||||
|
log::error!("Error in user antispam task: {}", e);
|
||||||
|
}
|
||||||
|
log::info!("Completed user antispam task");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn enforce_user_dirs(root: impl AsRef<std::path::Path>, keep_duplicates: usize) -> std::io::Result<()> {
|
||||||
|
let mut to_remove: Vec<community_settings_core::v1::Metadata> = Vec::new();
|
||||||
|
let mut seen_names = std::collections::HashMap::<String, Vec<community_settings_core::v1::Metadata>>::new();
|
||||||
|
for dir_entry in root.as_ref()
|
||||||
|
.join(crate::consts::SETTING_FOLDER)
|
||||||
|
.join(crate::consts::USER_ID_FOLDER)
|
||||||
|
.read_dir()? {
|
||||||
|
match dir_entry {
|
||||||
|
Ok(dir_entry) => {
|
||||||
|
log::info!("Scanning {} for user antispam", dir_entry.path().display());
|
||||||
|
if dir_entry.file_type()?.is_dir() {
|
||||||
|
seen_names.clear();
|
||||||
|
to_remove.clear();
|
||||||
|
for user_entry in dir_entry.path().read_dir()? {
|
||||||
|
match user_entry {
|
||||||
|
Ok(user_entry) => {
|
||||||
|
let f_path = user_entry.path();
|
||||||
|
if let Some(ext) = f_path.extension() {
|
||||||
|
if ext == crate::consts::RON_EXTENSION {
|
||||||
|
let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
|
||||||
|
match ron::de::from_reader::<_, community_settings_core::v1::Metadata>(reader) {
|
||||||
|
Ok(x) => {
|
||||||
|
let sani_name = sanitise_name(&x.name);
|
||||||
|
if let Some(seen_in_ids) = seen_names.get_mut(&sani_name) {
|
||||||
|
seen_in_ids.push(x);
|
||||||
|
} else {
|
||||||
|
seen_names.insert(sani_name, vec![x]);
|
||||||
|
}
|
||||||
|
|
||||||
|
},
|
||||||
|
Err(e) => log::debug!("Error while reading {}: {}", f_path.display(), e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(e) => log::warn!("Skipping file in {} for user antispam task due to error: {}", dir_entry.path().display(), e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for seen in seen_names.values_mut() {
|
||||||
|
seen.sort_by_key(|meta| meta.get_id()); // sort lowest to highest
|
||||||
|
// keep the highest id (i.e. latest uploaded) settings
|
||||||
|
for _ in 0..keep_duplicates {
|
||||||
|
seen.pop();
|
||||||
|
}
|
||||||
|
to_remove.append(seen);
|
||||||
|
}
|
||||||
|
log::info!("Found {} spammy entries in {}", to_remove.len(), dir_entry.path().display());
|
||||||
|
// remove settings (and related symlinks) from the filesystem
|
||||||
|
for meta in to_remove.iter() {
|
||||||
|
let filename_ron = file_util::filename(meta.get_id(), crate::consts::RON_EXTENSION);
|
||||||
|
let filename_json = file_util::filename(meta.get_id(), crate::consts::JSON_EXTENSION);
|
||||||
|
// delete tag symlinks
|
||||||
|
for tag in meta.tags.iter() {
|
||||||
|
let app_id_tag_folder = file_util::setting_folder_by_app_id_tag(root.as_ref(), meta.steam_app_id, tag);
|
||||||
|
let user_id_tag_folder = file_util::setting_folder_by_user_id_tag(root.as_ref(), meta.steam_user_id, tag);
|
||||||
|
let tag_folder = file_util::setting_folder_by_tag(root.as_ref(), tag);
|
||||||
|
let paths = [
|
||||||
|
app_id_tag_folder.join(&filename_json),
|
||||||
|
app_id_tag_folder.join(&filename_ron),
|
||||||
|
user_id_tag_folder.join(&filename_json),
|
||||||
|
user_id_tag_folder.join(&filename_ron),
|
||||||
|
tag_folder.join(&filename_json),
|
||||||
|
tag_folder.join(&filename_ron),
|
||||||
|
];
|
||||||
|
for path in paths {
|
||||||
|
if path.exists() {
|
||||||
|
if let Err(e) = std::fs::remove_file(&path) {
|
||||||
|
log::warn!("Failed to delete {}: {}", path.display(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// delete first-order symlinks and finally the actual files
|
||||||
|
let app_id_folder = file_util::setting_folder_by_app_id(root.as_ref(), meta.steam_app_id);
|
||||||
|
let user_id_folder = file_util::setting_folder_by_user_id(root.as_ref(), meta.steam_user_id);
|
||||||
|
let paths = [
|
||||||
|
app_id_folder.join(&filename_json),
|
||||||
|
app_id_folder.join(&filename_ron),
|
||||||
|
user_id_folder.join(&filename_json),
|
||||||
|
user_id_folder.join(&filename_ron),
|
||||||
|
file_util::setting_path_by_id(root.as_ref(), meta.get_id(), crate::consts::JSON_EXTENSION),
|
||||||
|
file_util::setting_path_by_id(root.as_ref(), meta.get_id(), crate::consts::RON_EXTENSION),
|
||||||
|
];
|
||||||
|
for path in paths {
|
||||||
|
if path.exists() {
|
||||||
|
if let Err(e) = std::fs::remove_file(&path) {
|
||||||
|
log::warn!("Failed to delete {}: {}", path.display(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log::info!("Encountered non-folder path in user dir: {}", dir_entry.path().display());
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(e) => log::warn!("Skipping file for user antispam task due to error: {}", e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sanitise_name(name: &str) -> String {
|
||||||
|
name.trim().to_lowercase()
|
||||||
|
}
|
|
@ -26,6 +26,59 @@ pub fn build_folder_layout(root: impl AsRef<Path>) -> std::io::Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn rebuild_symlinks(root: impl AsRef<Path>) -> std::io::Result<()> {
|
||||||
|
for dir_entry in root.as_ref()
|
||||||
|
.join(SETTING_FOLDER)
|
||||||
|
.join(ID_FOLDER)
|
||||||
|
.read_dir()? {
|
||||||
|
let dir_entry = dir_entry?;
|
||||||
|
if dir_entry.file_type()?.is_file() {
|
||||||
|
let f_path = dir_entry.path();
|
||||||
|
if let Some(ext) = f_path.extension() {
|
||||||
|
if ext == RON_EXTENSION {
|
||||||
|
let reader = std::io::BufReader::new(std::fs::File::open(&f_path)?);
|
||||||
|
let setting: community_settings_core::v1::Metadata = match ron::de::from_reader(reader) {
|
||||||
|
Ok(x) => x,
|
||||||
|
Err(e) => {
|
||||||
|
log::debug!("Error while reading {}: {}", f_path.display(), e);
|
||||||
|
let e_msg = format!("{}", e);
|
||||||
|
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e_msg));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let to_symlink = crate::file_util::symlinks(&root, &setting)?;
|
||||||
|
let path_ron_canon = f_path.canonicalize()?;
|
||||||
|
let path_json_canon = crate::file_util::setting_path_by_id(&root, setting.get_id(), JSON_EXTENSION).canonicalize()?;
|
||||||
|
for ron_link in to_symlink.ron {
|
||||||
|
if ron_link.exists() { continue; }
|
||||||
|
log::info!("Rebuilding symlink {} -> {}", ron_link.display(), path_ron_canon.display());
|
||||||
|
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
|
||||||
|
{
|
||||||
|
std::os::windows::fs::symlink_file(&path_ron_canon, &ron_link)?;
|
||||||
|
}
|
||||||
|
#[cfg(target_family = "unix")]
|
||||||
|
{
|
||||||
|
std::os::unix::fs::symlink(&path_ron_canon, &ron_link)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for json_link in to_symlink.json {
|
||||||
|
if json_link.exists() { continue; }
|
||||||
|
log::info!("Rebuilding symlink {} -> {}", json_link.display(), path_json_canon.display());
|
||||||
|
#[cfg(target_family = "windows")] // NOTE: windows support is untested and unmaintained
|
||||||
|
{
|
||||||
|
std::os::windows::fs::symlink_file(&path_json_canon, json_link)?;
|
||||||
|
}
|
||||||
|
#[cfg(target_family = "unix")]
|
||||||
|
{
|
||||||
|
std::os::unix::fs::symlink(&path_json_canon, json_link)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn fix_symlinks(root: impl AsRef<Path>) -> std::io::Result<()> {
|
pub fn fix_symlinks(root: impl AsRef<Path>) -> std::io::Result<()> {
|
||||||
log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display());
|
log::info!("root setttings folder: {} aka {} (absolute)", root.as_ref().display(), root.as_ref().canonicalize()?.display());
|
||||||
for dir_entry in root.as_ref()
|
for dir_entry in root.as_ref()
|
||||||
|
|
Loading…
Reference in a new issue