From d680bd05492c1543a060eb5a288277b44e5a9870 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Mon, 27 Jul 2026 18:38:18 -0400 Subject: [PATCH] =?UTF-8?q?Gate=20arr=20orphan=20deletion=20on=20ctime=20a?= =?UTF-8?q?nd=20stop=20the=20nightly=20permissions=20pass=20from=20restamp?= =?UTF-8?q?ing=20it=20=E2=80=94=20imports=20preserve=20the=20release's=20o?= =?UTF-8?q?riginal=20mtime,=20so=20the=20age=20gate=20never=20actually=20f?= =?UTF-8?q?ired=20for=20real=20content?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Arrs_Stack/lidarr_cleanup.sh | 22 ++++++++++++++++------ Arrs_Stack/radarr_cleanup.sh | 22 ++++++++++++++++------ Arrs_Stack/sonarr_cleanup.sh | 22 ++++++++++++++++------ Media/media_shares_permissions.sh | 30 +++++++++++++++++++++++++----- Tools/bulk_permissions_repair.sh | 7 +++++++ 5 files changed, 80 insertions(+), 23 deletions(-) mode change 100755 => 100644 Arrs_Stack/lidarr_cleanup.sh mode change 100755 => 100644 Arrs_Stack/radarr_cleanup.sh mode change 100755 => 100644 Arrs_Stack/sonarr_cleanup.sh diff --git a/Arrs_Stack/lidarr_cleanup.sh b/Arrs_Stack/lidarr_cleanup.sh old mode 100755 new mode 100644 index 88d56d9..3f57007 --- a/Arrs_Stack/lidarr_cleanup.sh +++ b/Arrs_Stack/lidarr_cleanup.sh @@ -29,7 +29,7 @@ # The filesystem is walked once per run, not twice — classification records which paths are # eligible for deletion as it goes, and the delete pass (once the size-threshold check below # passes) just acts on that list instead of re-walking and re-classifying the whole tree. -# That single walk also gets size+mtime straight from find -printf instead of a separate stat +# That single walk also gets size+ctime straight from find -printf instead of a separate stat # fork per file — find already has to stat() every entry to know it's -type f, so this is # free by comparison. Measured ~130x faster per file (0.033ms vs 4.3ms). # @@ -62,6 +62,9 @@ # Files under LIDARR_ORPHAN_AGE are left alone regardless of tracked status. # Lidarr's import pipeline writes files before registering them — acting # immediately would delete files mid-import. +# Age is measured from ctime, not mtime — an import preserves the release's original +# mtime, so a file that landed today can read as years old and skip this gate. Depends +# on media_shares_permissions.sh touching only entries that are actually wrong. # # Seven-Gate Safety Model # Multiple independent sanity checks must all pass before any file is touched. @@ -455,9 +458,9 @@ NOW=$(date +%s) TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt" > "$TO_DELETE_FILE" -while read -r FILE_SIZE FILE_MTIME filepath; do +while read -r FILE_SIZE FILE_CTIME filepath; do [[ -z "$filepath" ]] && continue - FILE_MTIME="${FILE_MTIME%%.*}" + FILE_CTIME="${FILE_CTIME%%.*}" # Tracked — leave alone if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then @@ -473,7 +476,14 @@ while read -r FILE_SIZE FILE_MTIME filepath; do fi if has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"; then - FILE_AGE=$(( NOW - FILE_MTIME )) + # ctime, not mtime — an import preserves the release's original mtime, so a file + # Lidarr moved in today can read as years old and skip this gate entirely. + # Measured 2026-07-27: 400 of 400 files imported that week had mtimes over 7 + # days, one of them 9613 days. ctime is stamped when the file lands on this + # filesystem and cannot be carried in from an archive. This only holds because + # media_shares_permissions.sh applies owner/mode conditionally — a blanket + # chown/chmod restamps every inode nightly and would peg every file at age 0. + FILE_AGE=$(( NOW - FILE_CTIME )) if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_AGE_CHECK" != true ]]; then log "RECENT (skipping): $filepath" @@ -495,7 +505,7 @@ while read -r FILE_SIZE FILE_MTIME filepath; do # -printf gets size + mtime directly from find's own stat() during the walk, instead of a # separate stat fork per file (2026-07-17) — measured ~130x faster per file (0.033ms vs # 4.3ms), since find already has to stat() every entry anyway to know it's -type f. -done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %T@ %p\n' 2>/dev/null) +done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %C@ %p\n' 2>/dev/null) TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) @@ -558,4 +568,4 @@ if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then >> "$ARR_CLEANUP_STATS" 2>/dev/null || true fi -exit 0 \ No newline at end of file +exit 0 diff --git a/Arrs_Stack/radarr_cleanup.sh b/Arrs_Stack/radarr_cleanup.sh old mode 100755 new mode 100644 index 48729f3..0985d32 --- a/Arrs_Stack/radarr_cleanup.sh +++ b/Arrs_Stack/radarr_cleanup.sh @@ -35,7 +35,7 @@ # once per run, not twice — classification records which paths are eligible for deletion as # it goes, and the delete pass (once the size-threshold check below passes) just acts on that # list instead of re-walking and re-classifying the whole tree. That single walk also gets -# size+mtime straight from find -printf instead of a separate stat fork per file — find +# size+ctime straight from find -printf instead of a separate stat fork per file — find # already has to stat() every entry to know it's -type f, so this is free by comparison. # Measured ~130x faster per file (0.033ms vs 4.3ms). # @@ -72,6 +72,9 @@ # Files under RADARR_ORPHAN_AGE are left alone regardless of tracked status. # Radarr's import pipeline writes files before registering them — acting # immediately would delete files mid-import. +# Age is measured from ctime, not mtime — an import preserves the release's original +# mtime, so a file that landed today can read as years old and skip this gate. Depends +# on media_shares_permissions.sh touching only entries that are actually wrong. # # Emby Cleanup Is Part of the Job # Deleting a file without telling Emby leaves ghost entries that show as @@ -451,9 +454,9 @@ NOW=$(date +%s) TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt" > "$TO_DELETE_FILE" -while read -r FILE_SIZE FILE_MTIME filepath; do +while read -r FILE_SIZE FILE_CTIME filepath; do [[ -z "$filepath" ]] && continue - FILE_MTIME="${FILE_MTIME%%.*}" + FILE_CTIME="${FILE_CTIME%%.*}" if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then log "TRACKED: $filepath" @@ -467,7 +470,14 @@ while read -r FILE_SIZE FILE_MTIME filepath; do fi if has_extension "$filepath" "${RADARR_EXTENSIONS[@]}"; then - FILE_AGE=$(( NOW - FILE_MTIME )) + # ctime, not mtime — an import preserves the release's original mtime, so a file + # Radarr moved in today can read as years old and skip this gate entirely. + # Measured 2026-07-27: 400 of 400 files imported that week had mtimes over 7 + # days, one of them 9613 days. ctime is stamped when the file lands on this + # filesystem and cannot be carried in from an archive. This only holds because + # media_shares_permissions.sh applies owner/mode conditionally — a blanket + # chown/chmod restamps every inode nightly and would peg every file at age 0. + FILE_AGE=$(( NOW - FILE_CTIME )) if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_AGE_CHECK" != true ]]; then log "RECENT (skipping): $filepath" @@ -491,7 +501,7 @@ while read -r FILE_SIZE FILE_MTIME filepath; do # 4.3ms), since find already has to stat() every entry anyway to know it's -type f. done < <( for host_path in "${SCAN_ROOTS[@]}"; do - [[ -d "$host_path" ]] && find "$host_path" -type f -printf '%s %T@ %p\n' 2>/dev/null + [[ -d "$host_path" ]] && find "$host_path" -type f -printf '%s %C@ %p\n' 2>/dev/null done | sort -u ) @@ -558,4 +568,4 @@ if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then >> "$ARR_CLEANUP_STATS" 2>/dev/null || true fi -exit 0 \ No newline at end of file +exit 0 diff --git a/Arrs_Stack/sonarr_cleanup.sh b/Arrs_Stack/sonarr_cleanup.sh old mode 100755 new mode 100644 index b6a6b67..c678002 --- a/Arrs_Stack/sonarr_cleanup.sh +++ b/Arrs_Stack/sonarr_cleanup.sh @@ -29,7 +29,7 @@ # per run, not twice — classification records which paths are eligible for deletion as it # goes, and the delete pass (once the size-threshold check below passes) just acts on that # list instead of re-walking and re-classifying the whole tree. That single walk also gets -# size+mtime straight from find -printf instead of a separate stat fork per file — find +# size+ctime straight from find -printf instead of a separate stat fork per file — find # already has to stat() every entry to know it's -type f, so this is free by comparison. # Measured ~130x faster per file (0.033ms vs 4.3ms). # @@ -66,6 +66,9 @@ # Files under SONARR_ORPHAN_AGE are left alone regardless of tracked status. # Sonarr's import pipeline writes files before registering them — acting # immediately would delete files mid-import. +# Age is measured from ctime, not mtime — an import preserves the release's original +# mtime, so a file that landed today can read as years old and skip this gate. Depends +# on media_shares_permissions.sh touching only entries that are actually wrong. # # Emby Cleanup Is Part of the Job # Deleting a file without telling Emby leaves ghost entries that show as @@ -438,9 +441,9 @@ NOW=$(date +%s) TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt" > "$TO_DELETE_FILE" -while read -r FILE_SIZE FILE_MTIME filepath; do +while read -r FILE_SIZE FILE_CTIME filepath; do [[ -z "$filepath" ]] && continue - FILE_MTIME="${FILE_MTIME%%.*}" + FILE_CTIME="${FILE_CTIME%%.*}" if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then log "TRACKED: $filepath" @@ -454,7 +457,14 @@ while read -r FILE_SIZE FILE_MTIME filepath; do fi if has_extension "$filepath" "${SONARR_EXTENSIONS[@]}"; then - FILE_AGE=$(( NOW - FILE_MTIME )) + # ctime, not mtime — an import preserves the release's original mtime, so a file + # Sonarr moved in today can read as years old and skip this gate entirely. + # Measured 2026-07-27: 400 of 400 files imported that week had mtimes over 7 + # days, one of them 9613 days. ctime is stamped when the file lands on this + # filesystem and cannot be carried in from an archive. This only holds because + # media_shares_permissions.sh applies owner/mode conditionally — a blanket + # chown/chmod restamps every inode nightly and would peg every file at age 0. + FILE_AGE=$(( NOW - FILE_CTIME )) if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_AGE_CHECK" != true ]]; then log "RECENT (skipping): $filepath" @@ -478,7 +488,7 @@ while read -r FILE_SIZE FILE_MTIME filepath; do # 4.3ms), since find already has to stat() every entry anyway to know it's -type f. done < <( for host_path in "${SCAN_ROOTS[@]}"; do - [[ -d "$host_path" ]] && find "$host_path" -type f -printf '%s %T@ %p\n' 2>/dev/null + [[ -d "$host_path" ]] && find "$host_path" -type f -printf '%s %C@ %p\n' 2>/dev/null done | sort -u ) @@ -545,4 +555,4 @@ if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then >> "$ARR_CLEANUP_STATS" 2>/dev/null || true fi -exit 0 \ No newline at end of file +exit 0 diff --git a/Media/media_shares_permissions.sh b/Media/media_shares_permissions.sh index 4a0eca0..3e2e852 100755 --- a/Media/media_shares_permissions.sh +++ b/Media/media_shares_permissions.sh @@ -40,6 +40,9 @@ # Empty array guard — warns and exits cleanly if no shares configured # Folder existence — skips missing shares with warning, continues others # Separate passes — directories and files chmod'd separately for correctness +# Conditional passes — only entries whose owner/mode is actually wrong are touched. +# chown/chmod restamp ctime even when the value doesn't change, +# and the arr cleanups gate orphan deletion on ctime # platform_require_cmd — notify script validated before use # Silent by default — only failures produce output, success is silent # @@ -143,6 +146,10 @@ SKIPPED=() TOTAL_DIRS_FIXED=0 TOTAL_FILES_FIXED=0 +# Split for find's -user/-group predicates, which take them separately +PERMISSIONS_USER="${PERMISSIONS_OWNER%%:*}" +PERMISSIONS_GROUP="${PERMISSIONS_OWNER##*:}" + for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do SHARE_NAME=$(basename "$SHARE") @@ -158,7 +165,7 @@ for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do 2>/dev/null | wc -l) FILE_COUNT=$(find "$SHARE" -type f ! -perm "${PERMISSIONS_FILE_MODE:-664}" \ 2>/dev/null | wc -l) - OWNER_COUNT=$(find "$SHARE" ! -user nobody -o ! -group users \ + OWNER_COUNT=$(find "$SHARE" \( ! -user "$PERMISSIONS_USER" -o ! -group "$PERMISSIONS_GROUP" \) \ 2>/dev/null | wc -l) warn "DRY RUN — $SHARE_NAME: $DIR_COUNT dirs, $FILE_COUNT files, $OWNER_COUNT ownership fixes needed" continue @@ -170,21 +177,34 @@ for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do CHMOD_FILE_OK=true CHOWN_OK=true + # Every pass below is conditional — it touches only entries that are actually wrong. + # This is not just an optimisation. chown/chmod rewrite an inode's ctime even when the + # value is unchanged, so a blanket pass restamps every file in the share each night and + # erases ctime as an age signal. The arr cleanups need that signal to tell a file that + # just landed from one that has sat untracked for days — mtime can't do it, because an + # import preserves the release's original timestamp (measured 2026-07-27: 400 of 400 + # files imported that week had mtimes over 7 days old, one of them 9613 days). + # Count files with wrong ownership before fixing (diagnostic) - WRONG_OWNER=$(find "$SHARE" \( ! -user nobody -o ! -group users \) \ + WRONG_OWNER=$(find "$SHARE" \( ! -user "$PERMISSIONS_USER" -o ! -group "$PERMISSIONS_GROUP" \) \ 2>/dev/null | wc -l) # Apply ownership first — affects all files and directories - chown -R "$PERMISSIONS_OWNER" "$SHARE" 2>/dev/null || CHOWN_OK=false + if [[ "$WRONG_OWNER" -gt 0 ]]; then + find "$SHARE" \( ! -user "$PERMISSIONS_USER" -o ! -group "$PERMISSIONS_GROUP" \) \ + -exec chown "$PERMISSIONS_OWNER" {} + 2>/dev/null || CHOWN_OK=false + fi # Apply directory permissions — separate pass for correctness # Directories need execute bit — different from files - find "$SHARE" -type d -exec chmod "${PERMISSIONS_DIR_MODE:-755}" {} + \ + find "$SHARE" -type d ! -perm "${PERMISSIONS_DIR_MODE:-755}" \ + -exec chmod "${PERMISSIONS_DIR_MODE:-755}" {} + \ 2>/dev/null || CHMOD_DIR_OK=false # Apply file permissions — no execute bit on media files # Ignore "No such file" errors: race condition with volatile dirs (e.g. Emby transcodes) - _chmod_errs=$(find "$SHARE" -type f -exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + 2>&1 | \ + _chmod_errs=$(find "$SHARE" -type f ! -perm "${PERMISSIONS_FILE_MODE:-664}" \ + -exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + 2>&1 | \ grep -v "No such file or directory" | grep -c "chmod:" || true) [[ "$_chmod_errs" -gt 0 ]] && CHMOD_FILE_OK=false diff --git a/Tools/bulk_permissions_repair.sh b/Tools/bulk_permissions_repair.sh index baf1e41..b8391e9 100755 --- a/Tools/bulk_permissions_repair.sh +++ b/Tools/bulk_permissions_repair.sh @@ -15,6 +15,13 @@ # written share means a container has wrong PUID/PGID — add PUID=99 PGID=100 # to its Docker template. Common culprits: SABnzbd, qBittorrent, slskd. # +# Side effect worth knowing: this applies owner/mode unconditionally, and chown/chmod +# restamp an inode's ctime even when the value doesn't change. sonarr/radarr/lidarr_cleanup.sh +# gate orphan deletion on ctime, so running this over a whole media root resets that clock +# and pauses orphan collection there for *_ORPHAN_AGE days. That is why the nightly +# media_shares_permissions.sh applies its passes conditionally. Harmless for the targeted +# repairs this tool is meant for — worth remembering before pointing it at an entire share. +# # ============================================================================================== # DESIGN PRINCIPLES # ==============================================================================================