Use find -printf instead of per-file stat fork in cleanup scripts
lidarr_cleanup.sh/sonarr_cleanup.sh/radarr_cleanup.sh each forked a separate stat call per file during classification. find already has to stat() every entry to know it's -type f, so -printf '%s %T@ %p' gets size+mtime for free during the walk itself. Measured: 5.77s for all 175,954 files in the Lidarr music root (walk + stat combined) vs 85.98s for stat alone on a 20K-file subset of the same library -- roughly 130x faster per file, and collapses two passes into one. Verified path parsing preserves spaces/parens/unicode exactly via read's trailing- field capture before switching.
This commit is contained in:
@@ -29,6 +29,9 @@
|
|||||||
# The filesystem is walked once per run, not twice — classification records which paths are
|
# The filesystem is walked once per run, not twice — classification records which paths are
|
||||||
# eligible for deletion as it goes, and the delete pass (once the size-threshold check below
|
# eligible for deletion as it goes, and the delete pass (once the size-threshold check below
|
||||||
# passes) just acts on that list instead of re-walking and re-classifying the whole tree.
|
# passes) just acts on that list instead of re-walking and re-classifying the whole tree.
|
||||||
|
# That single walk also gets size+mtime straight from find -printf instead of a separate stat
|
||||||
|
# fork per file — find already has to stat() every entry to know it's -type f, so this is
|
||||||
|
# free by comparison. Measured ~130x faster per file (0.033ms vs 4.3ms).
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# OPERATIONAL MODEL
|
# OPERATIONAL MODEL
|
||||||
@@ -452,8 +455,9 @@ NOW=$(date +%s)
|
|||||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||||
> "$TO_DELETE_FILE"
|
> "$TO_DELETE_FILE"
|
||||||
|
|
||||||
while IFS= read -r filepath; do
|
while read -r FILE_SIZE FILE_MTIME filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
|
FILE_MTIME="${FILE_MTIME%%.*}"
|
||||||
|
|
||||||
# Tracked — leave alone
|
# Tracked — leave alone
|
||||||
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
|
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
|
||||||
@@ -468,9 +472,6 @@ while IFS= read -r filepath; do
|
|||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Single stat call for both fields instead of two separate subprocess invocations.
|
|
||||||
read -r FILE_SIZE FILE_MTIME < <(stat -c '%s %Y' "$filepath" 2>/dev/null || echo "0 0")
|
|
||||||
|
|
||||||
if has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"; then
|
if has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"; then
|
||||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||||
|
|
||||||
@@ -491,7 +492,10 @@ while IFS= read -r filepath; do
|
|||||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
done < <(find "$LIDARR_MUSIC_ROOT" -type f 2>/dev/null)
|
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||||
|
# separate stat fork per file (2026-07-17) — measured ~130x faster per file (0.033ms vs
|
||||||
|
# 4.3ms), since find already has to stat() every entry anyway to know it's -type f.
|
||||||
|
done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %T@ %p\n' 2>/dev/null)
|
||||||
|
|
||||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||||
|
|||||||
@@ -27,7 +27,10 @@
|
|||||||
# individual API calls is now a jq filter over data already in hand. The filesystem is
|
# individual API calls is now a jq filter over data already in hand. The filesystem is
|
||||||
# walked once per run, not twice — classification records which paths are eligible for
|
# walked once per run, not twice — classification records which paths are eligible for
|
||||||
# deletion as it goes, and the delete pass (once the size-threshold check below passes)
|
# deletion as it goes, and the delete pass (once the size-threshold check below passes)
|
||||||
# just acts on that list instead of re-walking and re-classifying the whole tree.
|
# just acts on that list instead of re-walking and re-classifying the whole tree. That single
|
||||||
|
# walk also gets size+mtime straight from find -printf instead of a separate stat fork per
|
||||||
|
# file — find already has to stat() every entry to know it's -type f, so this is free by
|
||||||
|
# comparison. Measured ~130x faster per file (0.033ms vs 4.3ms).
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# OPERATIONAL MODEL
|
# OPERATIONAL MODEL
|
||||||
@@ -424,8 +427,9 @@ NOW=$(date +%s)
|
|||||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||||
> "$TO_DELETE_FILE"
|
> "$TO_DELETE_FILE"
|
||||||
|
|
||||||
while IFS= read -r filepath; do
|
while read -r FILE_SIZE FILE_MTIME filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
|
FILE_MTIME="${FILE_MTIME%%.*}"
|
||||||
|
|
||||||
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
|
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
|
||||||
log "TRACKED: $filepath"
|
log "TRACKED: $filepath"
|
||||||
@@ -438,9 +442,6 @@ while IFS= read -r filepath; do
|
|||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Single stat call for both fields instead of two separate subprocess invocations.
|
|
||||||
read -r FILE_SIZE FILE_MTIME < <(stat -c '%s %Y' "$filepath" 2>/dev/null || echo "0 0")
|
|
||||||
|
|
||||||
if has_extension "$filepath" "${RADARR_EXTENSIONS[@]}"; then
|
if has_extension "$filepath" "${RADARR_EXTENSIONS[@]}"; then
|
||||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||||
|
|
||||||
@@ -461,9 +462,12 @@ while IFS= read -r filepath; do
|
|||||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||||
|
# separate stat fork per file (2026-07-17) — measured ~130x faster per file (0.033ms vs
|
||||||
|
# 4.3ms), since find already has to stat() every entry anyway to know it's -type f.
|
||||||
done < <(
|
done < <(
|
||||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
[[ -d "$host_path" ]] && find "$host_path" -type f -printf '%s %T@ %p\n' 2>/dev/null
|
||||||
done | sort -u
|
done | sort -u
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -28,7 +28,10 @@
|
|||||||
# lidarr_missing_art.sh, but the data's there once one does. The filesystem is walked once
|
# lidarr_missing_art.sh, but the data's there once one does. The filesystem is walked once
|
||||||
# per run, not twice — classification records which paths are eligible for deletion as it
|
# per run, not twice — classification records which paths are eligible for deletion as it
|
||||||
# goes, and the delete pass (once the size-threshold check below passes) just acts on that
|
# goes, and the delete pass (once the size-threshold check below passes) just acts on that
|
||||||
# list instead of re-walking and re-classifying the whole tree.
|
# list instead of re-walking and re-classifying the whole tree. That single walk also gets
|
||||||
|
# size+mtime straight from find -printf instead of a separate stat fork per file — find
|
||||||
|
# already has to stat() every entry to know it's -type f, so this is free by comparison.
|
||||||
|
# Measured ~130x faster per file (0.033ms vs 4.3ms).
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# OPERATIONAL MODEL
|
# OPERATIONAL MODEL
|
||||||
@@ -435,8 +438,9 @@ NOW=$(date +%s)
|
|||||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||||
> "$TO_DELETE_FILE"
|
> "$TO_DELETE_FILE"
|
||||||
|
|
||||||
while IFS= read -r filepath; do
|
while read -r FILE_SIZE FILE_MTIME filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
|
FILE_MTIME="${FILE_MTIME%%.*}"
|
||||||
|
|
||||||
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
|
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
|
||||||
log "TRACKED: $filepath"
|
log "TRACKED: $filepath"
|
||||||
@@ -449,9 +453,6 @@ while IFS= read -r filepath; do
|
|||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Single stat call for both fields instead of two separate subprocess invocations.
|
|
||||||
read -r FILE_SIZE FILE_MTIME < <(stat -c '%s %Y' "$filepath" 2>/dev/null || echo "0 0")
|
|
||||||
|
|
||||||
if has_extension "$filepath" "${SONARR_EXTENSIONS[@]}"; then
|
if has_extension "$filepath" "${SONARR_EXTENSIONS[@]}"; then
|
||||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||||
|
|
||||||
@@ -472,9 +473,12 @@ while IFS= read -r filepath; do
|
|||||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||||
|
# separate stat fork per file (2026-07-17) — measured ~130x faster per file (0.033ms vs
|
||||||
|
# 4.3ms), since find already has to stat() every entry anyway to know it's -type f.
|
||||||
done < <(
|
done < <(
|
||||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
[[ -d "$host_path" ]] && find "$host_path" -type f -printf '%s %T@ %p\n' 2>/dev/null
|
||||||
done | sort -u
|
done | sort -u
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user