diff --git a/dedupe b/dedupe index 4ecafa0..9a04bad 100755 --- a/dedupe +++ b/dedupe @@ -1,16 +1,70 @@ #!/bin/sh depth="-maxdepth 1" -case "$1" in - -r|--recursive) depth="" ;; - -h|--help) - echo "Usage: $(basename "$0") [-r]" - echo " -r, --recursive search subdirectories too" - exit 0 - ;; -esac +dry_run="" -find . $depth -type f -exec md5sum {} + | sort | awk '{ - if ($1 == prev) print "rm " $2 +while [ $# -gt 0 ]; do + case "$1" in + -r|--recursive) depth="" ;; + -n|--dry-run) dry_run=1 ;; + -h|--help) + echo "Usage: $(basename "$0") [-r] [-n]" + echo " -r, --recursive search subdirectories too" + echo " -n, --dry-run show what would be removed without deleting" + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac + shift +done + +find . $depth -type f -exec md5sum {} + | sort | awk -v dry_run="${dry_run:-0}" ' +BEGIN { first = 1; total = 0 } + +$1 != prev && prev != "" { + count = 0 + for (i = 0; i < n; i++) + if (i != keep) count++ + + if (count > 0) { + if (dry_run && !first) print "" + first = 0 + for (i = 0; i < n; i++) + if (i != keep) print f[i] + } + + total += count + n = 0; keep = 0 +} + +{ + fn = substr($0, 35) + f[n++] = fn + if ((fn ~ /schmeeve-AI-/ && f[keep] !~ /schmeeve-AI-/) || (fn !~ /\/ComfyUI/ && f[keep] ~ /\/ComfyUI/)) keep = n - 1 prev = $1 -}' +} +END { + count = 0 + for (i = 0; i < n; i++) + if (i != keep) count++ + + if (count > 0) { + if (dry_run && !first) print "" + for (i = 0; i < n; i++) + if (i != keep) print f[i] + } + + total += count + + if (dry_run) + printf "Summary: %d duplicate file(s) found\n", total + else + printf "Removed %d duplicate file(s)\n", total > "/dev/stderr" +}' | if [ -n "$dry_run" ]; then + cat + else + while IFS= read -r f; do rm -f "$f" 2>/dev/null; done + fi diff --git a/organize-images b/organize-images index 2c54b81..fa4bad2 100755 --- a/organize-images +++ b/organize-images @@ -215,7 +215,7 @@ for f in "${FILES[@]}"; do if [ "$DRY_RUN" = false ]; then mkdir -p "$dest" - mv -n "$f" "$dest" + mv -f "$f" "$dest" fi case "$cat" in