From c490a6ce56a8a26cc6dc80f46cc7dc90e8d69ee7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 19:25:45 +0000 Subject: [PATCH 1/2] Hash only files whose size another file shares [patch] Two files of different lengths can never be duplicates, but every verb read and SHA-256-hashed every scanned file in full. A new FileHasher.HashPossibleDuplicates groups the scan by size first and hashes only the files whose size is shared, so a file with a unique size is never read. Scan, DryRun, Deduplicate and Stats all go through it. Empty files still group with each other, and a file that cannot be sized is still hashed so it is reported as unreadable as before. Stats counts the files skipped for a unique size as unique and includes them in the total size. Fixes ktsu-dev/FileDeduplicator#147 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FtvW1bPVVfCTF7fCWpeyRE --- .../FileScannerAndHasherTests.cs | 45 +++++++++++++++++ FileDeduplicator.Test/VerbOutputTests.cs | 49 ++++++++++++++++++- FileDeduplicator/Deduplicator.cs | 2 +- FileDeduplicator/FileHasher.cs | 23 +++++++++ FileDeduplicator/Verbs/Deduplicate.cs | 4 +- FileDeduplicator/Verbs/DryRun.cs | 4 +- FileDeduplicator/Verbs/Scan.cs | 4 +- FileDeduplicator/Verbs/Stats.cs | 15 +++--- 8 files changed, 130 insertions(+), 16 deletions(-) diff --git a/FileDeduplicator.Test/FileScannerAndHasherTests.cs b/FileDeduplicator.Test/FileScannerAndHasherTests.cs index 44c8daa..b507240 100644 --- a/FileDeduplicator.Test/FileScannerAndHasherTests.cs +++ b/FileDeduplicator.Test/FileScannerAndHasherTests.cs @@ -403,4 +403,49 @@ private static void MakeNamedPipe(string path) mkfifo.WaitForExit(); Assert.AreEqual(0, mkfifo.ExitCode, $"mkfifo could not create {path}."); } + + /// + /// A file whose size no other file shares cannot have a duplicate, so it is never read. + /// + [TestMethod] + public void HashPossibleDuplicatesSkipsFilesWithAUniqueSize() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath[] files = + [ + tree.Write("one.txt", "a"), + tree.Write("two.txt", "bb"), + tree.Write("three.txt", "ccc"), + ]; + + // Act + Dictionary hashes = FileHasher.HashPossibleDuplicates(files, out IReadOnlyList uniqueSize); + + // Assert + Assert.IsEmpty(hashes); + CollectionAssert.AreEquivalent(files, uniqueSize.ToArray()); + } + + /// + /// Files that share a size are hashed whatever their content, and empty files still group. + /// + [TestMethod] + public void HashPossibleDuplicatesHashesEveryFileThatSharesASize() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath sameA = tree.Write("a.txt", "ab"); + AbsoluteFilePath sameB = tree.Write("b.txt", "cd"); + AbsoluteFilePath emptyA = tree.Write("x/__init__.py", ""); + AbsoluteFilePath emptyB = tree.Write("y/__init__.py", ""); + AbsoluteFilePath alone = tree.Write("alone.txt", "unique length"); + + // Act + Dictionary hashes = FileHasher.HashPossibleDuplicates([sameA, sameB, emptyA, emptyB, alone], out IReadOnlyList uniqueSize); + + // Assert + CollectionAssert.AreEquivalent(new[] { sameA, sameB, emptyA, emptyB }, hashes.Keys.ToArray()); + CollectionAssert.AreEqual(new[] { alone }, uniqueSize.ToArray()); + } } diff --git a/FileDeduplicator.Test/VerbOutputTests.cs b/FileDeduplicator.Test/VerbOutputTests.cs index 4dcf70b..610eea7 100644 --- a/FileDeduplicator.Test/VerbOutputTests.cs +++ b/FileDeduplicator.Test/VerbOutputTests.cs @@ -121,6 +121,51 @@ public void ScanMarksEveryCopyKeepOrDelete() } } + /// + /// A tree where no two files are the same size has no duplicates, so nothing in it is read. + /// + [TestMethod] + public void ScanHashesNothingWhenEveryFileHasItsOwnSize() + { + // Arrange + using TempTree tree = new(); + _ = tree.Write("a.txt", "a"); + _ = tree.Write("b.txt", "bb"); + _ = tree.Write("nested/c.txt", "ccc"); + + // Act + string output = ConsoleCapture.Normalize(ConsoleCapture.Run(new Scan { PathString = tree.Root.WeakString })); + + // Assert + Assert.DoesNotContain("Hashed:", output); + Assert.Contains("No duplicate files found.", output); + } + + /// + /// Stats still counts and sizes the files it skipped hashing because their size is unique. + /// + [TestMethod] + public void StatsCountsFilesWithAUniqueSizeAsUnique() + { + // Arrange + using TempTree tree = new(); + _ = tree.Write("a.txt", "a"); + _ = tree.Write("b.txt", "bb"); + _ = tree.Write("nested/c.txt", "ccc"); + + // Act + string output = ConsoleCapture.Normalize(ConsoleCapture.Run(new Stats { PathString = tree.Root.WeakString })); + + // Assert + Assert.DoesNotContain("Hashed:", output); + Assert.Contains("Total files: 3", output); + Assert.DoesNotContain("Unreadable files:", output); + Assert.Contains("Total size: 6 B", output); + Assert.Contains("Unique files: 3", output); + Assert.Contains("Duplicate files: 0", output); + Assert.Contains("Duplicate groups: 0", output); + } + /// /// Stats counts the tree and breaks the duplicates down without touching anything. /// @@ -266,10 +311,10 @@ public void StatsCountsExtensionsThatDifferOnlyInCaseTogether() [TestMethod] public void AHashingErrorNamesTheFullPath() { - // Arrange + // Arrange -- the same length as its sibling, or it would never be read using TempTree tree = new(); _ = tree.Write("one/IMG_0001.jpg", "alpha"); - AbsoluteFilePath locked = tree.Write("two/IMG_0001.jpg", "beta"); + AbsoluteFilePath locked = tree.Write("two/IMG_0001.jpg", "gamma"); using FileStream holder = new(locked.WeakString, FileMode.Open, FileAccess.ReadWrite, FileShare.None); // Act diff --git a/FileDeduplicator/Deduplicator.cs b/FileDeduplicator/Deduplicator.cs index a00a7ea..2f63373 100644 --- a/FileDeduplicator/Deduplicator.cs +++ b/FileDeduplicator/Deduplicator.cs @@ -74,7 +74,7 @@ internal static long TotalSize(IEnumerable files) => /// The file to size. /// The size in bytes, when it could be read. /// if the file is still there and its size was read. - private static bool TryGetSize(AbsoluteFilePath file, out long size) + internal static bool TryGetSize(AbsoluteFilePath file, out long size) { try { diff --git a/FileDeduplicator/FileHasher.cs b/FileDeduplicator/FileHasher.cs index 84b5ce9..0c9d424 100644 --- a/FileDeduplicator/FileHasher.cs +++ b/FileDeduplicator/FileHasher.cs @@ -5,6 +5,7 @@ namespace ktsu.FileDeduplicator; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Security.Cryptography; using System.Threading.Tasks; @@ -14,6 +15,28 @@ internal static class FileHasher { private static readonly Lock ConsoleLock = new(); + /// + /// Hashes only the files that could have a duplicate, which are those whose size some other file + /// shares. + /// + /// + /// Two files of different lengths can never match, so hashing a file whose size is unique in the + /// tree reads it end to end for nothing. In a media library most large files have a size of their + /// own, and the cold disks, USB drives and network shares this tool is pointed at make that wasted + /// read the bulk of a run. Zero-byte files share a size like any other, so they still group. A file + /// that cannot be sized is hashed anyway, so it is reported as unreadable exactly as before. + /// + /// The scanned files. + /// The files left unhashed because no other file has their size. + /// The hash of every file that might have a duplicate and could be read. + internal static Dictionary HashPossibleDuplicates(IReadOnlyList filePaths, out IReadOnlyList uniqueSize) + { + IGrouping[] bySize = [.. filePaths.GroupBy(file => Deduplicator.TryGetSize(file, out long size) ? size : (long?)null)]; + + uniqueSize = [.. bySize.Where(g => g.Key.HasValue && g.Count() == 1).SelectMany(g => g)]; + return HashFiles([.. bySize.Where(g => !g.Key.HasValue || g.Count() > 1).SelectMany(g => g)]); + } + internal static Dictionary HashFiles(IReadOnlyList filePaths) { ConcurrentDictionary results = new(); diff --git a/FileDeduplicator/Verbs/Deduplicate.cs b/FileDeduplicator/Verbs/Deduplicate.cs index 7fbb099..8562577 100644 --- a/FileDeduplicator/Verbs/Deduplicate.cs +++ b/FileDeduplicator/Verbs/Deduplicate.cs @@ -45,9 +45,9 @@ internal override void Run(Deduplicate options) return; } - // Step 2: Hash all files in parallel + // Step 2: Hash, in parallel, every file whose size another file shares Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashFiles(files); + Dictionary fileHashes = FileHasher.HashPossibleDuplicates(files, out _); Console.WriteLine(); // Step 3: Group by hash and find duplicates diff --git a/FileDeduplicator/Verbs/DryRun.cs b/FileDeduplicator/Verbs/DryRun.cs index 6bdd9b0..cae1afb 100644 --- a/FileDeduplicator/Verbs/DryRun.cs +++ b/FileDeduplicator/Verbs/DryRun.cs @@ -45,9 +45,9 @@ internal override void Run(DryRun options) return; } - // Step 2: Hash all files in parallel + // Step 2: Hash, in parallel, every file whose size another file shares Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashFiles(files); + Dictionary fileHashes = FileHasher.HashPossibleDuplicates(files, out _); Console.WriteLine(); // Step 3: Group by hash and find duplicates diff --git a/FileDeduplicator/Verbs/Scan.cs b/FileDeduplicator/Verbs/Scan.cs index a2df2b3..5e15211 100644 --- a/FileDeduplicator/Verbs/Scan.cs +++ b/FileDeduplicator/Verbs/Scan.cs @@ -45,9 +45,9 @@ internal override void Run(Scan options) return; } - // Step 2: Hash all files in parallel + // Step 2: Hash, in parallel, every file whose size another file shares Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashFiles(files); + Dictionary fileHashes = FileHasher.HashPossibleDuplicates(files, out _); Console.WriteLine(); // Step 3: Group by hash and find duplicates diff --git a/FileDeduplicator/Verbs/Stats.cs b/FileDeduplicator/Verbs/Stats.cs index 90159bc..23c6550 100644 --- a/FileDeduplicator/Verbs/Stats.cs +++ b/FileDeduplicator/Verbs/Stats.cs @@ -46,9 +46,9 @@ internal override void Run(Stats options) return; } - // Step 2: Hash all files + // Step 2: Hash every file whose size another file shares Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashFiles(files); + Dictionary fileHashes = FileHasher.HashPossibleDuplicates(files, out IReadOnlyList uniqueSize); Console.WriteLine(); // Step 3: Compute statistics @@ -61,13 +61,14 @@ internal override void Run(Stats options) DuplicateGroup[] duplicates = [.. allGroups.Where(Deduplicator.IsDeletable)]; int emptyCopies = allGroups.Where(g => !Deduplicator.IsDeletable(g)).Sum(g => g.Files.Count - 1); - // Every count below is taken over the files that hashed. HashFiles drops a file it cannot + // Every count below is taken over the files that hashed, plus those left unhashed because their + // size is unique and so are unique files without being read. HashFiles drops a file it cannot // read, so counting against the scanned list would report each one as a duplicate of nothing, // and sizing it would throw if it had vanished since the scan. - long totalSize = Deduplicator.TotalSize(fileHashes.Keys); - int uniqueFiles = hashGroups.Count; - int duplicateFiles = fileHashes.Count - uniqueFiles - emptyCopies; - int unreadableFiles = files.Count - fileHashes.Count; + long totalSize = Deduplicator.TotalSize(fileHashes.Keys.Concat(uniqueSize)); + int uniqueFiles = hashGroups.Count + uniqueSize.Count; + int duplicateFiles = fileHashes.Count - hashGroups.Count - emptyCopies; + int unreadableFiles = files.Count - fileHashes.Count - uniqueSize.Count; Console.WriteLine("=== FileDeduplicator Statistics ==="); Console.WriteLine(); From 3f51e9007d2da620f1b89eedd5732421ce6b83f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 19:38:19 +0000 Subject: [PATCH 2/2] Run every verb's discover-and-hash steps through one DuplicateScan Scan, DryRun and Deduplicate each carried the same discover, hash and group block, and Stats a near copy of it. The size pre-filter changed a line in every copy, which SonarCloud then counted as new duplicated code. DuplicateScan.Run now does those steps once, printing the same progress lines, and all four verbs read the files, hashes, unique-size files and groups from it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FtvW1bPVVfCTF7fCWpeyRE --- FileDeduplicator/DuplicateScan.cs | 67 +++++++++++++++++++++++++++ FileDeduplicator/Verbs/Deduplicate.cs | 20 ++------ FileDeduplicator/Verbs/DryRun.cs | 20 ++------ FileDeduplicator/Verbs/Scan.cs | 18 ++----- FileDeduplicator/Verbs/Stats.cs | 29 ++++-------- 5 files changed, 85 insertions(+), 69 deletions(-) create mode 100644 FileDeduplicator/DuplicateScan.cs diff --git a/FileDeduplicator/DuplicateScan.cs b/FileDeduplicator/DuplicateScan.cs new file mode 100644 index 0000000..d855711 --- /dev/null +++ b/FileDeduplicator/DuplicateScan.cs @@ -0,0 +1,67 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.FileDeduplicator; + +using System.Collections.Generic; + +using ktsu.Semantics.Paths; + +/// +/// What every verb works from: the files under a root, the hashes of those that might have a +/// duplicate, and the duplicate groups those hashes form. +/// +internal sealed class DuplicateScan +{ + private DuplicateScan( + IReadOnlyList files, + Dictionary fileHashes, + IReadOnlyList uniqueSize, + Dictionary> hashGroups) + { + Files = files; + FileHashes = fileHashes; + UniqueSize = uniqueSize; + HashGroups = hashGroups; + Duplicates = Deduplicator.FindDuplicates(hashGroups); + } + + /// Gets every file found under the root. + internal IReadOnlyList Files { get; } + + /// Gets the hash of every file that might have a duplicate and could be read. + internal Dictionary FileHashes { get; } + + /// Gets the files left unhashed because no other file has their size. + internal IReadOnlyList UniqueSize { get; } + + /// Gets the hashed files grouped by hash. + internal Dictionary> HashGroups { get; } + + /// Gets the groups that still have at least two copies on disk. + internal IReadOnlyList Duplicates { get; } + + /// + /// Discovers the files under and hashes those whose size another file + /// shares, reporting each step on the console. + /// + /// The directory to scan. + /// The scan, or when the root holds no files. + internal static DuplicateScan? Run(AbsoluteDirectoryPath root) + { + Console.WriteLine("Discovering files..."); + IReadOnlyList files = FileScanner.ScanForFiles(root); + Console.WriteLine($"Found {files.Count} file(s)."); + Console.WriteLine(); + + if (files.Count == 0) + { + return null; + } + + Console.WriteLine("Hashing files..."); + Dictionary fileHashes = FileHasher.HashPossibleDuplicates(files, out IReadOnlyList uniqueSize); + Console.WriteLine(); + + return new DuplicateScan(files, fileHashes, uniqueSize, Deduplicator.GroupByHash(fileHashes)); + } +} diff --git a/FileDeduplicator/Verbs/Deduplicate.cs b/FileDeduplicator/Verbs/Deduplicate.cs index 8562577..2ee0f0a 100644 --- a/FileDeduplicator/Verbs/Deduplicate.cs +++ b/FileDeduplicator/Verbs/Deduplicate.cs @@ -6,8 +6,6 @@ namespace ktsu.FileDeduplicator.Verbs; using CommandLine; -using ktsu.Semantics.Paths; - [Verb("Deduplicate", HelpText = "Scan a directory and remove duplicate files, keeping the copy with the shortest name.")] internal sealed class Deduplicate : BaseVerb { @@ -34,25 +32,13 @@ internal override void Run(Deduplicate options) Console.WriteLine($"Deduplicating: {options.Path}"); Console.WriteLine(); - // Step 1: Discover all files - Console.WriteLine("Discovering files..."); - IReadOnlyList files = FileScanner.ScanForFiles(options.Path); - Console.WriteLine($"Found {files.Count} file(s)."); - Console.WriteLine(); - - if (files.Count == 0) + DuplicateScan? scan = DuplicateScan.Run(options.Path); + if (scan is null) { return; } - // Step 2: Hash, in parallel, every file whose size another file shares - Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashPossibleDuplicates(files, out _); - Console.WriteLine(); - - // Step 3: Group by hash and find duplicates - Dictionary> hashGroups = Deduplicator.GroupByHash(fileHashes); - IReadOnlyList duplicates = Deduplicator.FindDuplicates(hashGroups); + IReadOnlyList duplicates = scan.Duplicates; if (duplicates.Count == 0) { diff --git a/FileDeduplicator/Verbs/DryRun.cs b/FileDeduplicator/Verbs/DryRun.cs index cae1afb..d9f94d1 100644 --- a/FileDeduplicator/Verbs/DryRun.cs +++ b/FileDeduplicator/Verbs/DryRun.cs @@ -6,8 +6,6 @@ namespace ktsu.FileDeduplicator.Verbs; using CommandLine; -using ktsu.Semantics.Paths; - [Verb("DryRun", HelpText = "Scan for duplicates and show what would be deleted without actually deleting.")] internal sealed class DryRun : BaseVerb { @@ -34,25 +32,13 @@ internal override void Run(DryRun options) Console.WriteLine($"Dry run for: {options.Path}"); Console.WriteLine(); - // Step 1: Discover all files - Console.WriteLine("Discovering files..."); - IReadOnlyList files = FileScanner.ScanForFiles(options.Path); - Console.WriteLine($"Found {files.Count} file(s)."); - Console.WriteLine(); - - if (files.Count == 0) + DuplicateScan? scan = DuplicateScan.Run(options.Path); + if (scan is null) { return; } - // Step 2: Hash, in parallel, every file whose size another file shares - Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashPossibleDuplicates(files, out _); - Console.WriteLine(); - - // Step 3: Group by hash and find duplicates - Dictionary> hashGroups = Deduplicator.GroupByHash(fileHashes); - IReadOnlyList duplicates = Deduplicator.FindDuplicates(hashGroups); + IReadOnlyList duplicates = scan.Duplicates; if (duplicates.Count == 0) { diff --git a/FileDeduplicator/Verbs/Scan.cs b/FileDeduplicator/Verbs/Scan.cs index 5e15211..4b6fb15 100644 --- a/FileDeduplicator/Verbs/Scan.cs +++ b/FileDeduplicator/Verbs/Scan.cs @@ -34,25 +34,13 @@ internal override void Run(Scan options) Console.WriteLine($"Scanning: {options.Path}"); Console.WriteLine(); - // Step 1: Discover all files - Console.WriteLine("Discovering files..."); - IReadOnlyList files = FileScanner.ScanForFiles(options.Path); - Console.WriteLine($"Found {files.Count} file(s)."); - Console.WriteLine(); - - if (files.Count == 0) + DuplicateScan? scan = DuplicateScan.Run(options.Path); + if (scan is null) { return; } - // Step 2: Hash, in parallel, every file whose size another file shares - Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashPossibleDuplicates(files, out _); - Console.WriteLine(); - - // Step 3: Group by hash and find duplicates - Dictionary> hashGroups = Deduplicator.GroupByHash(fileHashes); - IReadOnlyList duplicates = Deduplicator.FindDuplicates(hashGroups); + IReadOnlyList duplicates = scan.Duplicates; if (duplicates.Count == 0) { diff --git a/FileDeduplicator/Verbs/Stats.cs b/FileDeduplicator/Verbs/Stats.cs index 23c6550..14d7cae 100644 --- a/FileDeduplicator/Verbs/Stats.cs +++ b/FileDeduplicator/Verbs/Stats.cs @@ -35,25 +35,14 @@ internal override void Run(Stats options) Console.WriteLine($"Analyzing: {options.Path}"); Console.WriteLine(); - // Step 1: Discover all files - Console.WriteLine("Discovering files..."); - IReadOnlyList files = FileScanner.ScanForFiles(options.Path); - Console.WriteLine($"Found {files.Count} file(s)."); - Console.WriteLine(); - - if (files.Count == 0) + DuplicateScan? scan = DuplicateScan.Run(options.Path); + if (scan is null) { return; } - // Step 2: Hash every file whose size another file shares - Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashPossibleDuplicates(files, out IReadOnlyList uniqueSize); - Console.WriteLine(); - - // Step 3: Compute statistics - Dictionary> hashGroups = Deduplicator.GroupByHash(fileHashes); - IReadOnlyList allGroups = Deduplicator.FindDuplicates(hashGroups); + // Compute statistics + IReadOnlyList allGroups = scan.Duplicates; // Empty files group together but are never deleted (Deduplicator.IsDeletable), so counting // them here reported dozens of __init__.py files as redundant copies that DryRun and @@ -65,14 +54,14 @@ internal override void Run(Stats options) // size is unique and so are unique files without being read. HashFiles drops a file it cannot // read, so counting against the scanned list would report each one as a duplicate of nothing, // and sizing it would throw if it had vanished since the scan. - long totalSize = Deduplicator.TotalSize(fileHashes.Keys.Concat(uniqueSize)); - int uniqueFiles = hashGroups.Count + uniqueSize.Count; - int duplicateFiles = fileHashes.Count - hashGroups.Count - emptyCopies; - int unreadableFiles = files.Count - fileHashes.Count - uniqueSize.Count; + long totalSize = Deduplicator.TotalSize(scan.FileHashes.Keys.Concat(scan.UniqueSize)); + int uniqueFiles = scan.HashGroups.Count + scan.UniqueSize.Count; + int duplicateFiles = scan.FileHashes.Count - scan.HashGroups.Count - emptyCopies; + int unreadableFiles = scan.Files.Count - scan.FileHashes.Count - scan.UniqueSize.Count; Console.WriteLine("=== FileDeduplicator Statistics ==="); Console.WriteLine(); - Console.WriteLine($"Total files: {files.Count}"); + Console.WriteLine($"Total files: {scan.Files.Count}"); if (unreadableFiles > 0) { Console.WriteLine($"Unreadable files: {unreadableFiles}");