diff --git a/FileDeduplicator.Test/FileScannerAndHasherTests.cs b/FileDeduplicator.Test/FileScannerAndHasherTests.cs index 44c8daa..b507240 100644 --- a/FileDeduplicator.Test/FileScannerAndHasherTests.cs +++ b/FileDeduplicator.Test/FileScannerAndHasherTests.cs @@ -403,4 +403,49 @@ private static void MakeNamedPipe(string path) mkfifo.WaitForExit(); Assert.AreEqual(0, mkfifo.ExitCode, $"mkfifo could not create {path}."); } + + /// + /// A file whose size no other file shares cannot have a duplicate, so it is never read. + /// + [TestMethod] + public void HashPossibleDuplicatesSkipsFilesWithAUniqueSize() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath[] files = + [ + tree.Write("one.txt", "a"), + tree.Write("two.txt", "bb"), + tree.Write("three.txt", "ccc"), + ]; + + // Act + Dictionary hashes = FileHasher.HashPossibleDuplicates(files, out IReadOnlyList uniqueSize); + + // Assert + Assert.IsEmpty(hashes); + CollectionAssert.AreEquivalent(files, uniqueSize.ToArray()); + } + + /// + /// Files that share a size are hashed whatever their content, and empty files still group. + /// + [TestMethod] + public void HashPossibleDuplicatesHashesEveryFileThatSharesASize() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath sameA = tree.Write("a.txt", "ab"); + AbsoluteFilePath sameB = tree.Write("b.txt", "cd"); + AbsoluteFilePath emptyA = tree.Write("x/__init__.py", ""); + AbsoluteFilePath emptyB = tree.Write("y/__init__.py", ""); + AbsoluteFilePath alone = tree.Write("alone.txt", "unique length"); + + // Act + Dictionary hashes = FileHasher.HashPossibleDuplicates([sameA, sameB, emptyA, emptyB, alone], out IReadOnlyList uniqueSize); + + // Assert + CollectionAssert.AreEquivalent(new[] { sameA, sameB, emptyA, emptyB }, hashes.Keys.ToArray()); + CollectionAssert.AreEqual(new[] { alone }, uniqueSize.ToArray()); + } } diff --git a/FileDeduplicator.Test/VerbOutputTests.cs b/FileDeduplicator.Test/VerbOutputTests.cs index 4dcf70b..610eea7 100644 --- a/FileDeduplicator.Test/VerbOutputTests.cs +++ b/FileDeduplicator.Test/VerbOutputTests.cs @@ -121,6 +121,51 @@ public void ScanMarksEveryCopyKeepOrDelete() } } + /// + /// A tree where no two files are the same size has no duplicates, so nothing in it is read. + /// + [TestMethod] + public void ScanHashesNothingWhenEveryFileHasItsOwnSize() + { + // Arrange + using TempTree tree = new(); + _ = tree.Write("a.txt", "a"); + _ = tree.Write("b.txt", "bb"); + _ = tree.Write("nested/c.txt", "ccc"); + + // Act + string output = ConsoleCapture.Normalize(ConsoleCapture.Run(new Scan { PathString = tree.Root.WeakString })); + + // Assert + Assert.DoesNotContain("Hashed:", output); + Assert.Contains("No duplicate files found.", output); + } + + /// + /// Stats still counts and sizes the files it skipped hashing because their size is unique. + /// + [TestMethod] + public void StatsCountsFilesWithAUniqueSizeAsUnique() + { + // Arrange + using TempTree tree = new(); + _ = tree.Write("a.txt", "a"); + _ = tree.Write("b.txt", "bb"); + _ = tree.Write("nested/c.txt", "ccc"); + + // Act + string output = ConsoleCapture.Normalize(ConsoleCapture.Run(new Stats { PathString = tree.Root.WeakString })); + + // Assert + Assert.DoesNotContain("Hashed:", output); + Assert.Contains("Total files: 3", output); + Assert.DoesNotContain("Unreadable files:", output); + Assert.Contains("Total size: 6 B", output); + Assert.Contains("Unique files: 3", output); + Assert.Contains("Duplicate files: 0", output); + Assert.Contains("Duplicate groups: 0", output); + } + /// /// Stats counts the tree and breaks the duplicates down without touching anything. /// @@ -266,10 +311,10 @@ public void StatsCountsExtensionsThatDifferOnlyInCaseTogether() [TestMethod] public void AHashingErrorNamesTheFullPath() { - // Arrange + // Arrange -- the same length as its sibling, or it would never be read using TempTree tree = new(); _ = tree.Write("one/IMG_0001.jpg", "alpha"); - AbsoluteFilePath locked = tree.Write("two/IMG_0001.jpg", "beta"); + AbsoluteFilePath locked = tree.Write("two/IMG_0001.jpg", "gamma"); using FileStream holder = new(locked.WeakString, FileMode.Open, FileAccess.ReadWrite, FileShare.None); // Act diff --git a/FileDeduplicator/Deduplicator.cs b/FileDeduplicator/Deduplicator.cs index a00a7ea..2f63373 100644 --- a/FileDeduplicator/Deduplicator.cs +++ b/FileDeduplicator/Deduplicator.cs @@ -74,7 +74,7 @@ internal static long TotalSize(IEnumerable files) => /// The file to size. /// The size in bytes, when it could be read. /// if the file is still there and its size was read. - private static bool TryGetSize(AbsoluteFilePath file, out long size) + internal static bool TryGetSize(AbsoluteFilePath file, out long size) { try { diff --git a/FileDeduplicator/DuplicateScan.cs b/FileDeduplicator/DuplicateScan.cs new file mode 100644 index 0000000..d855711 --- /dev/null +++ b/FileDeduplicator/DuplicateScan.cs @@ -0,0 +1,67 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.FileDeduplicator; + +using System.Collections.Generic; + +using ktsu.Semantics.Paths; + +/// +/// What every verb works from: the files under a root, the hashes of those that might have a +/// duplicate, and the duplicate groups those hashes form. +/// +internal sealed class DuplicateScan +{ + private DuplicateScan( + IReadOnlyList files, + Dictionary fileHashes, + IReadOnlyList uniqueSize, + Dictionary> hashGroups) + { + Files = files; + FileHashes = fileHashes; + UniqueSize = uniqueSize; + HashGroups = hashGroups; + Duplicates = Deduplicator.FindDuplicates(hashGroups); + } + + /// Gets every file found under the root. + internal IReadOnlyList Files { get; } + + /// Gets the hash of every file that might have a duplicate and could be read. + internal Dictionary FileHashes { get; } + + /// Gets the files left unhashed because no other file has their size. + internal IReadOnlyList UniqueSize { get; } + + /// Gets the hashed files grouped by hash. + internal Dictionary> HashGroups { get; } + + /// Gets the groups that still have at least two copies on disk. + internal IReadOnlyList Duplicates { get; } + + /// + /// Discovers the files under and hashes those whose size another file + /// shares, reporting each step on the console. + /// + /// The directory to scan. + /// The scan, or when the root holds no files. + internal static DuplicateScan? Run(AbsoluteDirectoryPath root) + { + Console.WriteLine("Discovering files..."); + IReadOnlyList files = FileScanner.ScanForFiles(root); + Console.WriteLine($"Found {files.Count} file(s)."); + Console.WriteLine(); + + if (files.Count == 0) + { + return null; + } + + Console.WriteLine("Hashing files..."); + Dictionary fileHashes = FileHasher.HashPossibleDuplicates(files, out IReadOnlyList uniqueSize); + Console.WriteLine(); + + return new DuplicateScan(files, fileHashes, uniqueSize, Deduplicator.GroupByHash(fileHashes)); + } +} diff --git a/FileDeduplicator/FileHasher.cs b/FileDeduplicator/FileHasher.cs index 84b5ce9..0c9d424 100644 --- a/FileDeduplicator/FileHasher.cs +++ b/FileDeduplicator/FileHasher.cs @@ -5,6 +5,7 @@ namespace ktsu.FileDeduplicator; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Security.Cryptography; using System.Threading.Tasks; @@ -14,6 +15,28 @@ internal static class FileHasher { private static readonly Lock ConsoleLock = new(); + /// + /// Hashes only the files that could have a duplicate, which are those whose size some other file + /// shares. + /// + /// + /// Two files of different lengths can never match, so hashing a file whose size is unique in the + /// tree reads it end to end for nothing. In a media library most large files have a size of their + /// own, and the cold disks, USB drives and network shares this tool is pointed at make that wasted + /// read the bulk of a run. Zero-byte files share a size like any other, so they still group. A file + /// that cannot be sized is hashed anyway, so it is reported as unreadable exactly as before. + /// + /// The scanned files. + /// The files left unhashed because no other file has their size. + /// The hash of every file that might have a duplicate and could be read. + internal static Dictionary HashPossibleDuplicates(IReadOnlyList filePaths, out IReadOnlyList uniqueSize) + { + IGrouping[] bySize = [.. filePaths.GroupBy(file => Deduplicator.TryGetSize(file, out long size) ? size : (long?)null)]; + + uniqueSize = [.. bySize.Where(g => g.Key.HasValue && g.Count() == 1).SelectMany(g => g)]; + return HashFiles([.. bySize.Where(g => !g.Key.HasValue || g.Count() > 1).SelectMany(g => g)]); + } + internal static Dictionary HashFiles(IReadOnlyList filePaths) { ConcurrentDictionary results = new(); diff --git a/FileDeduplicator/Verbs/Deduplicate.cs b/FileDeduplicator/Verbs/Deduplicate.cs index 7fbb099..2ee0f0a 100644 --- a/FileDeduplicator/Verbs/Deduplicate.cs +++ b/FileDeduplicator/Verbs/Deduplicate.cs @@ -6,8 +6,6 @@ namespace ktsu.FileDeduplicator.Verbs; using CommandLine; -using ktsu.Semantics.Paths; - [Verb("Deduplicate", HelpText = "Scan a directory and remove duplicate files, keeping the copy with the shortest name.")] internal sealed class Deduplicate : BaseVerb { @@ -34,25 +32,13 @@ internal override void Run(Deduplicate options) Console.WriteLine($"Deduplicating: {options.Path}"); Console.WriteLine(); - // Step 1: Discover all files - Console.WriteLine("Discovering files..."); - IReadOnlyList files = FileScanner.ScanForFiles(options.Path); - Console.WriteLine($"Found {files.Count} file(s)."); - Console.WriteLine(); - - if (files.Count == 0) + DuplicateScan? scan = DuplicateScan.Run(options.Path); + if (scan is null) { return; } - // Step 2: Hash all files in parallel - Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashFiles(files); - Console.WriteLine(); - - // Step 3: Group by hash and find duplicates - Dictionary> hashGroups = Deduplicator.GroupByHash(fileHashes); - IReadOnlyList duplicates = Deduplicator.FindDuplicates(hashGroups); + IReadOnlyList duplicates = scan.Duplicates; if (duplicates.Count == 0) { diff --git a/FileDeduplicator/Verbs/DryRun.cs b/FileDeduplicator/Verbs/DryRun.cs index 6bdd9b0..d9f94d1 100644 --- a/FileDeduplicator/Verbs/DryRun.cs +++ b/FileDeduplicator/Verbs/DryRun.cs @@ -6,8 +6,6 @@ namespace ktsu.FileDeduplicator.Verbs; using CommandLine; -using ktsu.Semantics.Paths; - [Verb("DryRun", HelpText = "Scan for duplicates and show what would be deleted without actually deleting.")] internal sealed class DryRun : BaseVerb { @@ -34,25 +32,13 @@ internal override void Run(DryRun options) Console.WriteLine($"Dry run for: {options.Path}"); Console.WriteLine(); - // Step 1: Discover all files - Console.WriteLine("Discovering files..."); - IReadOnlyList files = FileScanner.ScanForFiles(options.Path); - Console.WriteLine($"Found {files.Count} file(s)."); - Console.WriteLine(); - - if (files.Count == 0) + DuplicateScan? scan = DuplicateScan.Run(options.Path); + if (scan is null) { return; } - // Step 2: Hash all files in parallel - Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashFiles(files); - Console.WriteLine(); - - // Step 3: Group by hash and find duplicates - Dictionary> hashGroups = Deduplicator.GroupByHash(fileHashes); - IReadOnlyList duplicates = Deduplicator.FindDuplicates(hashGroups); + IReadOnlyList duplicates = scan.Duplicates; if (duplicates.Count == 0) { diff --git a/FileDeduplicator/Verbs/Scan.cs b/FileDeduplicator/Verbs/Scan.cs index a2df2b3..4b6fb15 100644 --- a/FileDeduplicator/Verbs/Scan.cs +++ b/FileDeduplicator/Verbs/Scan.cs @@ -34,25 +34,13 @@ internal override void Run(Scan options) Console.WriteLine($"Scanning: {options.Path}"); Console.WriteLine(); - // Step 1: Discover all files - Console.WriteLine("Discovering files..."); - IReadOnlyList files = FileScanner.ScanForFiles(options.Path); - Console.WriteLine($"Found {files.Count} file(s)."); - Console.WriteLine(); - - if (files.Count == 0) + DuplicateScan? scan = DuplicateScan.Run(options.Path); + if (scan is null) { return; } - // Step 2: Hash all files in parallel - Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashFiles(files); - Console.WriteLine(); - - // Step 3: Group by hash and find duplicates - Dictionary> hashGroups = Deduplicator.GroupByHash(fileHashes); - IReadOnlyList duplicates = Deduplicator.FindDuplicates(hashGroups); + IReadOnlyList duplicates = scan.Duplicates; if (duplicates.Count == 0) { diff --git a/FileDeduplicator/Verbs/Stats.cs b/FileDeduplicator/Verbs/Stats.cs index 90159bc..14d7cae 100644 --- a/FileDeduplicator/Verbs/Stats.cs +++ b/FileDeduplicator/Verbs/Stats.cs @@ -35,25 +35,14 @@ internal override void Run(Stats options) Console.WriteLine($"Analyzing: {options.Path}"); Console.WriteLine(); - // Step 1: Discover all files - Console.WriteLine("Discovering files..."); - IReadOnlyList files = FileScanner.ScanForFiles(options.Path); - Console.WriteLine($"Found {files.Count} file(s)."); - Console.WriteLine(); - - if (files.Count == 0) + DuplicateScan? scan = DuplicateScan.Run(options.Path); + if (scan is null) { return; } - // Step 2: Hash all files - Console.WriteLine("Hashing files..."); - Dictionary fileHashes = FileHasher.HashFiles(files); - Console.WriteLine(); - - // Step 3: Compute statistics - Dictionary> hashGroups = Deduplicator.GroupByHash(fileHashes); - IReadOnlyList allGroups = Deduplicator.FindDuplicates(hashGroups); + // Compute statistics + IReadOnlyList allGroups = scan.Duplicates; // Empty files group together but are never deleted (Deduplicator.IsDeletable), so counting // them here reported dozens of __init__.py files as redundant copies that DryRun and @@ -61,17 +50,18 @@ internal override void Run(Stats options) DuplicateGroup[] duplicates = [.. allGroups.Where(Deduplicator.IsDeletable)]; int emptyCopies = allGroups.Where(g => !Deduplicator.IsDeletable(g)).Sum(g => g.Files.Count - 1); - // Every count below is taken over the files that hashed. HashFiles drops a file it cannot + // Every count below is taken over the files that hashed, plus those left unhashed because their + // size is unique and so are unique files without being read. HashFiles drops a file it cannot // read, so counting against the scanned list would report each one as a duplicate of nothing, // and sizing it would throw if it had vanished since the scan. - long totalSize = Deduplicator.TotalSize(fileHashes.Keys); - int uniqueFiles = hashGroups.Count; - int duplicateFiles = fileHashes.Count - uniqueFiles - emptyCopies; - int unreadableFiles = files.Count - fileHashes.Count; + long totalSize = Deduplicator.TotalSize(scan.FileHashes.Keys.Concat(scan.UniqueSize)); + int uniqueFiles = scan.HashGroups.Count + scan.UniqueSize.Count; + int duplicateFiles = scan.FileHashes.Count - scan.HashGroups.Count - emptyCopies; + int unreadableFiles = scan.Files.Count - scan.FileHashes.Count - scan.UniqueSize.Count; Console.WriteLine("=== FileDeduplicator Statistics ==="); Console.WriteLine(); - Console.WriteLine($"Total files: {files.Count}"); + Console.WriteLine($"Total files: {scan.Files.Count}"); if (unreadableFiles > 0) { Console.WriteLine($"Unreadable files: {unreadableFiles}");