Hash only files whose size another file shares [patch]

FileDeduplicator.Test/FileScannerAndHasherTests.cs

@@ -403,4 +403,49 @@
 		mkfifo.WaitForExit();
 		Assert.AreEqual(0, mkfifo.ExitCode, $"mkfifo could not create {path}.");
 	}
+
+	/// <summary>
+	/// A file whose size no other file shares cannot have a duplicate, so it is never read.
+	/// </summary>
+	[TestMethod]
+	public void HashPossibleDuplicatesSkipsFilesWithAUniqueSize()
+	{
+		// Arrange
+		using TempTree tree = new();
+		AbsoluteFilePath[] files =
+		[
+			tree.Write("one.txt", "a"),
+			tree.Write("two.txt", "bb"),
+			tree.Write("three.txt", "ccc"),
+		];
+
+		// Act
+		Dictionary<AbsoluteFilePath, string> hashes = FileHasher.HashPossibleDuplicates(files, out IReadOnlyList<AbsoluteFilePath> uniqueSize);
+
+		// Assert
+		Assert.IsEmpty(hashes);
+		CollectionAssert.AreEquivalent(files, uniqueSize.ToArray());
+	}
+
+	/// <summary>
+	/// Files that share a size are hashed whatever their content, and empty files still group.
+	/// </summary>
+	[TestMethod]
+	public void HashPossibleDuplicatesHashesEveryFileThatSharesASize()
+	{
+		// Arrange
+		using TempTree tree = new();
+		AbsoluteFilePath sameA = tree.Write("a.txt", "ab");
+		AbsoluteFilePath sameB = tree.Write("b.txt", "cd");
+		AbsoluteFilePath emptyA = tree.Write("x/__init__.py", "");
+		AbsoluteFilePath emptyB = tree.Write("y/__init__.py", "");
+		AbsoluteFilePath alone = tree.Write("alone.txt", "unique length");
+
+		// Act
+		Dictionary<AbsoluteFilePath, string> hashes = FileHasher.HashPossibleDuplicates([sameA, sameB, emptyA, emptyB, alone], out IReadOnlyList<AbsoluteFilePath> uniqueSize);
+
+		// Assert
+		CollectionAssert.AreEquivalent(new[] { sameA, sameB, emptyA, emptyB }, hashes.Keys.ToArray());
+		CollectionAssert.AreEqual(new[] { alone }, uniqueSize.ToArray());
+	}
 }

FileDeduplicator.Test/VerbOutputTests.cs

@@ -121,6 +121,51 @@ public void ScanMarksEveryCopyKeepOrDelete()
 		}
 	}
 
+	/// <summary>
+	/// A tree where no two files are the same size has no duplicates, so nothing in it is read.
+	/// </summary>
+	[TestMethod]
+	public void ScanHashesNothingWhenEveryFileHasItsOwnSize()
+	{
+		// Arrange
+		using TempTree tree = new();
+		_ = tree.Write("a.txt", "a");
+		_ = tree.Write("b.txt", "bb");
+		_ = tree.Write("nested/c.txt", "ccc");
+
+		// Act
+		string output = ConsoleCapture.Normalize(ConsoleCapture.Run(new Scan { PathString = tree.Root.WeakString }));
+
+		// Assert
+		Assert.DoesNotContain("Hashed:", output);
+		Assert.Contains("No duplicate files found.", output);
+	}
+
+	/// <summary>
+	/// Stats still counts and sizes the files it skipped hashing because their size is unique.
+	/// </summary>
+	[TestMethod]
+	public void StatsCountsFilesWithAUniqueSizeAsUnique()
+	{
+		// Arrange
+		using TempTree tree = new();
+		_ = tree.Write("a.txt", "a");
+		_ = tree.Write("b.txt", "bb");
+		_ = tree.Write("nested/c.txt", "ccc");
+
+		// Act
+		string output = ConsoleCapture.Normalize(ConsoleCapture.Run(new Stats { PathString = tree.Root.WeakString }));
+
+		// Assert
+		Assert.DoesNotContain("Hashed:", output);
+		Assert.Contains("Total files: 3", output);
+		Assert.DoesNotContain("Unreadable files:", output);
+		Assert.Contains("Total size: 6 B", output);
+		Assert.Contains("Unique files: 3", output);
+		Assert.Contains("Duplicate files: 0", output);
+		Assert.Contains("Duplicate groups: 0", output);
+	}
+
 	/// <summary>
 	/// Stats counts the tree and breaks the duplicates down without touching anything.
 	/// </summary>
@@ -266,10 +311,10 @@ public void StatsCountsExtensionsThatDifferOnlyInCaseTogether()
 	[TestMethod]
 	public void AHashingErrorNamesTheFullPath()
 	{
-		// Arrange
+		// Arrange -- the same length as its sibling, or it would never be read
 		using TempTree tree = new();
 		_ = tree.Write("one/IMG_0001.jpg", "alpha");
-		AbsoluteFilePath locked = tree.Write("two/IMG_0001.jpg", "beta");
+		AbsoluteFilePath locked = tree.Write("two/IMG_0001.jpg", "gamma");
 		using FileStream holder = new(locked.WeakString, FileMode.Open, FileAccess.ReadWrite, FileShare.None);
 
 		// Act

FileDeduplicator/Deduplicator.cs

@@ -74,7 +74,7 @@ internal static long TotalSize(IEnumerable<AbsoluteFilePath> files) =>
 	/// <param name="file">The file to size.</param>
 	/// <param name="size">The size in bytes, when it could be read.</param>
 	/// <returns><see langword="true"/> if the file is still there and its size was read.</returns>
-	private static bool TryGetSize(AbsoluteFilePath file, out long size)
+	internal static bool TryGetSize(AbsoluteFilePath file, out long size)
 	{
 		try
 		{

FileDeduplicator/DuplicateScan.cs

@@ -0,0 +1,67 @@
+// Copyright (c) 2023-2026 ktsu-dev contributors
+
+namespace ktsu.FileDeduplicator;
+
+using System.Collections.Generic;
+
+using ktsu.Semantics.Paths;
+
+/// <summary>
+/// What every verb works from: the files under a root, the hashes of those that might have a
+/// duplicate, and the duplicate groups those hashes form.
+/// </summary>
+internal sealed class DuplicateScan
+{
+	private DuplicateScan(
+		IReadOnlyList<AbsoluteFilePath> files,
+		Dictionary<AbsoluteFilePath, string> fileHashes,
+		IReadOnlyList<AbsoluteFilePath> uniqueSize,
+		Dictionary<string, List<AbsoluteFilePath>> hashGroups)
+	{
+		Files = files;
+		FileHashes = fileHashes;
+		UniqueSize = uniqueSize;
+		HashGroups = hashGroups;
+		Duplicates = Deduplicator.FindDuplicates(hashGroups);
+	}
+
+	/// <summary>Gets every file found under the root.</summary>
+	internal IReadOnlyList<AbsoluteFilePath> Files { get; }
+
+	/// <summary>Gets the hash of every file that might have a duplicate and could be read.</summary>
+	internal Dictionary<AbsoluteFilePath, string> FileHashes { get; }
+
+	/// <summary>Gets the files left unhashed because no other file has their size.</summary>
+	internal IReadOnlyList<AbsoluteFilePath> UniqueSize { get; }
+
+	/// <summary>Gets the hashed files grouped by hash.</summary>
+	internal Dictionary<string, List<AbsoluteFilePath>> HashGroups { get; }
+
+	/// <summary>Gets the groups that still have at least two copies on disk.</summary>
+	internal IReadOnlyList<DuplicateGroup> Duplicates { get; }
+
+	/// <summary>
+	/// Discovers the files under <paramref name="root"/> and hashes those whose size another file
+	/// shares, reporting each step on the console.
+	/// </summary>
+	/// <param name="root">The directory to scan.</param>
+	/// <returns>The scan, or <see langword="null"/> when the root holds no files.</returns>
+	internal static DuplicateScan? Run(AbsoluteDirectoryPath root)
+	{
+		Console.WriteLine("Discovering files...");
+		IReadOnlyList<AbsoluteFilePath> files = FileScanner.ScanForFiles(root);
+		Console.WriteLine($"Found {files.Count} file(s).");
+		Console.WriteLine();
+
+		if (files.Count == 0)
+		{
+			return null;
+		}
+
+		Console.WriteLine("Hashing files...");
+		Dictionary<AbsoluteFilePath, string> fileHashes = FileHasher.HashPossibleDuplicates(files, out IReadOnlyList<AbsoluteFilePath> uniqueSize);
+		Console.WriteLine();
+
+		return new DuplicateScan(files, fileHashes, uniqueSize, Deduplicator.GroupByHash(fileHashes));
+	}
+}

FileDeduplicator/FileHasher.cs

@@ -5,6 +5,7 @@ namespace ktsu.FileDeduplicator;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Security.Cryptography;
 using System.Threading.Tasks;
 
@@ -14,6 +15,28 @@ internal static class FileHasher
 {
 	private static readonly Lock ConsoleLock = new();
 
+	/// <summary>
+	/// Hashes only the files that could have a duplicate, which are those whose size some other file
+	/// shares.
+	/// </summary>
+	/// <remarks>
+	/// Two files of different lengths can never match, so hashing a file whose size is unique in the
+	/// tree reads it end to end for nothing. In a media library most large files have a size of their
+	/// own, and the cold disks, USB drives and network shares this tool is pointed at make that wasted
+	/// read the bulk of a run. Zero-byte files share a size like any other, so they still group. A file
+	/// that cannot be sized is hashed anyway, so it is reported as unreadable exactly as before.
+	/// </remarks>
+	/// <param name="filePaths">The scanned files.</param>
+	/// <param name="uniqueSize">The files left unhashed because no other file has their size.</param>
+	/// <returns>The hash of every file that might have a duplicate and could be read.</returns>
+	internal static Dictionary<AbsoluteFilePath, string> HashPossibleDuplicates(IReadOnlyList<AbsoluteFilePath> filePaths, out IReadOnlyList<AbsoluteFilePath> uniqueSize)
+	{
+		IGrouping<long?, AbsoluteFilePath>[] bySize = [.. filePaths.GroupBy(file => Deduplicator.TryGetSize(file, out long size) ? size : (long?)null)];
+
+		uniqueSize = [.. bySize.Where(g => g.Key.HasValue && g.Count() == 1).SelectMany(g => g)];
+		return HashFiles([.. bySize.Where(g => !g.Key.HasValue || g.Count() > 1).SelectMany(g => g)]);
+	}
+
 	internal static Dictionary<AbsoluteFilePath, string> HashFiles(IReadOnlyList<AbsoluteFilePath> filePaths)
 	{
 		ConcurrentDictionary<AbsoluteFilePath, string> results = new();

FileDeduplicator/Verbs/Deduplicate.cs

@@ -6,8 +6,6 @@ namespace ktsu.FileDeduplicator.Verbs;
 
 using CommandLine;
 
-using ktsu.Semantics.Paths;
-
 [Verb("Deduplicate", HelpText = "Scan a directory and remove duplicate files, keeping the copy with the shortest name.")]
 internal sealed class Deduplicate : BaseVerb<Deduplicate>
 {
@@ -34,25 +32,13 @@ internal override void Run(Deduplicate options)
 		Console.WriteLine($"Deduplicating: {options.Path}");
 		Console.WriteLine();
 
-		// Step 1: Discover all files
-		Console.WriteLine("Discovering files...");
-		IReadOnlyList<AbsoluteFilePath> files = FileScanner.ScanForFiles(options.Path);
-		Console.WriteLine($"Found {files.Count} file(s).");
-		Console.WriteLine();
-
-		if (files.Count == 0)
+		DuplicateScan? scan = DuplicateScan.Run(options.Path);
+		if (scan is null)
 		{
 			return;
 		}
 
-		// Step 2: Hash all files in parallel
-		Console.WriteLine("Hashing files...");
-		Dictionary<AbsoluteFilePath, string> fileHashes = FileHasher.HashFiles(files);
-		Console.WriteLine();
-
-		// Step 3: Group by hash and find duplicates
-		Dictionary<string, List<AbsoluteFilePath>> hashGroups = Deduplicator.GroupByHash(fileHashes);
-		IReadOnlyList<DuplicateGroup> duplicates = Deduplicator.FindDuplicates(hashGroups);
+		IReadOnlyList<DuplicateGroup> duplicates = scan.Duplicates;
 
 		if (duplicates.Count == 0)
 		{

FileDeduplicator/Verbs/DryRun.cs

@@ -6,8 +6,6 @@ namespace ktsu.FileDeduplicator.Verbs;
 
 using CommandLine;
 
-using ktsu.Semantics.Paths;
-
 [Verb("DryRun", HelpText = "Scan for duplicates and show what would be deleted without actually deleting.")]
 internal sealed class DryRun : BaseVerb<DryRun>
 {
@@ -34,25 +32,13 @@ internal override void Run(DryRun options)
 		Console.WriteLine($"Dry run for: {options.Path}");
 		Console.WriteLine();
 
-		// Step 1: Discover all files
-		Console.WriteLine("Discovering files...");
-		IReadOnlyList<AbsoluteFilePath> files = FileScanner.ScanForFiles(options.Path);
-		Console.WriteLine($"Found {files.Count} file(s).");
-		Console.WriteLine();
-
-		if (files.Count == 0)
+		DuplicateScan? scan = DuplicateScan.Run(options.Path);
+		if (scan is null)
 		{
 			return;
 		}
 
-		// Step 2: Hash all files in parallel
-		Console.WriteLine("Hashing files...");
-		Dictionary<AbsoluteFilePath, string> fileHashes = FileHasher.HashFiles(files);
-		Console.WriteLine();
-
-		// Step 3: Group by hash and find duplicates
-		Dictionary<string, List<AbsoluteFilePath>> hashGroups = Deduplicator.GroupByHash(fileHashes);
-		IReadOnlyList<DuplicateGroup> duplicates = Deduplicator.FindDuplicates(hashGroups);
+		IReadOnlyList<DuplicateGroup> duplicates = scan.Duplicates;
 
 		if (duplicates.Count == 0)
 		{

FileDeduplicator/Verbs/Scan.cs

@@ -34,25 +34,13 @@ internal override void Run(Scan options)
 		Console.WriteLine($"Scanning: {options.Path}");
 		Console.WriteLine();
 
-		// Step 1: Discover all files
-		Console.WriteLine("Discovering files...");
-		IReadOnlyList<AbsoluteFilePath> files = FileScanner.ScanForFiles(options.Path);
-		Console.WriteLine($"Found {files.Count} file(s).");
-		Console.WriteLine();
-
-		if (files.Count == 0)
+		DuplicateScan? scan = DuplicateScan.Run(options.Path);
+		if (scan is null)
 		{
 			return;
 		}
 
-		// Step 2: Hash all files in parallel
-		Console.WriteLine("Hashing files...");
-		Dictionary<AbsoluteFilePath, string> fileHashes = FileHasher.HashFiles(files);
-		Console.WriteLine();
-
-		// Step 3: Group by hash and find duplicates
-		Dictionary<string, List<AbsoluteFilePath>> hashGroups = Deduplicator.GroupByHash(fileHashes);
-		IReadOnlyList<DuplicateGroup> duplicates = Deduplicator.FindDuplicates(hashGroups);
+		IReadOnlyList<DuplicateGroup> duplicates = scan.Duplicates;
 
 		if (duplicates.Count == 0)
 		{

FileDeduplicator/Verbs/Stats.cs

@@ -35,43 +35,33 @@
 		Console.WriteLine($"Analyzing: {options.Path}");
 		Console.WriteLine();
 
-		// Step 1: Discover all files
-		Console.WriteLine("Discovering files...");
-		IReadOnlyList<AbsoluteFilePath> files = FileScanner.ScanForFiles(options.Path);
-		Console.WriteLine($"Found {files.Count} file(s).");
-		Console.WriteLine();
-
-		if (files.Count == 0)
+		DuplicateScan? scan = DuplicateScan.Run(options.Path);
+		if (scan is null)
 		{
 			return;
 		}
 
-		// Step 2: Hash all files
-		Console.WriteLine("Hashing files...");
-		Dictionary<AbsoluteFilePath, string> fileHashes = FileHasher.HashFiles(files);
-		Console.WriteLine();
-
-		// Step 3: Compute statistics
-		Dictionary<string, List<AbsoluteFilePath>> hashGroups = Deduplicator.GroupByHash(fileHashes);
-		IReadOnlyList<DuplicateGroup> allGroups = Deduplicator.FindDuplicates(hashGroups);
+		// Compute statistics
+		IReadOnlyList<DuplicateGroup> allGroups = scan.Duplicates;
 
 		// Empty files group together but are never deleted (Deduplicator.IsDeletable), so counting
 		// them here reported dozens of __init__.py files as redundant copies that DryRun and
 		// Deduplicate would keep. They are reported on their own line instead.
 		DuplicateGroup[] duplicates = [.. allGroups.Where(Deduplicator.IsDeletable)];
 		int emptyCopies = allGroups.Where(g => !Deduplicator.IsDeletable(g)).Sum(g => g.Files.Count - 1);
 
-		// Every count below is taken over the files that hashed. HashFiles drops a file it cannot
+		// Every count below is taken over the files that hashed, plus those left unhashed because their
+		// size is unique and so are unique files without being read. HashFiles drops a file it cannot
 		// read, so counting against the scanned list would report each one as a duplicate of nothing,
 		// and sizing it would throw if it had vanished since the scan.
-		long totalSize = Deduplicator.TotalSize(fileHashes.Keys);
-		int uniqueFiles = hashGroups.Count;
-		int duplicateFiles = fileHashes.Count - uniqueFiles - emptyCopies;
-		int unreadableFiles = files.Count - fileHashes.Count;
+		long totalSize = Deduplicator.TotalSize(scan.FileHashes.Keys.Concat(scan.UniqueSize));
+		int uniqueFiles = scan.HashGroups.Count + scan.UniqueSize.Count;
+		int duplicateFiles = scan.FileHashes.Count - scan.HashGroups.Count - emptyCopies;
+		int unreadableFiles = scan.Files.Count - scan.FileHashes.Count - scan.UniqueSize.Count;
 
 		Console.WriteLine("=== FileDeduplicator Statistics ===");
 		Console.WriteLine();
-		Console.WriteLine($"Total files: {files.Count}");
+		Console.WriteLine($"Total files: {scan.Files.Count}");
 		if (unreadableFiles > 0)
 		{
 			Console.WriteLine($"Unreadable files: {unreadableFiles}");
@@ -127,7 +117,7 @@
 	private static Dictionary<string, int> CountRedundantCopiesByExtension(IReadOnlyList<DuplicateGroup> duplicates)
 	{
 		Dictionary<string, int> extensionCounts = [];
 		foreach (DuplicateGroup group in duplicates)
 		{
 			AbsoluteFilePath keeper = Deduplicator.SelectFileToKeep(group.Files);
 			IEnumerable<string> extensions = group.Files