diff --git a/FileDeduplicator.Test/SameFileTests.cs b/FileDeduplicator.Test/SameFileTests.cs new file mode 100644 index 0000000..3d4553b --- /dev/null +++ b/FileDeduplicator.Test/SameFileTests.cs @@ -0,0 +1,224 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.FileDeduplicator.Test; + +using System.Diagnostics; + +using ktsu.FileDeduplicator.Verbs; +using ktsu.Semantics.Paths; +using ktsu.Semantics.Strings; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Tests that one file reached at two paths -- a hardlink or a bind mount -- is never treated as a +/// duplicate of itself, and above all is never deleted as one. +/// +/// +/// Through a bind mount, the "other copy" is the keeper's own directory entry, so deleting it +/// deletes the only copy of the data, and the keeper re-hash cannot catch it. Through a hardlink, +/// deleting it removes a name from a snapshot while reclaiming nothing. +/// +[TestClass] +[DoNotParallelize] +public sealed class SameFileTests +{ + /// + /// Two hardlinks to one file share an identity, and two files with the same content do not. + /// + [TestMethod] + public void HardlinksShareAnIdentityAndCopiesDoNot() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath original = tree.Write("original.txt", "content"); + AbsoluteFilePath copy = tree.Write("copy.txt", "content"); + AbsoluteFilePath link = MakeHardLink(tree, original, "link.txt"); + + // Act + Assert.IsTrue(FileIdentity.TryGet(original.WeakString, out FileIdentity originalIdentity)); + Assert.IsTrue(FileIdentity.TryGet(copy.WeakString, out FileIdentity copyIdentity)); + Assert.IsTrue(FileIdentity.TryGet(link.WeakString, out FileIdentity linkIdentity)); + + // Assert + Assert.AreEqual(originalIdentity, linkIdentity); + Assert.AreNotEqual(originalIdentity, copyIdentity); + } + + /// + /// The index read on Unix is the inode number ls -i reports, which pins the offset the + /// identity is read from in the runtime's file status structure. + /// + [TestMethod] + [OSCondition(OperatingSystems.Linux | OperatingSystems.OSX)] + public void TheUnixIndexIsTheInodeNumber() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath file = tree.Write("file.txt", "content"); + + // Act + Assert.IsTrue(FileIdentity.TryGet(file.WeakString, out FileIdentity identity)); + string listing = RunTool("ls", "-i", file.WeakString); + + // Assert + Assert.AreEqual(ulong.Parse(listing.Trim().Split(' ')[0], System.Globalization.CultureInfo.InvariantCulture), identity.Index); + } + + /// + /// A missing file has no identity, rather than throwing. + /// + [TestMethod] + public void AMissingFileHasNoIdentity() + { + using TempTree tree = new(); + + Assert.IsFalse(FileIdentity.TryGet(Path.Combine(tree.Root.WeakString, "absent.txt"), out _)); + } + + /// + /// A file and its hardlink are one file, so they do not form a duplicate group. + /// + [TestMethod] + public void AHardlinkIsNotADuplicateOfItsOwnFile() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath original = tree.Write("original.txt", "content"); + AbsoluteFilePath link = MakeHardLink(tree, original, "nested/link.txt"); + + // Act + IReadOnlyList duplicates = Deduplicator.FindDuplicates( + Deduplicator.GroupByHash(FileHasher.HashFiles([original, link]))); + + // Assert + Assert.IsEmpty(duplicates); + } + + /// + /// A hardlink in a group with a genuine copy is collapsed into its file, so the group holds the + /// file once and the genuine copy once. + /// + [TestMethod] + public void AHardlinkIsCollapsedIntoItsFileAlongsideAGenuineCopy() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath original = tree.Write("a.txt", "content"); + AbsoluteFilePath link = MakeHardLink(tree, original, "linked.txt"); + AbsoluteFilePath copy = tree.Write("copy.txt", "content"); + + // Act + IReadOnlyList duplicates = Deduplicator.FindDuplicates( + Deduplicator.GroupByHash(FileHasher.HashFiles([original, link, copy]))); + + // Assert + Assert.HasCount(1, duplicates); + CollectionAssert.AreEquivalent(new[] { original, copy }, duplicates[0].Files); + } + + /// + /// The deletion loop refuses a path that leads to the keeper's own file, whatever grouping it is + /// handed, and reports it instead. + /// + [TestMethod] + public void DeletionSkipsAPathThatLeadsToTheKeeper() + { + // Arrange -- a group built by hand, bypassing the collapse in FindDuplicates + using TempTree tree = new(); + AbsoluteFilePath keeper = tree.Write("a.txt", "content"); + AbsoluteFilePath link = MakeHardLink(tree, keeper, "longer-name.txt"); + string hash = FileHasher.HashFiles([keeper])[keeper]; + DuplicateGroup group = new(hash, [keeper, link], "content".Length); + + // Act + DeduplicationResult result = Deduplicator.DeleteDuplicates([group]); + + // Assert + Assert.AreEqual(0, result.DeletedCount); + Assert.AreEqual(0, result.BytesReclaimed); + Assert.IsTrue(TempTree.Exists(keeper)); + Assert.IsTrue(TempTree.Exists(link)); + Assert.HasCount(1, result.SkippedFiles); + Assert.AreEqual(link, result.SkippedFiles[0].Path); + Assert.Contains("same file as the copy being kept", result.SkippedFiles[0].Reason); + } + + /// + /// The issue's reproduction: one directory visible at two paths through a bind mount. The run + /// ends with the file still present, nothing deleted and nothing reported as reclaimed. + /// + [TestMethod] + public void DeduplicateLeavesAFileReachedThroughABindMountAlone() + { + // Arrange + using TempTree tree = new(); + AbsoluteFilePath important = tree.Write("real/important.txt", "only copy of important data"); + string view = Path.Combine(tree.Root.WeakString, "view"); + _ = Directory.CreateDirectory(view); + BindMount(Path.Combine(tree.Root.WeakString, "real"), view); + + try + { + // Act + string output = ConsoleCapture.Normalize(ConsoleCapture.Run(new Deduplicate { PathString = tree.Root.WeakString }, "y")); + string dryRun = ConsoleCapture.Normalize(ConsoleCapture.Run(new DryRun { PathString = tree.Root.WeakString })); + + // Assert + Assert.IsTrue(TempTree.Exists(important), $"The only copy was deleted. Output was:\n{output}"); + Assert.AreEqual("only copy of important data", File.ReadAllText(important.WeakString)); + Assert.DoesNotContain("Deleted:", output); + Assert.Contains("No duplicate files found.", output); + Assert.Contains("No duplicate files found.", dryRun); + } + finally + { + _ = RunTool("umount", view); + } + } + + private static AbsoluteFilePath MakeHardLink(TempTree tree, AbsoluteFilePath target, string relativePath) + { + string link = Path.Combine(tree.Root.WeakString, relativePath.Replace('/', Path.DirectorySeparatorChar)); + _ = Directory.CreateDirectory(Path.GetDirectoryName(link)!); + + _ = OperatingSystem.IsWindows() + ? RunTool("cmd", "/c", "mklink", "/H", link, target.WeakString) + : RunTool("ln", target.WeakString, link); + + Assert.IsTrue(File.Exists(link), $"Could not create a hardlink at {link}."); + return link.As(); + } + + private static void BindMount(string source, string target) + { + if (!OperatingSystem.IsLinux()) + { + Assert.Inconclusive("Bind mounts are a Linux feature."); + } + + using Process mount = Process.Start(new ProcessStartInfo("mount", ["--bind", source, target]) + { + RedirectStandardError = true, + RedirectStandardOutput = true, + })!; + mount.WaitForExit(); + + if (mount.ExitCode != 0) + { + Assert.Inconclusive($"Could not bind-mount {source} (mount needs root): {mount.StandardError.ReadToEnd()}"); + } + } + + private static string RunTool(string fileName, params string[] arguments) + { + using Process process = Process.Start(new ProcessStartInfo(fileName, arguments) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + })!; + string output = process.StandardOutput.ReadToEnd(); + process.WaitForExit(); + return output; + } +} diff --git a/FileDeduplicator/Deduplicator.cs b/FileDeduplicator/Deduplicator.cs index a00a7ea..8c3a141 100644 --- a/FileDeduplicator/Deduplicator.cs +++ b/FileDeduplicator/Deduplicator.cs @@ -35,6 +35,11 @@ internal static Dictionary> GroupByHash(Dictionar /// The hash pass can take minutes on a large tree, and the directories this is pointed at are /// often live, so a file that hashed may be gone by now. A vanished copy is dropped rather than /// allowed to abort the run, and a group it leaves with a single copy is no longer a group. + /// + /// Paths that lead to one file (see ) are collapsed to the one + /// would prefer. They are one copy, not several, so a file the + /// scan reached through a bind mount or a hardlink is not a duplicate of itself. + /// /// /// The files that hashed, grouped by hash. /// Every group that still has at least two copies. @@ -46,7 +51,7 @@ internal static IReadOnlyList FindDuplicates(Dictionary (File: file, Size: TryGetSize(file, out long size) ? size : (long?)null)) .Where(copy => copy.Size.HasValue) .Select(copy => (copy.File, copy.Size!.Value))]; @@ -60,6 +65,24 @@ internal static IReadOnlyList FindDuplicates(Dictionary + /// Keeps one path per file, preferring the path would keep. + /// + /// The paths in one hash group. + /// The paths, with every further path to an already-listed file left out. + /// + /// A path whose identity cannot be read is kept: it is either gone, which the sizing that follows + /// drops, or it cannot be inspected, and the deletion guard in + /// still stands between it and the keeper. + /// + private static IEnumerable CollapseSameFile(IEnumerable paths) + { + HashSet seen = []; + + return OrderByPreference(paths) + .Where(path => !FileIdentity.TryGet(path.WeakString, out FileIdentity identity) || seen.Add(identity)); + } + /// /// Sums the sizes of the given files, counting nothing for one that is no longer there. /// @@ -95,7 +118,10 @@ private static bool TryGetSize(AbsoluteFilePath file, out long size) } internal static AbsoluteFilePath SelectFileToKeep(List duplicates) => - duplicates.OrderBy(f => f.FileName.WeakString.Length).ThenBy(f => f.WeakString, StringComparer.Ordinal).First(); + OrderByPreference(duplicates).First(); + + private static IOrderedEnumerable OrderByPreference(IEnumerable files) => + files.OrderBy(f => f.FileName.WeakString.Length).ThenBy(f => f.WeakString, StringComparer.Ordinal); /// /// Reports whether deduplication may delete copies from a group. @@ -130,6 +156,8 @@ internal static DeduplicationResult DeleteDuplicates(IReadOnlyList f != keeper)) { // Re-hash immediately before deleting: a file that changed since the scan is no @@ -141,29 +169,20 @@ internal static DeduplicationResult DeleteDuplicates(IReadOnlyList + /// Deletes one confirmed duplicate, recording a failure instead of letting it end the run. + /// + /// The copy to delete. + /// The list to record a failure on. + /// The size of the deleted copy, when it was deleted. + /// if the copy was deleted. + private static bool TryDelete(AbsoluteFilePath file, List errors, out long fileSize) + { + try + { + fileSize = new FileInfo(file.WeakString).Length; + File.Delete(file.WeakString); + Console.WriteLine($" Deleted: {file}"); + return true; + } + catch (IOException ex) + { + RecordDeleteError(file, ex, errors); + } + // A copy the process is not allowed to remove -- read-only on Windows, or in a + // write-protected directory on Unix -- must cost that one file, not the rest of the + // run. Letting this escape would abandon every group after it, with no summary and + // no report of what was already deleted. + catch (UnauthorizedAccessException ex) + { + RecordDeleteError(file, ex, errors); + } + + fileSize = 0; + return false; + } + + private static void RecordDeleteError(AbsoluteFilePath file, Exception ex, List errors) + { + string error = $" Error deleting {file}: {ex.Message}"; + errors.Add(error); + Console.WriteLine(error); + } + + /// + /// Reports whether a path leads to the file with the given identity. + /// + /// The path to check. + /// The identity to compare against. + /// only when the path's identity was read and matches. + private static bool IsSameFile(AbsoluteFilePath file, FileIdentity identity) => + FileIdentity.TryGet(file.WeakString, out FileIdentity fileIdentity) && fileIdentity == identity; + /// /// Preserves every copy in a group, because the copy that would have been kept no longer /// holds the group's content. diff --git a/FileDeduplicator/FileIdentity.cs b/FileDeduplicator/FileIdentity.cs new file mode 100644 index 0000000..7684e4c --- /dev/null +++ b/FileDeduplicator/FileIdentity.cs @@ -0,0 +1,116 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.FileDeduplicator; + +using System.IO; +using System.Runtime.InteropServices; + +using Microsoft.Win32.SafeHandles; + +/// +/// Identifies the file a path leads to, as opposed to the path itself. +/// +/// +/// Two paths can lead to one file: a hardlink, a Linux bind mount, a share mounted twice, or a macOS +/// directory hardlink. None of these is a reparse point, so the scan walks into both paths and both +/// hash the same. Deleting one of them as the "other copy" then deletes the copy being kept, or, for +/// a hardlink, removes a name while reclaiming nothing. The file system's own identity for a file is +/// the device and the file's index on it: (st_dev, st_ino) on Unix, and +/// (VolumeSerialNumber, FileIndexHigh/Low) on Windows. +/// +/// The device or volume the file lives on. +/// The file's index (inode number) on that device. +internal readonly partial record struct FileIdentity(ulong Device, ulong Index) +{ + /// + /// Reads the identity of the file at . + /// + /// The file to identify. + /// The file's identity, when it could be read. + /// when the file is gone or cannot be inspected. + internal static bool TryGet(string path, out FileIdentity identity) => + OperatingSystem.IsWindows() ? TryGetWindows(path, out identity) : TryGetUnix(path, out identity); + + /// + /// Reads the identity through SystemNative_Stat, the same runtime shim + /// uses, for the same reason: its structure has fixed-width fields on every Unix. + /// + private static bool TryGetUnix(string path, out FileIdentity identity) + { + if (Stat(path, out FileStatus status) != 0) + { + identity = default; + return false; + } + + identity = new FileIdentity(unchecked((ulong)status.Dev), unchecked((ulong)status.Ino)); + return true; + } + + private static bool TryGetWindows(string path, out FileIdentity identity) + { + try + { + using SafeFileHandle handle = File.OpenHandle(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + + if (!GetFileInformationByHandle(handle, out ByHandleFileInformation info)) + { + identity = default; + return false; + } + + identity = new FileIdentity(info.VolumeSerialNumber, ((ulong)info.FileIndexHigh << 32) | info.FileIndexLow); + return true; + } + catch (IOException) + { + identity = default; + return false; + } + catch (UnauthorizedAccessException) + { + identity = default; + return false; + } + } + + [LibraryImport("libSystem.Native", EntryPoint = "SystemNative_Stat", StringMarshalling = StringMarshalling.Utf8)] + [DefaultDllImportSearchPaths(DllImportSearchPath.SafeDirectories)] + private static partial int Stat(string path, out FileStatus output); + + [LibraryImport("kernel32.dll", SetLastError = true)] + [DefaultDllImportSearchPaths(DllImportSearchPath.System32)] + [return: MarshalAs(UnmanagedType.Bool)] + private static partial bool GetFileInformationByHandle(SafeFileHandle file, out ByHandleFileInformation information); + + /// + /// The runtime's FileStatus, of which only Dev and Ino are read. They follow + /// the flags, mode, owner, size and five timestamp pairs, all fixed-width. The buffer is oversized, + /// as in , so a field the runtime appends later cannot overrun it. + /// + [StructLayout(LayoutKind.Explicit, Size = 256)] + private struct FileStatus + { + [FieldOffset(88)] + internal long Dev; + + [FieldOffset(104)] + internal long Ino; + } + + /// + /// BY_HANDLE_FILE_INFORMATION, of which only the volume serial number and file index are read. + /// + [StructLayout(LayoutKind.Explicit, Size = 52)] + private struct ByHandleFileInformation + { + [FieldOffset(28)] + internal uint VolumeSerialNumber; + + [FieldOffset(44)] + internal uint FileIndexHigh; + + [FieldOffset(48)] + internal uint FileIndexLow; + } +} diff --git a/README.md b/README.md index dde47bc..d133e20 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,8 @@ When duplicates are found, the file with the **shortest filename** is kept. If t Empty files are reported as a group but never deleted. Deleting one frees nothing, and an empty file such as `__init__.py`, `.gitkeep` or `py.typed` usually matters because it exists. +One file reached at two paths is not a duplicate of itself. A hardlink, a bind mount, or a share mounted twice gives one file several paths, and deleting one of them either deletes the copy being kept or frees nothing. Such paths are counted as one file, and deletion never removes a path that leads to the kept file. + For example, given these duplicates: - `C:\photos\vacation\IMG_20240101_123456.jpg` - `C:\photos\beach.jpg`