Skip to content

Commit 8085847

Browse files
Merge pull request #3220 from Spirit019/add-duplicate-file-finder
feat: add duplicate_file_finder utility script and update README
2 parents 40f4cd2 + f58b50e commit 8085847

1 file changed

Lines changed: 144 additions & 0 deletions

File tree

‎duplicate_file_finder.py‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
"""
2+
Duplicate File Finder
3+
---------------------
4+
Scans a target directory to identify duplicate files by comparing their
5+
SHA-256 checksums in memory-efficient chunks.
6+
7+
Features:
8+
- Size-first filtering for fast scanning
9+
- Memory-efficient chunked reading for large files
10+
- Handles nested subdirectories and permission errors gracefully
11+
12+
Usage:
13+
python duplicate_file_finder.py [directory_path]
14+
"""
15+
16+
import argparse
17+
from collections import defaultdict
18+
import hashlib
19+
import os
20+
from pathlib import Path
21+
from typing import DefaultDict, Dict, List
22+
23+
24+
def calculate_file_hash(filepath: Path, chunk_size: int = 65536) -> str:
25+
"""
26+
Calculate the SHA-256 hash of a file reading in chunks.
27+
28+
:param filepath: Path to the file.
29+
:param chunk_size: Size of byte chunks to read at a time.
30+
:return: Hexadecimal SHA-256 hash string.
31+
"""
32+
hasher = hashlib.sha256()
33+
try:
34+
with open(filepath, "rb") as file_obj:
35+
while chunk := file_obj.read(chunk_size):
36+
hasher.update(chunk)
37+
return hasher.hexdigest()
38+
except (PermissionError, OSError) as error:
39+
print(f"[!] Warning: Could not read {filepath}: {error}")
40+
return ""
41+
42+
43+
def find_duplicate_files(target_dir: str) -> Dict[str, List[Path]]:
44+
"""
45+
Scan a directory tree and return a dictionary mapping file hashes
46+
to lists of duplicate file paths.
47+
48+
:param target_dir: The root directory to scan.
49+
:return: Dictionary of {hash: [list_of_duplicate_paths]}.
50+
"""
51+
path = Path(target_dir).resolve()
52+
if not path.is_dir():
53+
raise NotADirectoryError(f"Directory not found: {target_dir}")
54+
55+
print(f"[*] Scanning directory: {path}")
56+
57+
# Step 1: Group files by size (files with unique sizes cannot be duplicates)
58+
size_map: DefaultDict[int, List[Path]] = defaultdict(list)
59+
total_files = 0
60+
61+
for root, _, files in os.walk(path):
62+
for filename in files:
63+
file_path = Path(root) / filename
64+
try:
65+
if not file_path.is_symlink() and file_path.exists():
66+
file_size = file_path.stat().st_size
67+
if file_size > 0: # Skip empty files
68+
size_map[file_size].append(file_path)
69+
total_files += 1
70+
except (PermissionError, OSError):
71+
continue
72+
73+
print(f"[*] Found {total_files} files. Checking for duplicate candidates...")
74+
75+
# Step 2: For file sizes with multiple files, compare hashes
76+
hash_map: DefaultDict[str, List[Path]] = defaultdict(list)
77+
78+
for size, file_list in size_map.items():
79+
if len(file_list) > 1:
80+
for file_path in file_list:
81+
file_hash = calculate_file_hash(file_path)
82+
if file_hash:
83+
hash_map[file_hash].append(file_path)
84+
85+
# Return only groups that have duplicates (more than 1 path)
86+
duplicates = {h: paths for h, paths in hash_map.items() if len(paths) > 1}
87+
return duplicates
88+
89+
90+
def format_size(bytes_size: int) -> str:
91+
"""Format bytes into a human-readable string (KB, MB, GB)."""
92+
for unit in ["B", "KB", "MB", "GB", "TB"]:
93+
if bytes_size < 1024.0:
94+
return f"{bytes_size:.2f} {unit}"
95+
bytes_size /= 1024.0
96+
return f"{bytes_size:.2f} PB"
97+
98+
99+
def main() -> None:
100+
parser = argparse.ArgumentParser(
101+
description="Find duplicate files in a directory using SHA-256 hashing."
102+
)
103+
parser.add_argument(
104+
"directory",
105+
nargs="?",
106+
default=".",
107+
help="Target directory path to scan (default: current directory)",
108+
)
109+
args = parser.parse_args()
110+
111+
try:
112+
duplicates = find_duplicate_files(args.directory)
113+
except NotADirectoryError as e:
114+
print(f"[!] Error: {e}")
115+
return
116+
117+
if not duplicates:
118+
print("\n[+] No duplicate files found.")
119+
return
120+
121+
total_duplicate_groups = len(duplicates)
122+
total_wasted_space = 0
123+
124+
print(f"\n{'='*60}")
125+
print(f"[+] Found {total_duplicate_groups} group(s) of duplicate files:")
126+
print(f"{'='*60}\n")
127+
128+
for index, (file_hash, file_paths) in enumerate(duplicates.items(), start=1):
129+
file_size = file_paths[0].stat().st_size
130+
wasted = file_size * (len(file_paths) - 1)
131+
total_wasted_space += wasted
132+
133+
print(f"Group #{index} (Size: {format_size(file_size)} each, SHA-256: {file_hash[:12]}...):")
134+
for path in file_paths:
135+
print(f" -> {path}")
136+
print()
137+
138+
print(f"{'='*60}")
139+
print(f"[+] Total duplicate wasted space: {format_size(total_wasted_space)}")
140+
print(f"{'='*60}")
141+
142+
143+
if __name__ == "__main__":
144+
main()

0 commit comments

Comments
 (0)