1+ """
2+ Duplicate File Finder
3+ ---------------------
4+ Scans a target directory to identify duplicate files by comparing their
5+ SHA-256 checksums in memory-efficient chunks.
6+
7+ Features:
8+ - Size-first filtering for fast scanning
9+ - Memory-efficient chunked reading for large files
10+ - Handles nested subdirectories and permission errors gracefully
11+
12+ Usage:
13+ python duplicate_file_finder.py [directory_path]
14+ """
15+
16+ import argparse
17+ from collections import defaultdict
18+ import hashlib
19+ import os
20+ from pathlib import Path
21+ from typing import DefaultDict , Dict , List
22+
23+
24+ def calculate_file_hash (filepath : Path , chunk_size : int = 65536 ) -> str :
25+ """
26+ Calculate the SHA-256 hash of a file reading in chunks.
27+
28+ :param filepath: Path to the file.
29+ :param chunk_size: Size of byte chunks to read at a time.
30+ :return: Hexadecimal SHA-256 hash string.
31+ """
32+ hasher = hashlib .sha256 ()
33+ try :
34+ with open (filepath , "rb" ) as file_obj :
35+ while chunk := file_obj .read (chunk_size ):
36+ hasher .update (chunk )
37+ return hasher .hexdigest ()
38+ except (PermissionError , OSError ) as error :
39+ print (f"[!] Warning: Could not read { filepath } : { error } " )
40+ return ""
41+
42+
43+ def find_duplicate_files (target_dir : str ) -> Dict [str , List [Path ]]:
44+ """
45+ Scan a directory tree and return a dictionary mapping file hashes
46+ to lists of duplicate file paths.
47+
48+ :param target_dir: The root directory to scan.
49+ :return: Dictionary of {hash: [list_of_duplicate_paths]}.
50+ """
51+ path = Path (target_dir ).resolve ()
52+ if not path .is_dir ():
53+ raise NotADirectoryError (f"Directory not found: { target_dir } " )
54+
55+ print (f"[*] Scanning directory: { path } " )
56+
57+ # Step 1: Group files by size (files with unique sizes cannot be duplicates)
58+ size_map : DefaultDict [int , List [Path ]] = defaultdict (list )
59+ total_files = 0
60+
61+ for root , _ , files in os .walk (path ):
62+ for filename in files :
63+ file_path = Path (root ) / filename
64+ try :
65+ if not file_path .is_symlink () and file_path .exists ():
66+ file_size = file_path .stat ().st_size
67+ if file_size > 0 : # Skip empty files
68+ size_map [file_size ].append (file_path )
69+ total_files += 1
70+ except (PermissionError , OSError ):
71+ continue
72+
73+ print (f"[*] Found { total_files } files. Checking for duplicate candidates..." )
74+
75+ # Step 2: For file sizes with multiple files, compare hashes
76+ hash_map : DefaultDict [str , List [Path ]] = defaultdict (list )
77+
78+ for size , file_list in size_map .items ():
79+ if len (file_list ) > 1 :
80+ for file_path in file_list :
81+ file_hash = calculate_file_hash (file_path )
82+ if file_hash :
83+ hash_map [file_hash ].append (file_path )
84+
85+ # Return only groups that have duplicates (more than 1 path)
86+ duplicates = {h : paths for h , paths in hash_map .items () if len (paths ) > 1 }
87+ return duplicates
88+
89+
90+ def format_size (bytes_size : int ) -> str :
91+ """Format bytes into a human-readable string (KB, MB, GB)."""
92+ for unit in ["B" , "KB" , "MB" , "GB" , "TB" ]:
93+ if bytes_size < 1024.0 :
94+ return f"{ bytes_size :.2f} { unit } "
95+ bytes_size /= 1024.0
96+ return f"{ bytes_size :.2f} PB"
97+
98+
99+ def main () -> None :
100+ parser = argparse .ArgumentParser (
101+ description = "Find duplicate files in a directory using SHA-256 hashing."
102+ )
103+ parser .add_argument (
104+ "directory" ,
105+ nargs = "?" ,
106+ default = "." ,
107+ help = "Target directory path to scan (default: current directory)" ,
108+ )
109+ args = parser .parse_args ()
110+
111+ try :
112+ duplicates = find_duplicate_files (args .directory )
113+ except NotADirectoryError as e :
114+ print (f"[!] Error: { e } " )
115+ return
116+
117+ if not duplicates :
118+ print ("\n [+] No duplicate files found." )
119+ return
120+
121+ total_duplicate_groups = len (duplicates )
122+ total_wasted_space = 0
123+
124+ print (f"\n { '=' * 60 } " )
125+ print (f"[+] Found { total_duplicate_groups } group(s) of duplicate files:" )
126+ print (f"{ '=' * 60 } \n " )
127+
128+ for index , (file_hash , file_paths ) in enumerate (duplicates .items (), start = 1 ):
129+ file_size = file_paths [0 ].stat ().st_size
130+ wasted = file_size * (len (file_paths ) - 1 )
131+ total_wasted_space += wasted
132+
133+ print (f"Group #{ index } (Size: { format_size (file_size )} each, SHA-256: { file_hash [:12 ]} ...):" )
134+ for path in file_paths :
135+ print (f" -> { path } " )
136+ print ()
137+
138+ print (f"{ '=' * 60 } " )
139+ print (f"[+] Total duplicate wasted space: { format_size (total_wasted_space )} " )
140+ print (f"{ '=' * 60 } " )
141+
142+
143+ if __name__ == "__main__" :
144+ main ()
0 commit comments