教你用Python尋找重復文件並刪除的腳本寫法

Posted on by

在實際生活中，經常會有文件重復的困擾，即同一個文件可能既在A目錄中，又在B目錄中，更可惡的是，即便是同一個文件，文件名可能還不一樣。在文件較少的情況下，該類情況還比較容易處理，最不濟就是one by one的人工比較——即便如此，也很難保證你的眼神足夠犀利。倘若文件很多，這豈不是個impossible mission？最近在看《Python UNIX和Linux系統管理指南》，裡面就有有關“數據比較”的內容，在其基礎上，結合實際整理如下。

該腳本主要包括以下模塊:diskwalk,chechsum,find_dupes,delete。其中diskwalk模塊是遍歷文件的，給定路徑，遍歷輸出該路徑下的所有文件。chechsum模塊是求文件的md5值。find_dupes導入瞭diskwalk和chechsum模塊，根據md5的值來判斷文件是否相同。delete是刪除模塊。具體如下：

1. diskwalk.py

import os,sys
class diskwalk(object):
        def __init__(self,path):
                self.path = path
        def paths(self):
                path=self.path
                path_collection=[]
                for dirpath,dirnames,filenames in os.walk(path):
                        for file in filenames:
                                fullpath=os.path.join(dirpath,file)
                                path_collection.append(fullpath)
                return path_collection
if __name__ == '__main__':
        for file in diskwalk(sys.argv[1]).paths():
                print file

2.chechsum.py

import hashlib,sys
def create_checksum(path):
    fp = open(path)
    checksum = hashlib.md5()
    while True:
        buffer = fp.read(8192)
        if not buffer:break
        checksum.update(buffer)
    fp.close()    
    checksum = checksum.digest()
    return checksum
if __name__ == '__main__':
        create_checksum(sys.argv[1])

3. find_dupes.py

from checksum import create_checksum
from diskwalk import diskwalk
from os.path import getsize
import sys
def findDupes(path):
    record = {}
    dup = {}
    d = diskwalk(path)
    files = d.paths()
    for file in files:
        compound_key = (getsize(file),create_checksum(file))
        if compound_key in record:
            dup[file] = record[compound_key]    
        else:
            record[compound_key]=file
    return dup

if __name__ == '__main__':
    for file in  findDupes(sys.argv[1]).items():
        print "The duplicate file is %s" % file[0]
        print "The original file is %s\n" % file[1]

findDupes函數返回瞭字典dup，該字典的鍵是重復的文件，值是原文件。這樣就解答瞭很多人的疑惑，畢竟，你怎麼確保你輸出的是重復的文件呢？

4. delete.py

import os,sys
class deletefile(object):
    def __init__(self,file):
        self.file=file
    def delete(self):
        print "Deleting %s" % self.file
        os.remove(self.file)
    def dryrun(self):
        print "Dry Run: %s [NOT DELETED]" % self.file
    def interactive(self):
        answer=raw_input("Do you really want to delete: %s [Y/N]" % self.file)
        if answer.upper() == 'Y':
            os.remove(self.file)
        else:
            print "Skiping: %s" % self.file
        return
if __name__ == '__main__':
    from find_dupes import findDupes
        dup=findDupes(sys.argv[1])
    for file in dup.iterkeys():
        delete=deletefile(file)
        #delete.dryrun()
          delete.interactive()
        #delete.delete()

deletefile類構造瞭3個函數，實現的都是文件刪除功能、其中delete函數是直接刪除文件，dryrun函數是試運行，文件並沒有刪除，interactive函數是交互模式，讓用戶來確定是否刪除。這充分瞭考慮瞭客戶的需求。

總結：這四個模塊已封裝好，均可單獨使用實現各自的功能。組合起來就可批量刪除重復文件，隻需輸入一個路徑。

最後，貼個完整版本的，兼容Python 2.0, 3.0。

#!/usr/bin/python
# -*- coding: UTF-8 -*-
from __future__ import print_function
import os, sys, hashlib
class diskwalk(object):
    def __init__(self, path):
        self.path = path
    def paths(self):
        path = self.path
        files_in_path = []
        for dirpath, dirnames, filenames in os.walk(path):
            for each_file in filenames:
                fullpath = os.path.join(dirpath, each_file)
                files_in_path.append(fullpath)
        return files_in_path
def create_checksum(path):
    fp = open(path,'rb')
    checksum = hashlib.md5()
    while True:
        buffer = fp.read(8192)
        if not buffer: break
        checksum.update(buffer)
    fp.close()
    checksum = checksum.digest()
    return checksum
def findDupes(path):
    record = {}
    dup = {}
    d = diskwalk(path)
    files = d.paths()
    for each_file in files:
        compound_key = (os.path.getsize(each_file), create_checksum(each_file))
        if compound_key in record:
            dup[each_file] = record[compound_key]
        else:
            record[compound_key] = each_file
    return dup
class deletefile(object):
    def __init__(self, file_name):
        self.file_name = file_name
    def delete(self):
        print("Deleting %s" % self.file_name)
        os.remove(self.file_name)
    def dryrun(self):
        print("Dry Run: %s [NOT DELETED]" % self.file_name)
    def interactive(self):
        try:
            answer = raw_input("Do you really want to delete: %s [Y/N]" % self.file_name)
        except NameError:
            answer = input("Do you really want to delete: %s [Y/N]" % self.file_name)
        if answer.upper() == 'Y':
            os.remove(self.file_name)
        else:
            print("Skiping: %s" % self.file_name)
        return
def main():
    directory_to_check = sys.argv[1]
    duplicate_file = findDupes(directory_to_check)
    for each_file in duplicate_file:
        delete = deletefile(each_file)
        delete.interactive()
if __name__ == '__main__':
    main()

其中，第一個參數是待檢測的目錄。

到此這篇關於如何用Python尋找重復文件並刪除的文章就介紹到這瞭,更多相關Python刪除重復文件內容請搜索WalkonNet以前的文章或繼續瀏覽下面的相關文章希望大傢以後多多支持WalkonNet！

推薦閱讀：