import zlib,os

def Find_Repeat_File(file_path,file_type):
    Catalogue = os.listdir(file_path)
    CatalogueDict = {}  # 查詢字典,方便后期查詢鍵值對對應(yīng)參數(shù)
    for each in Catalogue:
        path = (file_path + each)
        if os.path.splitext(path)[1] == file_type:
            with open(path,"rb") as fp:
                crc32 = zlib.crc32(fp.read())
                # print("[*] 文件名: {} CRC32校驗(yàn): {}".format(path,str(crc32)))
                CatalogueDict[each] = str(crc32)
    CatalogueList = []
    for value in CatalogueDict.values():
    # 該過程實(shí)現(xiàn)提取字典中的crc32特征組合成列表 CatalogueList
        CatalogueList.append(value)

    CountDict = {}
    for each in CatalogueList:
    # 該過程用于存儲(chǔ)文件特征與特征重復(fù)次數(shù),放入 CountDict
        CountDict[each] = CatalogueList.count(each)
        
    RepeatFileFeatures = []
    for key,value in CountDict.items():
    # 循環(huán)查找字典中的數(shù)據(jù),如果value大于1就存入 RepeatFileFeatures
        if value > 1:
            print("[-] 文件特征: {} 重復(fù)次數(shù): {}".format(key,value))
            RepeatFileFeatures.append(key)

if __name__ == "__main__":
    Find_Repeat_File("D://lyshark/",".png")

import zlib,os,argparse
from operator import itemgetter
from itertools import groupby

def Find_Repeat_File(file_path,file_type):
    Catalogue = os.listdir(file_path)
    CatalogueList = []
    for each in Catalogue:
        path = (file_path + each)
        if os.path.splitext(path)[1] == file_type:
            with open(path,"rb") as fp:
                crc32 = zlib.crc32(fp.read())
                # print("[*] 文件名: {} CRC32校驗(yàn): {}".format(path,str(crc32)))
                CatalogueList.append({ "CRC32": str(crc32) , "FILE": path })
    
    # 首先排序,然后根據(jù)字典中的CRC32排序
    CatalogueList.sort(key=itemgetter("CRC32"))
    for key,value in groupby(CatalogueList,key=itemgetter("CRC32")):
        # print("[*] CRC32特征碼: {}\t".format(key))
        for each in value:
            RepeatNumber = len(list(value))
            if (RepeatNumber+1) > 1:
                try:
                    print("---> 重復(fù)圖片: {} 已移除".format(each.values()))
                    os.remove(str(list(each.values())[1]))
                except FileNotFoundError:
                    pass

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-d","--dirs",dest="dirs",help="指定目錄路徑")
    parser.add_argument("-t","--types",dest="types",help="指定文件類型")
    args = parser.parse_args()
    # 使用方式: main.py -d "d://lyshark/" -t ".png"
    if args.dirs and args.types:
        os.chdir(args.dirs)
        try:
            for each in range(0,len(os.listdir())):
                Find_Repeat_File("./",args.types)
        except:
            pass
    else:
        parser.print_help()

import os, time
from zlib import crc32

# 計(jì)算目標(biāo)CRC32
def Calculation_crc32(filename):
    try:
        with open(filename,"rb") as fp:
            crc = crc32(fp.read())
            while True:
                temp = fp.read(8196)
                if not temp:
                    break
            fp.close()
            return crc
    except Exception:
        fp.close()
        return 0
    return 0

# 開始監(jiān)控目錄
def MonitoringDirectory(path_to_watch):
    crc_poll = []

    before = dict([(f, None) for f in os.listdir(path_to_watch)])
    while True:
        time.sleep (0.3)

        after = dict ([(f, None) for f in os.listdir (path_to_watch)])
        added = [f for f in after if not f in before]
        removed = [f for f in before if not f in after]

        # 新增文件時(shí)觸發(fā)
        if added:
            # print("新建文件: {}".format(added))
            for index in range(0,len(added)):
                # 計(jì)算文件CRC32
                AbsolutePath = path_to_watch + "/" + added[index]
                crc = Calculation_crc32(AbsolutePath)
                print(f"計(jì)算文件路徑: {AbsolutePath} 文件CRC32: {crc}")

                # 只要有新文件則加入到crc_pool
                if crc != 0:
                    crc_poll.append(crc)

                print("池內(nèi)數(shù)據(jù): {}".format(crc_poll))
                crc_count = 0

                # 循環(huán)池內(nèi)的所有CRC數(shù)據(jù)
                for index in range(0,len(crc_poll)):

                    # 如果當(dāng)前文件CRC與池內(nèi)某個(gè)一致則遞增
                    if crc_poll[index] == crc:
                        crc_count = crc_count + 1

                    # 只要大于2則說明有重復(fù)的
                    if crc_count >= 2:
                        try:
                            print("存在校驗(yàn)值,刪除文件: {}".format(AbsolutePath))
                            os.remove(AbsolutePath)
                            crc_poll.remove(crc)
                        except Exception:
                            pass
        # 刪除文件時(shí)觸發(fā)
        if removed:
            print("移除文件: {}".format(removed))
        before = after

if __name__ == "__main__":
    MonitoringDirectory("d://lyshark")