import csv
import os
def split_csv(path, total_len, per):
    # 如果train.csv和vali.csv存在就刪除
    if os.path.exists('C:\\algo_file\\train.csv'):
        os.remove('C:\\algo_file\\train.csv')
    if os.path.exists('C:\\algo_file\\vali.csv'):
        os.remove('C:\\algo_file\\vali.csv')
    with open(path, 'r', newline='') as file:
        csvreader = csv.reader(file)
        i = 0
        for row in csvreader:
            if i < round(total_len * per/100):
                # train.csv存放路徑
                csv_path = os.path.join("C:\\algo_file", 'train.csv')
                print(csv_path)
                # 不存在此文件的時候，就創(chuàng)建
                if not os.path.exists(csv_path):
                    with open(csv_path, 'w', newline='') as file:
                        csvwriter = csv.writer(file)
                        csvwriter.writerow(row)
                    i += 1
                # 存在的時候就往里面添加
                else:
                    with open(csv_path, 'a', newline='') as file:
                        csvwriter = csv.writer(file)
                        csvwriter.writerow(row)
                    i += 1
            elif (i >= round(total_len * per/100)) and (i < total_len):
            	# vali.csv存放路徑
                csv_path = os.path.join("C:\\algo_file", 'vali.csv')
                print(csv_path)
                # 不存在此文件的時候，就創(chuàng)建
                if not os.path.exists(csv_path):
                    with open(csv_path, 'w', newline='') as file:
                        csvwriter = csv.writer(file)
                        csvwriter.writerow(row)
                    i += 1
                # 存在的時候就往里面添加
                else:
                    with open(csv_path, 'a', newline='') as file:
                        csvwriter = csv.writer(file)
                        csvwriter.writerow(row)
                    i += 1
            else:
                break
    print("訓(xùn)練集和驗證集分離成功")
    return

if __name__ == '__main__':
? ? path = 'C:\\algo_file\\data.csv'
? ? total_len = len(open(path, 'r').readlines())# csv文件行數(shù)
? ? per = 80 # 分割比例%
? ? split_csv(path, total_len, per)

import pandas as pd
# 讀取文件數(shù)據(jù)
df=pd.read_csv('D:\\接收的文件\\lqf.csv', sep=',',engine='python',header=[0])
# 列csv文件中所有列
df.columns = ['year', 'date', 'statefips', 'countyfips', 'ctfips', 'latitude', 'longitude', 'DS_PM_pred', 'DS_PM_stdd']
#  刪除date列中的重復(fù)項，也就是說剩下的date都是已經(jīng)分好的類別
date_cate = df.drop_duplicates(subset=['date'])
print(date_cate.date)
print(range(len(date_cate)))    # date中的所有類，也就是文件數(shù)
for name in date_cate.date:
    print(name)
    # 當(dāng)date為某一個類時，存入一個小的csv文件中，文件名為類名
    df[df.date == name].to_csv("D:\\接收的文件\\data\\"+u"%s" %name+".csv") 

import pandas as pd
import os
j = 347
for info in os.listdir('D:\\接收的文件\year_02'):
    domain = os.path.abspath(r'D:\\接收的文件\year_02') # 獲取文件夾的路徑
    info = os.path.join(domain, info) # 將路徑與文件名結(jié)合起來就是每個文件的完整路徑
    # 讀取文件數(shù)據(jù)
    df = pd.read_csv(info, sep=',', engine='python', header=[0])
    # 列csv文件中所有列
    df.columns = ['year', 'date', 'statefips', 'countyfips', 'ctfips', 'latitude', 'longitude', 'DS_PM_pred', 'DS_PM_stdd']
    grouped = df.groupby('date')
    # delete duplicated data
    ind_frame = df.drop_duplicates(subset=['date'])
    # print(ind_frame.date)
    # print(range(len(ind_frame)))
    for name in ind_frame.date:
        # print(name)
        j += 1
        print("已經(jīng)掃描到第" + '{}'.format(j)+'個文件')
        df[df.date == name].to_csv("D:\\接收的文件\\data2\\" + u"%s" %name+".csv")