import pandas as pd

# 創(chuàng)建Series
s = pd.Series([1, 3, 5, np.nan, 6, 8])

# 創(chuàng)建DataFrame
df = pd.DataFrame({
    'A': 1.,
    'B': pd.Timestamp('20230101'),
    'C': pd.Series(1, index=list(range(4)), dtype='float32'),
    'D': np.array([3] * 4, dtype='int32'),
    'E': pd.Categorical(["test", "train", "test", "train"]),
    'F': 'foo'
})

# 檢測缺失值
df.isna().sum()

# 可視化缺失值
import missingno as msno
msno.matrix(df)

# 刪除缺失值
df.dropna(how='any')  # 任何列有缺失就刪除
df.dropna(subset=['col1', 'col2'])  # 指定列有缺失才刪除

# 填充缺失值
df.fillna(value={'col1': 0, 'col2': 'unknown'})  # 不同列不同填充值
df.fillna(method='ffill')  # 前向填充
df.fillna(method='bfill', limit=2)  # 后向填充，最多填充2個

# 插值法填充
df.interpolate(method='linear')  # 線性插值
df.interpolate(method='time')  # 時間序列插值

# 使用描述性統(tǒng)計
df.describe()

# 使用Z-score方法
from scipy import stats
z_scores = stats.zscore(df['numeric_col'])
abs_z_scores = np.abs(z_scores)
filtered_entries = (abs_z_scores < 3)
df_clean = df[filtered_entries]

# 使用IQR方法
Q1 = df['numeric_col'].quantile(0.25)
Q3 = df['numeric_col'].quantile(0.75)
IQR = Q3 - Q1
df_clean = df[~((df['numeric_col'] < (Q1 - 1.5 * IQR)) | (df['numeric_col'] > (Q3 + 1.5 * IQR)))]

# 替換為邊界值
df['numeric_col'] = np.where(df['numeric_col'] > upper_bound, upper_bound, 
                            np.where(df['numeric_col'] < lower_bound, lower_bound, df['numeric_col']))

# 使用分箱處理
df['binned'] = pd.cut(df['numeric_col'], bins=5, labels=False)

# Min-Max歸一化
df['normalized'] = (df['col'] - df['col'].min()) / (df['col'].max() - df['col'].min())

# Z-score標準化
df['standardized'] = (df['col'] - df['col'].mean()) / df['col'].std()

# One-Hot編碼
pd.get_dummies(df, columns=['categorical_col'])

# Label編碼
from sklearn.preprocessing import LabelEncoder
df['encoded'] = LabelEncoder().fit_transform(df['categorical_col'])

# eval方法加速計算
df.eval('new_col = col1 + col2', inplace=True)

# query方法高效過濾
df.query('col1 > col2 & col3 == "value"')

# 轉(zhuǎn)換category類型
df['category_col'] = df['category_col'].astype('category')

# 查看內(nèi)存使用
df.memory_usage(deep=True)

# 多函數(shù)聚合
df.groupby('group_col').agg({
    'col1': ['mean', 'max', 'min'],
    'col2': lambda x: np.percentile(x, 95)
})

# 命名聚合(Pandas 0.25+)
df.groupby('group_col').agg(
    mean_col1=('col1', 'mean'),
    max_col2=('col2', 'max'),
    custom=('col3', lambda x: x.std() / x.mean())
)

# transform保持原DataFrame形狀
df['group_mean'] = df.groupby('group_col')['value_col'].transform('mean')

# apply靈活應(yīng)用函數(shù)
def custom_func(group):
    return (group - group.mean()) / group.std()

df.groupby('group_col').apply(custom_func)

# 降采樣
df.resample('M').mean()  # 按月平均

# 升采樣
df.resample('D').ffill()  # 按天填充

# 自定義重采樣
def custom_resampler(array_like):
    return np.sum(array_like) * 1.5

df.resample('W').apply(custom_resampler)

# 簡單滾動平均
df.rolling(window=7).mean()

# 擴展窗口
df.expanding().sum()

# 自定義滾動函數(shù)
def custom_roll(x):
    return x[-1] * 2 + x[0]

df.rolling(window=3).apply(custom_roll)

# 線圖
df.plot.line(x='date_col', y=['col1', 'col2'], figsize=(12, 6))

# 箱線圖
df.plot.box(column=['col1', 'col2', 'col3'])

# 六邊形箱圖
df.plot.hexbin(x='col1', y='col2', gridsize=20)

# 使用seaborn集成
import seaborn as sns
sns.pairplot(df, hue='category_col')

# 使用plotly交互式可視化
import plotly.express as px
fig = px.scatter_matrix(df, dimensions=['col1', 'col2', 'col3'], color='category_col')
fig.show()

# 優(yōu)化數(shù)值類型
df['int_col'] = pd.to_numeric(df['int_col'], downcast='integer')
df['float_col'] = pd.to_numeric(df['float_col'], downcast='float')

# 使用布爾類型
df['bool_col'] = df['bool_col'].astype('bool')

# 不好的做法 - 鏈式賦值
df[df['col'] > 100]['new_col'] = 1  # 可能不會生效

# 好的做法 - 使用loc
df.loc[df['col'] > 100, 'new_col'] = 1

# 使用swifter加速apply
import swifter
df['new_col'] = df['col'].swifter.apply(lambda x: x*2)

# 使用dask處理大數(shù)據(jù)
import dask.dataframe as dd
ddf = dd.from_pandas(df, npartitions=4)
result = ddf.groupby('group_col').mean().compute()

# 讀取數(shù)據(jù)
orders = pd.read_csv('orders.csv', parse_dates=['order_date'])
products = pd.read_csv('products.csv')
customers = pd.read_csv('customers.csv')

# 合并數(shù)據(jù)
merged = pd.merge(orders, products, on='product_id')
merged = pd.merge(merged, customers, on='customer_id')

# 計算RFM指標
now = pd.to_datetime('today')
rfm = merged.groupby('customer_id').agg({
    'order_date': lambda x: (now - x.max()).days,  # Recency
    'order_id': 'count',  # Frequency
    'total_price': 'sum'  # Monetary
}).rename(columns={
    'order_date': 'recency',
    'order_id': 'frequency',
    'total_price': 'monetary'
})

# RFM評分
rfm['r_score'] = pd.qcut(rfm['recency'], q=5, labels=[5,4,3,2,1])
rfm['f_score'] = pd.qcut(rfm['frequency'], q=5, labels=[1,2,3,4,5])
rfm['m_score'] = pd.qcut(rfm['monetary'], q=5, labels=[1,2,3,4,5])
rfm['rfm_score'] = rfm['r_score'].astype(str) + rfm['f_score'].astype(str) + rfm['m_score'].astype(str)

# 客戶分群
segment_map = {
    r'[4-5][4-5][4-5]': '高價值客戶',
    r'[3-5][3-5][3-5]': '潛力客戶',
    r'[1-2][1-2][1-2]': '流失風險客戶',
    r'.*': '一般客戶'
}
rfm['segment'] = rfm['rfm_score'].replace(segment_map, regex=True)