# 直接刪除缺失值 - 可能導(dǎo)致樣本代表性喪失
df = df.dropna()

# 用固定值填充 - 可能扭曲數(shù)據(jù)分布
df['income'] = df['income'].fillna(0)

# 1. 分析缺失模式
import missingno as msno
msno.matrix(df)

# 2. 根據(jù)缺失機(jī)制選擇適當(dāng)方法
# 隨機(jī)缺失 - 可用均值/中位數(shù)/眾數(shù)填充
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy='median')
df['income'] = imputer.fit_transform(df[['income']])

# 非隨機(jī)缺失 - 需要模型預(yù)測
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
iter_imputer = IterativeImputer()
df['income'] = iter_imputer.fit_transform(df[['income']])

# 縮尾處理示例
def winsorize(series, lower=0.05, upper=0.95):
    quantiles = series.quantile([lower, upper])
    return series.clip(quantiles.iloc[0], quantiles.iloc[1])

df['income'] = winsorize(df['income'])

# 計(jì)算兩列差值 - 低效循環(huán)方式
result = []
for i in range(len(df)):
    result.append(df['price'][i] - df['cost'][i])
df['profit'] = result

# 直接列運(yùn)算 - 效率提升數(shù)百倍
df['profit'] = df['price'] - df['cost']

# 使用numba加速循環(huán)
from numba import jit

@jit(nopython=True)
def calculate_profit(prices, costs):
    profits = np.empty(len(prices))
    for i in range(len(prices)):
        profits[i] = prices[i] - costs[i]
    return profits

df['profit'] = calculate_profit(df['price'].values, df['cost'].values)

df.eval('profit = price * quantity - cost', inplace=True)

# 同時(shí)處理多列計(jì)算
df[['profit','margin']] = df.eval('profit=price-cost; margin=profit/price')

# 避免循環(huán)編碼類別變量
df['category_code'] = df['category'].astype('category').cat.codes

# 查看內(nèi)存使用
df.info(memory_usage='deep')

# 優(yōu)化數(shù)值列
df['user_id'] = df['user_id'].astype('int32')

# 優(yōu)化字符串列
df['category'] = df['category'].astype('category')

# 分塊讀取CSV
chunk_size = 100000
chunks = pd.read_csv('large_data.csv', chunksize=chunk_size)

results = []
for chunk in chunks:
    # 處理每個(gè)分塊
    processed = chunk[chunk['value'] > 0]
    results.append(processed)

# 合并結(jié)果
final_df = pd.concat(results)

import dask.dataframe as dd

# 創(chuàng)建Dask DataFrame
ddf = dd.read_csv('very_large_*.csv')

# 執(zhí)行延遲計(jì)算
result = ddf.groupby('category')['value'].mean().compute()

import seaborn as sns
import matplotlib.pyplot as plt

# 專業(yè)級(jí)可視化示例
plt.figure(figsize=(10,6))
sns.set_style("whitegrid")

ax = sns.barplot(x='category', y='value', data=df, 
                 ci=95,  # 95%置信區(qū)間
                 palette="viridis")

ax.set_title("Category Performance with 95% CI", pad=20)
ax.set_xlabel("Product Category", labelpad=15)
ax.set_ylabel("Average Value", labelpad=15)

# 添加數(shù)據(jù)標(biāo)簽
for p in ax.patches:
    ax.annotate(f"{p.get_height():.1f}", 
                (p.get_x() + p.get_width() / 2., p.get_height()),
                ha='center', va='center', 
                xytext=(0, 10), 
                textcoords='offset points')

plt.tight_layout()
plt.show()

import plotly.express as px

fig = px.scatter(df, x='price', y='sales', 
                 color='category', size='profit',
                 hover_data=['product_name', 'region'],
                 trendline="lowess",
                 title="Price vs Sales by Category")

fig.update_layout(width=1000, height=600)
fig.show()

import shap
from sklearn.ensemble import RandomForestRegressor

# 訓(xùn)練模型
model = RandomForestRegressor()
model.fit(X_train, y_train)

# 計(jì)算SHAP值
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(X_test)

# 可視化
shap.summary_plot(shap_values, X_test, plot_type="bar")
shap.summary_plot(shap_values, X_test)

import lime
import lime.lime_tabular

explainer = lime.lime_tabular.LimeTabularExplainer(
    X_train.values,
    feature_names=X_train.columns,
    class_names=['price'],
    verbose=True,
    mode='regression'
)

# 解釋單個(gè)樣本
exp = explainer.explain_instance(X_test.iloc[0], model.predict)
exp.show_in_notebook()

# 公平性審計(jì)示例
from aif360.datasets import BinaryLabelDataset
from aif360.metrics import BinaryLabelDatasetMetric

# 將數(shù)據(jù)轉(zhuǎn)換為AI Fairness格式
privileged_group = [{'gender': 1}]  # 假設(shè)1表示男性
unprivileged_group = [{'gender': 0}]

dataset = BinaryLabelDataset(
    favorable_label=1,
    unfavorable_label=0,
    df=df,
    label_names=['approved'],
    protected_attribute_names=['gender']
)

metric = BinaryLabelDatasetMetric(
    dataset,
    unprivileged_groups=unprivileged_group,
    privileged_groups=privileged_group
)

print("統(tǒng)計(jì)差異:", metric.mean_difference())
print("不一致性:", metric.consistency())