1. 核心觀念：從原始數據到模型就緒

import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer, KNNImputer

# 模擬數據：Age 有缺失
df = pd.DataFrame({
    'Age': [25, np.nan, 30, 45, np.nan],
    'Salary': [50000, 60000, 55000, 100000, 72000]
})

# 方法 1: 簡單統計量填補 (Simple Imputation)
# 適用於 MCAR，但會減少數據的變異數 (Variance)
simple_imp = SimpleImputer(strategy='median') 
df['Age_Simple'] = simple_imp.fit_transform(df[['Age']])

# 方法 2: KNN 填補 (Advanced)
# 利用特徵空間的距離，找最像的 K 個樣本平均
# 較能保留數據結構，適用於 MAR
knn_imp = KNNImputer(n_neighbors=2)
df_knn = knn_imp.fit_transform(df[['Age', 'Salary']])

import pandas as pd

df = pd.DataFrame({
    'User_ID': [101, 102, 101, 103],
    'Transaction': ['A', 'B', 'A', 'C'],
    'Amount': [100, 200, 100, 300]
})

# 1. 檢查重複 (全欄位完全相同)
print(f"重複筆數: {df.duplicated().sum()}") 
# Output: 1

# 2. 刪除重複 (預設保留第一筆 keep='first')
df_clean = df.drop_duplicates()

# 3. 針對特定子集 (Subset) 處理
# 例如：假設同一 User_ID 只能有一筆紀錄，保留最新的一筆 (假設資料已按時間排序)
df_unique_users = df.drop_duplicates(subset=['User_ID'], keep='last')

import pandas as pd

# 模擬髒資料
df = pd.DataFrame({
    'Price': ['$1,200', '$500', 'Not Available'], # 混雜字元
    'Discount': ['10%', '5%', '0%'],             # 百分比
    'Is_Member': ['Yes', 'No', 'Yes'],           # 文字布林
    'Date': ['2023/01/01', '2023-02-15', 'Error']
})

# 1. 處理數值字串 (去除 $ 和 ,)
# errors='coerce' 會將 'Not Available' 轉為 NaN，方便後續補值
df['Price'] = pd.to_numeric(
    df['Price'].str.replace('$', '').str.replace(',', ''), 
    errors='coerce'
)

# 2. 處理百分比 (去除 % 並除以 100)
df['Discount'] = df['Discount'].str.replace('%', '').astype(float) / 100

# 3. 處理二元類別 (Mapping)
df['Is_Member'] = df['Is_Member'].map({'Yes': 1, 'No': 0})

# 4. 處理日期 (轉為 Datetime 物件)
df['Date'] = pd.to_datetime(df['Date'], errors='coerce')

print(df.dtypes)
# Output:
# Price        float64
# Discount     float64
# Is_Member    int64
# Date         datetime64[ns]

# 離群值處理策略：Capping (蓋帽法 / Winsorization)
# 不直接刪除數據，而是將超過邊界的值替換為邊界值
# 這樣可以保留樣本數，但限制極端值的影響

def cap_outliers(series):
    Q1 = series.quantile(0.25)
    Q3 = series.quantile(0.75)
    IQR = Q3 - Q1
    
    lower_limit = Q1 - 1.5 * IQR
    upper_limit = Q3 + 1.5 * IQR
    
    # 將大於上界的設為上界，小於下界的設為下界
    return series.clip(lower=lower_limit, upper=upper_limit)

# 應用
df['Age_Cleaned'] = cap_outliers(df['Age'])

import pandas as pd
import numpy as np

df = pd.DataFrame({
    'Transaction_ID': ['T01', 'T02', 'T03', 'T04'], # 應刪除
    'Price': [100, 200, 150, 300],
    'Quantity': [1, 2, 1, 5],
    'Date': pd.to_datetime(['2023-01-01', '2023-01-02', '2023-01-03', '2023-01-08']),
    'Constant_Col': [1, 1, 1, 1] # 低變異數，應刪除
})

# 1. 刪除 (Dropping)
# 刪除 ID 欄位和 常數欄位 (axis=1 代表欄位)
cols_to_drop = ['Transaction_ID', 'Constant_Col']
df = df.drop(columns=cols_to_drop)

# 2. 新增特徵 (Creation)
# 2.1 數學交互作用：計算總金額
df['Total_Amount'] = df['Price'] * df['Quantity']

# 2.2 時間特徵提取
df['Month'] = df['Date'].dt.month
df['Weekday'] = df['Date'].dt.weekday
df['Is_Weekend'] = df['Weekday'].apply(lambda x: 1 if x >= 5 else 0)

print(df.columns)
# Output: Index(['Price', 'Quantity', 'Date', 'Total_Amount', 'Month', 'Weekday', 'Is_Weekend'], dtype='object')

import pandas as pd
import numpy as np

# 1. 原始房屋數據 (只有經緯度)
df_houses = pd.DataFrame({
    'House_ID': [1, 2],
    'Lat': [25.0330, 25.0420], # 台北某處座標
    'Lon': [121.5654, 121.5500]
})

# 2. 外部數據：捷運站清單 (從政府 Open Data 取得)
df_mrt = pd.DataFrame({
    'Station_Name': ['Taipei 101', 'Sun Yat-Sen Memorial Hall'],
    'Lat': [25.0328, 25.0415],
    'Lon': [121.5641, 121.5578]
})

# 3. 定義 Haversine 公式 (計算地球表面兩點距離)
def haversine_np(lon1, lat1, lon2, lat2):
    lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])
    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = np.sin(dlat/2.0)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2.0)**2
    c = 2 * np.arcsin(np.sqrt(a))
    km = 6367 * c
    return km

# 4. 計算每間房子距離最近捷運站的距離
def get_nearest_mrt_dist(house_row):
    # 計算該房子到"所有"捷運站的距離
    dists = haversine_np(
        house_row['Lon'], house_row['Lat'], 
        df_mrt['Lon'].values, df_mrt['Lat'].values
    )
    # 取最小值 (最近的站)
    return dists.min()

df_houses['Dist_to_MRT_km'] = df_houses.apply(get_nearest_mrt_dist, axis=1)

print(df_houses)
# House 1 (Near 101): Distance very short
# House 2 (Near SYS Hall): Distance calculated

from sklearn.preprocessing import StandardScaler, MinMaxScaler

data = [[-1, 2], [-0.5, 6], [0, 10], [1, 18]]

# StandardScaler (推薦首選)
scaler = StandardScaler()
print(scaler.fit_transform(data))

# MinMaxScaler
min_max = MinMaxScaler()
print(min_max.fit_transform(data))