📊 高频数据清洗基础

# 高频数据清洗 pipeline (Python 伪代码)
import pandas as pd
import numpy as np

def clean_high_freq(df: pd.DataFrame) -> pd.DataFrame:
    # 1. 时间戳排序 & 索引化
    df = df.sort_values('timestamp').reset_index(drop=True)
    
    # 2. 缺失值前向填充 (限制最大间隔 5 笔)
    df['price'] = df['price'].ffill(limit=5)
    df['volume'] = df['volume'].fillna(0)
    
    # 3. 去除完全重复行
    df = df.drop_duplicates(subset=['timestamp', 'price', 'volume'])
    
    # 4. 异常值检测 (滚动 IQR)
    rolling_iqr = df['price'].rolling(20).quantile(0.75) - df['price'].rolling(20).quantile(0.25)
    median = df['price'].rolling(20).median()
    df['is_anomaly'] = (np.abs(df['price'] - median) > 3 * rolling_iqr)
    df = df[~df['is_anomaly']].copy()
    
    # 5. 时间戳重采样 (可选)
    # df = df.set_index('timestamp').resample('1ms').ffill().reset_index()
    return df