Kaggle--处理缺失值

清疚 2024-04-18 10:29 78阅读 0赞

一：按缺失百分比去除缺失值过多的特征

#缺失超过77%的特征被去除
    many_null_cols = [col for col in train_x.columns if train_x[col].isnull().sum() / train_x.shape[0] > 0.77]
    many_null_cols_test = [col for col in test.columns if test[col].isnull().sum() / test.shape[0] > 0.77]
    
    cols_to_drop = list(set(many_null_cols))
    cols_to_drop_test = list(set(many_null_cols_test))
    
    train_x = train_x.drop(cols_to_drop, axis=1)
    test_x = test_x.drop(cols_to_drop, axis=1)

二：绘制数据集缺失百分比图

def describe_missing_values(df):
        na_percent = {}
        N = df.shape[0]
        for column in df:
            na_percent[column] = df[column].isnull().sum() * 100 / N
    
        na_percent = dict(filter(lambda x: x[1] != 0, na_percent.items()))
        plt.bar(range(len(na_percent)), na_percent.values())
        plt.ylabel('Percent')
        plt.xticks(range(len(na_percent)), na_percent.keys(), rotation='vertical')
        plt.show()
    
    print("Missing values for Train dataset")
    describe_missing_values(train_df)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70]

[参考][Link 1]

三：分类填充缺失值，object类使用每列最多的值填充，int，float类使用每列的均值填充

obj_cols = list(train_x_drop.select_dtypes(include=['object']).columns)
    for col in obj_cols:
        train_x_drop[col] = train_x_drop[col].fillna(value=train_x_drop[col].mode()[0])
        test_x_drop[col] = test_x_drop[col].fillna(value=test_x_drop[col].mode()[0])
        
    other_cols = list(train_x_drop.select_dtypes(include=['int64']).columns)   
    other_cols.append(list(train_x_drop.select_dtypes(include=['float64']).columns))
    
    for col in other_cols:
        train_x_drop[col] = train_x_drop[col].fillna(value=train_x_drop[col].mean())
        test_x_drop[col] = test_x_drop[col].fillna(value=test_x_drop[col].mean())

四：dropna方法的使用

#删除所有有缺失值的行
    dataframe.dropna(inplace=True)

#只有当某行全部值都缺失时，才删除该行
    dataframe.dropna(how='all',inplace=True)

#删除那些缺失值超过四个的行
    dataframe.dropna(thresh=4,inplace=True)

#用该缺失值后一行的值填充缺失
    dataframe.fillna(method='bfill',inplace=True)
    #用该缺失值前一行的值填充缺失
    dataframe.fillna(method='ffill',inplace=True)

五：均值填充数值型缺失变量

from sklearn.preprocessing import Imputer
    imputer = Imputer(missing_values=np.nan, strategy='median', axis=0)
    census_data[['fnlwgt']] = imputer.fit_transform(census_data[['fnlwgt']])

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/18/3f3b1771e4914b3281c7a34dc0d49a03.png
[Link 1]: https://www.kaggle.com/kostya17/simple-approach-to-handle-missing-values