数据挖掘——数据预处理（2）

约定不等于承诺〃 2022-01-21 02:57 359阅读 0赞

#统计缺失值
    train = pd.read_csv("train.csv")
    null_columns=train.columns[train.isnull().any()]
    train[null_columns].isnull().sum()

#得到dataframe中，数据类型为object的列
    obj_cols = list(train_x.select_dtypes(include=['object']).columns)
    obj_cols

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70]

#展示一列数据各数值分布情况
    import seaborn as sns
    import matplotlib.pyplot as plt
    
    ProductCD_counts = train_obj_cols['ProductCD'].value_counts()
    plt.figure(figsize=(7,4))
    sns.barplot(ProductCD_counts.index, ProductCD_counts.values, alpha=0.8)
    plt.show()

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70 1]

#使用散点图查看特征之间的关系
    # create dataframe from data in X_train
    # label the columns using the strings in iris_dataset.feature_names
    iris_dataframe = pd.DataFrame(X_train, columns=iris_dataset.feature_names)
    # create a scatter matrix from the dataframe, color by y_train
    grr = pd.scatter_matrix(iris_dataframe, c=y_train, figsize=(15, 15), marker='o',
    hist_kwds={'bins': 20}, s=60, alpha=.8, cmap=mglearn.cm3)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70 2]

#使用lasso选取特征
    indexs = np.argwhere(lasso_clf.coef_ != 0)#选取lasso分类器中系数不为0的特征
    select_cols = np.array(train_x.columns[indexs])
    select_cols = select_cols.transpose()[0]#转成训练集dataframe列名
    train_x=train_x[select_cols]#训练集只使用选取的特征

#绘制决策树特征重要性的图
    def plot_feature_importances(dt_model):
    	n_features = cancer.data.shape[1]
    	plt.barh(range(n_features), model.feature_importances_, align='center')
    	plt.yticks(np.arange(n_features), cancer.feature_names)
    	plt.xlabel("Feature importance")
    	plt.ylabel("Feature")
    	
    plot_feature_importances_cancer(tree)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70 3]

#使用get_dummies将dataframe中的object类特征值转为数值特征
    print("Original features:\n", list(data.columns), "\n")
    data_dummies = pd.get_dummies(data)
    print("Features after get_dummies:\n", list(data_dummies.columns))

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70 4]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70]: /images/20220121/5b4654277ac94b53ab2456935005b387.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70 1]: /images/20220121/607b20456e7241f99d9b70c751d3d60b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70 2]: /images/20220121/0ed90d14cfcf47cba357f62cc60351f7.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70 3]: /images/20220121/775c7fde45fd4fc9afeb531ca4b8b9e1.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70 4]: /images/20220121/48e4cf53a39840509a775e40745fb4d3.png