<jupyter_start><jupyter_text>数据的读取与基本信息<jupyter_code>data_path = Path('/media/bnu/data/deepshare-project/project1')
raw_train_data = pd.read_csv(data_path / 'first_round_training_data.csv')
raw_test_data = pd.read_csv(data_path / 'first_round_testing_data.csv')
raw_train_data.head()<jupyter_output><empty_output><jupyter_text>数据处理  标签分布<jupyter_code>raw_train_data['Label'] = raw_train_data['Quality_label'].map(
    {'Excellent': 0, 'Good': 1, 'Pass':2, 'Fail':3}
)
raw_train_data['Label'].hist()
plt.xlabel('Label')
plt.ylabel('Count')
plt.title('Label Distribution')
plt.show()<jupyter_output><empty_output><jupyter_text>查看数据空值<jupyter_code>print('Train Data:')
raw_train_data.isnull().sum()
print('Test Data:')
raw_test_data.isnull().sum()<jupyter_output>Test Data:<jupyter_text>类别数目<jupyter_code>print('Train Data:')
for c in raw_train_data.columns:
    print(c, len(raw_train_data[c].unique()))
print('Test Data:')
for c in raw_test_data.columns:
    print(c, len(raw_test_data[c].unique()))<jupyter_output>Test Data:
Group 120
Parameter1 5675
Parameter10 41
Parameter2 5675
Parameter3 5675
Parameter4 5675
Parameter5 130
Parameter6 76
Parameter7 15
Parameter8 23
Parameter9 16<jupyter_text>数据分布<jupyter_code>raw_train_data['Train'] = 1
raw_test_data['Train'] = 0
raw_data = pd.concat([raw_train_data, raw_test_data])

param_cols = raw_data.filter(regex='Parameter*').columns
for c in param_cols:
    fig, ax = plt.subplots(1, 1, figsize=(8, 4))
    sns.kdeplot(raw_data[raw_data['Train'] == 1][c], ax=ax, label='Train Data')
    sns.kdeplot(raw_data[raw_data['Train'] == 0][c], ax=ax, label='Test Data')
    ax.set_title(c)
    plt.show()<jupyter_output><empty_output><jupyter_text>从上面的密度估计图可以看出Parameter1-4在训练集和测试集中分布不同<jupyter_code>raw_data = raw_data.reset_index(drop=True)

for c in param_cols:
    plt.figure(figsize=(8, 4))
    plt.scatter(raw_data[raw_data['Train'] == 1].index, 
                raw_data[raw_data['Train'] == 1][c], 
                color='r')
    plt.scatter(raw_data[raw_data['Train'] == 0].index, 
                raw_data[raw_data['Train'] == 0][c], 
                color='b')
    plt.xlabel('Index')
    plt.title(c)
    plt.show()<jupyter_output><empty_output><jupyter_text>数据预处理<jupyter_code>def preprocess_data(raw_train_data, raw_test_data):
    raw_train_data['Train'], raw_test_data['Train'] = 1, 0
    # 合并训练集与测试集方便统一处理
    raw_data = raw_train_data.append(raw_test_data).reset_index(drop=True)
    # 对标签进行编码
    raw_data['Label'] = raw_data['Quality_label'].map(
        {'Excellent': 0, 'Good': 1, 'Pass':2, 'Fail':3}
    )
    # 处理小数位数清理部分异常值
    raw_data = np.round(raw_data, decimals=7)
    
    # 对Paramter5,7,8处理离群点
    for c, t in zip(
        ['Parameter5', 'Parameter7', 'Parameter8'],
        [50, 5000, 2000]
    ):
        raw_data.loc[raw_data[c] > t, c] = raw_data[c].mode().values[0]
    
    # 选择对Paramter5-10作为特征
    feature_cols = ['Parameter{}'.format(i) for i in range(5, 11)]
    x_train = raw_data.loc[raw_data['Train'] == 1, feature_cols].reset_index(drop=True)
    y_train = raw_data.loc[raw_data['Train'] == 1, 'Label']
    x_test = raw_data.loc[raw_data['Train'] == 0, feature_cols].reset_index(drop=True)
    
    # Paramter5,6为类别型特征，保证训练集与测试集的统一
    for c in ['Parameter5', 'Parameter6']:
        for v in x_train[c].unique():
            if v not in x_test[c]:
                x_train.loc[x_train[c] == v, c] = x_train[c].mode().values[0]

    # 根据观察Paramter7,8进行组合时效果较好
    x_train['Parameter7+8'] = x_train['Parameter7'] + x_train['Parameter8']
    x_test['Parameter7+8'] = x_test['Parameter7'] + x_test['Parameter8']
    
    return x_train, y_train, x_test
    
    
    
raw_train_data = pd.read_csv(data_path / 'first_round_training_data.csv')
raw_test_data = pd.read_csv(data_path / 'first_round_testing_data.csv')
x_train, y_train, x_test = preprocess_data(raw_train_data, raw_test_data)
print(x_train.shape)
print(y_train.shape)
print(x_test.shape)<jupyter_output>(6000, 7)
(6000,)
(6000, 7)<jupyter_text>模型训练<jupyter_code>model_cat = CatBoostClassifier(
    iterations=1500,
    learning_rate=0.05,
    verbose=0,
    task_type='CPU'
)
model_cat.fit(x_train, y_train)
y_pred_cat = model_cat.predict_proba(x_test)
y_pred_cat
model_xgb = XGBClassifier(
    num_leaves=30,
    reg_alpha=3,
    reg_lambda=10,
    max_depth=28,
    n_estimators=200,
    objective='multiclass',
    subsample=0.7, 
    colsample_bytree=0.7,
    random_state=0,
    task_type='CPU'
)
model_xgb.fit(x_train, y_train)
y_pred_xgb = model_xgb.predict_proba(x_test)
y_pred_xgb
sub_data = pd.read_csv(data_path / 'submit_example.csv')
prob_cols = [c for c in sub_data.columns if c not in ['Group']]

sub_data = raw_test_data[['Group']].copy()
for i, c in enumerate(prob_cols):
    # 简单融合
    sub_data[c] = 0.25 * y_pred_cat[:,i] + 0.75 * y_pred_xgb[:, i]
sub_data = sub_data.groupby('Group').mean()
sub_data.to_csv('../results/sub.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1531586431037985, 'fraction_numerical': 0.03148486695104611, 'mean_word_length': 4.946859903381642, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '404755', 'license': 'MIT', 'n_tokens_mistral': 2169, 'n_tokens_neox': 2093, 'n_words': 281, 'path': 'project1/notebooks/Project Solution.ipynb', 'repo_name': 'RegiusQuant/deepshare-project', 'size': '4923'}