<jupyter_start><jupyter_text>somte sampling<jupyter_code>def Smoter(X, y, is_random=False):
    if is_random == True:
        random_lst = list(np.random.randint(0, 1000, 4))
    elif is_random == False:
        random_lst = [0] * 4

    print("rs:", random_lst)
    sm = SMOTE(random_state=random_lst[2], kind = 0.24)
    X_smote, y_smote = sm.fit_sample(X, y)

    return X_smote, y_smote<jupyter_output><empty_output><jupyter_text>load all data<jupyter_code>path = os.getcwd()+'/../data/water/csv/20122018freshwater.csv'

data = pd.read_csv(path, na_values = np.nan)

print(data.dtypes)
print(data.shape)
drop_columns = []
continuous_features = ['pH', 'DO(mg/l)', 'CODMn(mg/l)', 'NH3-N(mg/l)']
cat_features =['水系', '点位名称', '河流名称']
# 独热编码
data_dummies = pd.get_dummies(data, columns=cat_features)
# 舍弃无用特征
data_dummies.drop(drop_columns, 1, inplace=True)
# 删除空行
data_dummies = data_dummies.dropna(axis=0)
data_dummies[data_dummies.isnull().values==True]
X = data_dummies.drop(['本周水质'], axis=1) # Series
y = data_dummies['本周水质'] # Series
print("水质分布情况:")
print(y.value_counts())
print("\n各特征类型分布情况:")
print(data_dummies.dtypes.value_counts())
data_dummies.head()
output_path = os.getcwd()+'/../data/water/csv/20122018freshwater_dummies.csv'
data_dummies.to_csv(output_path, encoding='utf-8', index=False)
# 再存储只保存数字类特征的data
drop_columns = ['水系', '点位名称', '河流名称']
data.drop(drop_columns, 1, inplace=True)
# 删除空行
data = data.dropna(axis=0)
data[data.isnull().values==True]
output_path = os.getcwd()+'/../data/water/csv/20122018freshwater_four_feature.csv'
data.to_csv(output_path, encoding='utf-8', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17266625991458207, 'fraction_numerical': 0.026845637583892617, 'mean_word_length': 6.224669603524229, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '564113', 'license': 'MIT', 'n_tokens_mistral': 759, 'n_tokens_neox': 745, 'n_words': 96, 'path': 'notebook/20122018_pre_processing.ipynb', 'repo_name': 'chenkangyang/CMAESEforWaterPrediction', 'size': '1639'}