<jupyter_start><jupyter_text>模块导入<jupyter_code>import pandas as pd
import numpy as np
import json
from keras.utils import np_utils
data = pd.read_csv("data/train_clean.csv")<jupyter_output><empty_output><jupyter_text>连续值特征数据处理<jupyter_code>def continuous_features(data):
	def normalize_feature(df):
		return df.apply(lambda column: (column - column.mean()) / column.std())

	continuous_train = data[["time", "resolution_ratio"]]
	continuous_train = normalize_feature(continuous_train)
	continuous_train["ip"] = data["ip"]
	continuous_train = np.array(continuous_train)

	return continuous_train
continuous_train = continuous_features(data)
continuous_train.shape<jupyter_output><empty_output><jupyter_text>one-hot特征数据处理<jupyter_code>def one_hot_features(data):
	def make_one_hot(feature, one_hot_data):
		try:
			n = len(index_json[feature])
		except:
			n = int(data[feature].max())

		return np_utils.to_categorical(one_hot_data, n + 1)

	with open("data/index_json.json", "r") as f:
		index_json = json.load(f)

	one_hot_set = {}
	for i in ["apptype", "dvctype", "ntt", "carrier", "orientation", "lan"]:
		one_hot_set[i] = make_one_hot(i, np.uint8(data[i].values.reshape(len(data[i]), 1)))

	one_hot_train = np.hstack(tuple(one_hot_set.values()))

	return one_hot_train
one_hot_train = one_hot_features(data)
one_hot_train.shape<jupyter_output><empty_output><jupyter_text>embedding特征数据处理<jupyter_code>def embedding_features(data):
	embedding_train = data[["pkgname", "adunitshowid", "mediashowid", "city", \
					"adidmd5", "imeimd5","openudidmd5", "macmd5", \
					"model", "osv"]]
	embedding_train = np.array(embedding_train)

	return embedding_train
embedding_train = embedding_features(data)
embedding_train.shape<jupyter_output><empty_output><jupyter_text>整合<jupyter_code>Y_train = data["label"]
Y_train.shape
unembedding_train = np.hstack((continuous_train, one_hot_train))
unembedding_train.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13429752066115702, 'fraction_numerical': 0.003615702479338843, 'mean_word_length': 8.223809523809523, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '496390', 'license': 'MIT', 'n_tokens_mistral': 776, 'n_tokens_neox': 741, 'n_words': 105, 'path': '.ipynb_checkpoints/data_process-checkpoint.ipynb', 'repo_name': 'Fieldhunter/2019-XunFei-AI-competition-demo', 'size': '1936'}