<jupyter_start><jupyter_text>生成用户日志本代码的目的是把用户分组，并在每个组中统计用户的行为日志，以方便后续的并行化处理。This code aims to group users into serveral groups, then statistic the user behaivors into each group. This process can simplify the operation in multi-processing<jupyter_code>import multiprocessing as mp
import time
import pandas as pd
import numpy as np



def reduce_mem_usage(df):
    """ iterate through all the columns of a dataframe and modify the data type
        to reduce memory usage.        
    """
    start_mem = df.memory_usage().sum() 
    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
    
    for col in df.columns:
        col_type = df[col].dtype
        
        if col_type != object:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
        else:
            df[col] = df[col].astype('category')

    end_mem = df.memory_usage().sum() 
    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
    
    return df

## 把[user_id, item_id]矩阵 转换成 {user_id: [item_id, item_id, ...., item_id]}字典
def generate_logs_for_each_group(matrix, q):
    user_log = dict()
    for row in matrix:
        user_log.setdefault(row[0], [])
        user_log[row[0]].append(row[1])
    print('This batc is finished')
    ## 把结果放到消息队列里
    q.put(user_log)
## 指定用几个CPU处理
CPU_NUMS = 8
# round2 train的路径
path = '../ECommAI_EUIR_round2_train_20190816/'
data = reduce_mem_usage(pd.read_csv(path+'user_behavior.csv', header=None))
user = pd.read_csv(path+'user.csv', header=None)
item = pd.read_csv(path+'item.csv', header=None)

data['day'] = data[3] // 86400
data['hour'] = data[3] // 3600 % 24

data = data.drop(3, axis=1)

data.columns = ['userID','itemID','behavoir','day','hour']
user.columns = ['userID', 'sex', 'age', 'ability']
item.columns = ['itemID', 'category', 'shop', 'band']

data = data.drop_duplicates(['userID','itemID'],keep="last")
data = data.sort_values(['day','hour'], ascending=True).reset_index(drop=True)

users = list(set(user['userID']))

user_groups = [users[i: i + len(users) // CPU_NUMS] for i in range(0, len(users), len(users) // CPU_NUMS)]

## 进程用消息队列沟通
q = mp.Queue()
for groupID in range(len(user_groups)):
    matrix = data[data['userID'].isin(user_groups[groupID])][['userID','itemID']].values
    task = mp.Process(target=generate_logs_for_each_group, args=(matrix, q, ))
    task.start()
    
start_time = time.time()
print('Waiting for the son processing')
while q.qsize() != len(user_groups):
    pass
end_time = time.time()
print("Over, the time cost is:"  + str(end_time - start_time))
for i in range(len(user_groups)):
    temp = q.get()
    ## 把生成的字典保存在文件中
    f = open('full_logs/userlogs_group' + str(i) + '.txt','w')
    f.write(str(temp))
    f.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15748440748440748, 'fraction_numerical': 0.018711018711018712, 'mean_word_length': 2.9517453798767965, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '490229', 'license': 'MIT', 'n_tokens_mistral': 1496, 'n_tokens_neox': 1458, 'n_words': 282, 'path': 'Semi-Finals/underline_trainning/Step1 itemCF_based_on_Apriori/1_generate_user_logs.ipynb', 'repo_name': '965784749-rgb/CIKM-2019-AnalytiCup', 'size': '3848'}