<jupyter_start><jupyter_text>Preparation test<jupyter_code>%run "Preparation.ipynb"<jupyter_output><empty_output><jupyter_text>Tests<jupyter_code>df2 = pd.DataFrame({
	'customData.localplayerguid' : 1,
	'playerId' : 2,
	'C' : 3,
	'D' : '4',
	'E' : pd.Categorical(["test","train","test","train","test","train"]),
	'F' : 'aaaaaa'
	}, index=[0,1,2,3,4,5])
df2
newColumns = np.unique(np.concatenate((minimalInitializationColumns, df2.columns.values)))
newColumns
df3 = df2.loc[:,newColumns]
df3
df4 = df3.rename(index=str, columns={'customData.localplayerguid' : 'userId', 'playerId': 'sessionId'})
df4
df5 = getNormalizedRedMetricsCSV( df2 )
df5<jupyter_output><empty_output><jupyter_text>getAllSessions tinkering<jupyter_code>_rmDF = rmdf152
dropna = False

#def getAllSessions( _rmDF, dropna ):
_result = _rmDF.loc[:, renamedRelevantColumns]
_result = _result[_result['type']=='start']
_result = _result.drop('type', 1)
if dropna:
    _result = _result.dropna(how='any')
_result<jupyter_output><empty_output><jupyter_text>getTestSessions tinkering<jupyter_code>_rmDF = rmdf152
_rmTestDF = dftest

# gets sessions which either:
#  - have 'android' or '...editor' as platform
#  - are in the RedMetrics test channel
#  - belong to a user who has a session of the type above
#def getTestSessions(_rmDF, _rmTestDF):

# tables of association of user-sessions
rmDFUsersSessions = getAllSessions(_rmDF, False)
rmTestDFUsersSessions = getAllSessions(_rmTestDF, False)
userSessions = pd.concat([rmDFUsersSessions,rmTestDFUsersSessions])

#  - have 'android' or '...editor' as platform
rmDFTestSessions = set(_rmDF[_rmDF['customData.platform'].apply(lambda s: str(s).endswith('editor"'))]['sessionId'])
rmDFTestSessions |= set(_rmDF[_rmDF['customData.platform'].isin(['"android"'])]['sessionId'])
#print(str(len(rmDFTestSessions)))

#  - are in the RedMetrics test channel
rmTestDFTestSessions = set(_rmTestDF['sessionId'])
#print(str(len(rmTestDFTestSessions)))

#  - belong to a user who has a session of the type above
# all the sessions above
testSessions = rmDFTestSessions | rmTestDFTestSessions
# all the users
rmDFTestUsers = set(rmDFUsersSessions[rmDFUsersSessions['sessionId'].isin(testSessions)]['userId'].dropna())
rmTestDFTestUsers = set(_rmTestDF['userId'].dropna())
testUsers = rmDFTestUsers | rmTestDFTestUsers
# all the sessions which belong to these users
result = set(rmDFUsersSessions[rmDFUsersSessions['userId'].isin(testUsers)]['sessionId'].dropna())

len(result)
testSessions = result
filteredrmDF152 = rmdf152[~rmdf152['sessionId'].isin(testSessions)]

len(rmdf152['sessionId'].unique()),len(testSessions),len(filteredrmDF152['sessionId'].unique())

len(rmdf152[rmdf152['customData.platform']=='"linuxplayer"']['userId'].unique()),\
len(filteredrmDF152[filteredrmDF152['customData.platform']=='"linuxplayer"']['userId'].unique())

_rmDF = filteredrmDF152
temporaryMax = 0
userIdOfMax = 0
_sessionsList = getAllSessions( _rmDF, True )
for userId in _rmDF[_rmDF['customData.platform']=='"linuxplayer"']['userId'].unique():
    allSessions = _sessionsList[_sessionsList['userId']==userId].drop_duplicates()
    count = allSessions['sessionId'].nunique()
    if(count > temporaryMax):
        temporaryMax = count
        userIdOfMax = userId
userIdOfMax, temporaryMax

rmdf152['customData.platform'].dropna().unique()

filteredrmDF152['customData.platform'].dropna().unique()<jupyter_output><empty_output><jupyter_text>getTestSessions tinkering<jupyter_code>_rmDF = rmdf152
_rmTestDF = normalizedRMDFTest
includeAndroid = True
includeEditor = True
includeTest = True
#def getTestSessions(_rmDF, _rmTestDF, includeAndroid = True, includeEditor = True, includeTest = True):
    
rmDFTestSessions = set()
rmTestDFTestSessions = set()

#  - have 'android' or '...editor' as platform
if(includeAndroid):
    rmDFTestSessions |= set(_rmDF[_rmDF['customData.platform'].isin(['"android"'])]['sessionId'])
if(includeEditor):
    rmDFTestSessions |= set(_rmDF[_rmDF['customData.platform'].apply(lambda s: str(s).endswith('editor"'))]['sessionId'])
#print(str(len(rmDFTestSessions)))

#  - are in the RedMetrics test channel
if(includeTest):
    rmTestDFTestSessions = set(_rmTestDF['sessionId'])
#print(str(len(rmTestDFTestSessions)))

#  - belong to a user who has a session of the type above
# all the sessions above
testSessions = rmDFTestSessions | rmTestDFTestSessions

testSessions<jupyter_output><empty_output><jupyter_text>getTestUsersSessions tinkering<jupyter_code>_rmDF = rmdf152
_rmTestDF = normalizedRMDFTest
includeAndroid = True
includeEditor = True
includeTest = True

# gets sessions which either:
#  - have 'android' or '...editor' as platform
#  - are in the RedMetrics test channel
#  - belong to a user who has a session of the type above
#def getTestUsersSessions(_rmDF, _rmTestDF, includeAndroid = True, includeEditor = True, includeTest = True):

# tables of association of user-sessions
rmDFUsersSessions = getAllSessions(_rmDF, False)
rmTestDFUsersSessions = getAllSessions(_rmTestDF, False)
userSessions = pd.concat([rmDFUsersSessions,rmTestDFUsersSessions])

testSessions = getTestSessions(_rmDF, _rmTestDF,
                               includeAndroid = includeAndroid, includeEditor = includeEditor, includeTest = includeTest)

# all the users
rmDFTestUsers = set(rmDFUsersSessions[rmDFUsersSessions['sessionId'].isin(testSessions)]['userId'].dropna())
rmTestDFTestUsers = set(_rmTestDF['userId'].dropna())
rmTestDFTestUsers.remove('')
testUsers = rmDFTestUsers | rmTestDFTestUsers
# all the sessions which belong to these users
allTestSessions = set(rmDFUsersSessions[rmDFUsersSessions['userId'].isin(testUsers)]['sessionId'].dropna())

#(testUsers,allTestSessions)
testUsers = pd.DataFrame(data=list(testUsers), columns=['userId'])
testUsers152Path = dataFolderPath + dataFilesNamesStem + ".1.52-testUsers.csv"
testUsers.to_csv(testUsers152Path, encoding=csvEncoding)
testUsers2 = pd.read_csv(testUsers152Path, dtype=str).loc[:,['userId']]
(testUsers == testUsers2).all()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1365594961882665, 'fraction_numerical': 0.014418296320848525, 'mean_word_length': 7.5240112994350286, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '635571', 'license': 'CC0-1.0', 'n_tokens_mistral': 2076, 'n_tokens_neox': 2032, 'n_words': 375, 'path': 'v1.52/Utilities/Preparation test.ipynb', 'repo_name': 'CyberCRI/herocoli-metrics-redwire', 'size': '6034'}