<jupyter_start><jupyter_text>distribution = np.random.uniform(0, len(df), 10000).astype('int')autotags_list = []for ind in distribution:    text = df['autotags'][ind]    tags = re.split('[,:0-9]', str(text))    for tag in tags:        if len(tag) > 1:            autotags_list.append(tag)<jupyter_code>autotags_list = []

for ind, item in df['autotags'].iteritems():
    tags = re.split('[,:0-9]', str(item))
    for tag in tags:
        if len(tag) > 1:
            autotags_list.append(tag)
autotags_list = pd.Series(autotags_list).value_counts()
print(len(autotags_list))
autotags_list.to_json('json/autotags_list')
autotags_list[:20]
array = pd.DataFrame(np.zeros(len(autotags_list)), autotags_list.index)
for tag in autotags_list.index:
    array[tag] = np.zeros(len(autotags_list))

array = array.drop(columns = 0)<jupyter_output><empty_output><jupyter_text>df_ = df.iloc[distribution]counter = 0for ind, item in df_['autotags'].iteritems():        tags = re.findall('[A-z\s-\']+', str(item))    values = re.findall('[0-9\.]+', str(item))    values = np.asarray(values).astype('float')        for one in range(len(tags)):        for two in range(len(tags)):            try:                if (values[one] >= values[two]):                    array[tags[one]][tags[two]] += values[one]*values[two]                else:                    array[tags[two]][tags[one]] += values[one]*values[two]            except:                pass                counter += 1    if counter%1000 == 0:        print(str(100*counter/10000)[:5] + '%')<jupyter_code>for ind, item in df['autotags'].iteritems():
    
    tags = re.findall('[A-z\s\-\']+', str(item))
    values = re.findall('[0-9\.]+', str(item))
    values = np.asarray(values).astype('float')
    
    for one in range(len(tags)):
        for two in range(len(tags)):
            try:
                if (values[one] >= values[two]):
                    array[tags[one]][tags[two]] += values[one]*values[two]
                else:
                    array[tags[two]][tags[one]] += values[one]*values[two]
            except:
                pass
array.to_csv('tags_colocation_matrix.csv')
array
G = nx.Graph()
G.add_nodes_from(array.columns)
list_edges = []

for ind, row in array.iterrows():
    for column in array.columns:
        if row[column] > 0.:
            list_edges.append((ind, column, row[column]/2.))
G.add_weighted_edges_from(list_edges)
print(nx.info(G))
partition = community_louvain.best_partition(G)
partition = pd.Series(partition)
partition
partition.value_counts()
partition[partition == 0].index[:25] # indoor, food, people => USA ?
partition[partition == 1].index[:25] # urban, geometry => Asie / Japon ?
partition[partition == 2].index[:25] # landscape, sky, water
partition[partition == 3].index[:25] # light effect, circular => Scandinavie ?
partition[partition == 4].index[:25] # plants
df_tot = pd.read_csv('data/corpus.csv').drop(columns = 'Unnamed: 0')
df_tot = df_tot[~df_tot['autotags'].isna()]
new_autotags = []
for ind, item in df_tot['autotags'].iteritems():
    row = []
    tags = re.findall('[A-z\s\-\']+', str(item))
    values = re.findall('[0-9\.]+', str(item))
    values = np.asarray(values).astype('float')
    
    for i in range(len(tags)):
        row.append((tags[i], values[i]))
    new_autotags.append(row)
    
df_tot['json_autotags'] = new_autotags
louvain_theme_cluster = []

for ind, item in df_tot['json_autotags'].iteritems():
    partition_prob = np.zeros(5)
    for i in range(len(item)):
        try:
            cluster = partition[item[i][0]]
            partition_prob[cluster] += item[i][1]
        except:
            pass
    if np.sum(partition_prob) == 0.:
        louvain_theme_cluster.append(np.nan)
    else:
        louvain_theme_cluster.append(int(np.argmax(partition_prob)))
df_tot['tag_cluster'] = louvain_theme_cluster
df_tot = df_tot.drop(columns = ['description', 'flickr_url', 'image_direct_url',
                               'user_tags', 'autotags'])
tags = ['instagram', 'flickrandroidapp', 'flickriosapp', 'taxonomy', 'camera',
       'lastfm', 'astrometrydotnet', 'foursquare', 'geocode', 'geo', 'dopplr', 'inaturalist', 'http']

external_upload = []
for ind, item in df_tot['external_upload_mean'].iteritems():
    text = str(item)
    new_item = np.nan
    if ~(text == 'nan'):
        for tag in tags:
            found = re.findall(tag, text)
            if len(found) > 0:
                new_item = tag
                break
    external_upload.append(new_item)
df_tot['external_upload'] = external_upload
df_tot = df_tot.drop(columns = 'external_upload_mean')
df_tot.head(1)
dummies_license = pd.get_dummies(df['license_name'])
dummies_upload = pd.get_dummies(df['external_upload'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16715727502102606, 'fraction_numerical': 0.012825904121110177, 'mean_word_length': 3.23220640569395, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '81709', 'license': 'MIT', 'n_tokens_mistral': 1696, 'n_tokens_neox': 1637, 'n_words': 295, 'path': 'Tags_colocation.ipynb', 'repo_name': 'RPetitpierre/Minimalism_on_flickr', 'size': '4756'}