<jupyter_start><jupyter_text>get all images with relevant query<jupyter_code>re_execute = False
if re_execute:
    obj = Search(all_fields='<NAME>', dc_type='image', dc_creator='<NAME>')
    obj.execute()
    print(f'number of fetched documents: {len(obj.documents)}')
    print(f'total records {obj.total_records}')
    print('retry fetching ...')
    obj.retry()
    print(f'number of fetched documents: {len(obj.documents)}')

    df = json_normalize(json.loads(json.dumps(obj.records)))
    print('save documents to local')
    # list handling becomes a pain of we save as csv
    df.to_pickle('raw_df.pkl')
df = pd.read_pickle('raw_df.pkl')<jupyter_output><empty_output><jupyter_text>parse identifier for easier access:<jupyter_code>image_dataframe = pd.DataFrame(df['dc:identifier']\
                               .map(lambda x: x[0] if type(x) == list else x))\
                               .rename(columns={'dc:identifier':'identifier'})

df['id'] = image_dataframe
df.head()<jupyter_output><empty_output><jupyter_text>useless columns Useless because identical for all images, remove them for ease of use, next ones contain all the same content<jupyter_code>df['dc:rights'].value_counts()
df['dc:type'].value_counts()
df['dc:language'].value_counts()
df['dc:format'].value_counts()
df['dc:format'].map(lambda x: 'photo' in x[0]).value_counts()<jupyter_output><empty_output><jupyter_text>Almost all are paper, except for a few:<jupyter_code>df['dc:format'].map(lambda x: 'papier' in x[0]).value_counts()
df[~df['dc:format'].map(lambda x: 'papier' in x[0])].id<jupyter_output><empty_output><jupyter_text>we don't care about those, so remove:<jupyter_code>df = df[df['dc:format'].map(lambda x: 'papier' in x[0])]
df['dc:identifier'].head()
df['dc:identifier'].map(lambda x: x[0]).duplicated().value_counts()
df['dc:publisher'].value_counts()<jupyter_output><empty_output><jupyter_text>removing them<jupyter_code>del df['dc:type']
del df['dc:language']
del df['dc:format']
del df['dc:identifier']
del df['dc:rights']
del df['dc:publisher']<jupyter_output><empty_output><jupyter_text>getting information about subjects<jupyter_code>#some objects are stored as list, others arent, use if else to handle
df['subject_is_list'] = df['dc:subject'].map(lambda x: type(x) == list)
df['len_list'] = df[df.subject_is_list]['dc:subject'].map(len)<jupyter_output><empty_output><jupyter_text>length is non uniform ... we will have to unnest<jupyter_code>df.len_list.value_counts()
df[df.subject_is_list]['dc:subject'].map(lambda x: x).value_counts().head()<jupyter_output><empty_output><jupyter_text>unnest list to get better view<jupyter_code>to_unnest = df[df.subject_is_list]['dc:subject'].tolist()
to_unnest = df[df.subject_is_list][['dc:subject', 'id']].apply(lambda x: (x['dc:subject'], x.id), axis=1).tolist()
unnested = [(i, id_) for (list_, id_) in to_unnest for i in list_]<jupyter_output><empty_output><jupyter_text>we can also use the **title**, more about this later<jupyter_code>df[~df.subject_is_list][['dc:subject', 'id', 'dc:title']].head()
to_append = df[~df.subject_is_list][['dc:subject', 'id']].rename(columns={'dc:subject':'descriptor'})
unnested.__len__()
subject_frame = pd.DataFrame(unnested, columns=['descriptor', 'id'])
subject_frame = pd.concat([subject_frame, to_append], sort=False)
subject_frame.count()<jupyter_output><empty_output><jupyter_text>associate each descriptior of one object (given by id)<jupyter_code>subject_frame.head()<jupyter_output><empty_output><jupyter_text>do some keyword searching<jupyter_code>subject_frame['19e'] = subject_frame.descriptor.map(lambda x: x[-len('-- 19e siècle'):] == '-- 19e siècle')
subject_frame['Portraits'] = subject_frame.descriptor.map(lambda x: 'portraits' in x.lower())
is_19e = subject_frame[subject_frame['19e']].id.unique()
subject_frame[subject_frame.Portraits].id.unique().__len__()
portrait_ids = subject_frame[subject_frame.Portraits].id.unique()<jupyter_output><empty_output><jupyter_text>dessin humoristiques & caricatures seams important<jupyter_code>subject_frame[~subject_frame.id.isin(portrait_ids)].head()<jupyter_output><empty_output><jupyter_text>we also get sculptures!<jupyter_code>subject_frame[~subject_frame.id.isin(portrait_ids)].descriptor.value_counts().head()
subject_frame['caricature'] = subject_frame.descriptor.map(lambda x: 'Caricatures et dessins humoristiques' in x)
subject_frame[~subject_frame.id.isin(portrait_ids) & ~subject_frame.caricature].descriptor.value_counts().head()<jupyter_output><empty_output><jupyter_text>Extracting individual names:<jupyter_code>import re
name = "<NAME> (1827-1892)"
name_2 = "Gobin (1982"
name_3 = '<NAME> -- Portraits'
def find_names(name):
    matches = re.findall('([\w\'.-]* [\w\'.-]* [\w\'.]*|[\w\'.-]* [\w\'.]*|[\w\'.-]*|[\w\'.]*\, [\w\'.-]*[ \w\'.]+) \([\d]{2}', name)
    return [x for x in matches if len(x) > 0]

def find_names_undated(name):
    matches = re.findall('([\w\'.-]*|[\w\'.]*\, [\w\'.-]*[ \w\'.]+) -- Portraits', name)
    return [x for x in matches if len(x) > 0]
find_names(name), find_names(name_2), find_names_undated(name_3)
subject_frame.dropna()[subject_frame.dropna().named_subject.map(lambda x: 'Balthasar' in x)]<jupyter_output><empty_output><jupyter_text>unicode normalize first<jupyter_code>import unicodedata
subject_frame['descriptor'] = subject_frame.descriptor.map(lambda x: unicodedata.normalize('NFC', x))
subject_frame['subject_name'] = subject_frame.descriptor.map(find_names)
no_name = subject_frame.subject_name.map(len) == 0
subject_frame.loc[ no_name, 'subject_name'] = subject_frame[no_name].descriptor.map(find_names_undated)
no_name = subject_frame.subject_name.map(len) == 0
subject_frame[no_name]['descriptor'].drop_duplicates()
named_subjects = subject_frame[subject_frame.subject_name.map(len) > 0].id.unique()<jupyter_output><empty_output><jupyter_text>number of images we can find a name to:<jupyter_code>len(named_subjects)<jupyter_output><empty_output><jupyter_text>at most 1837<jupyter_code>subject_frame[subject_frame.subject_name.map(len) > 0].subject_name.map(str).unique().__len__()
subject_frame[subject_frame.subject_name.map(len) > 0].id.unique().__len__()
subject_frame.loc[~no_name, 'named_subject'] = subject_frame[~no_name].subject_name.map(lambda x: x[0])
subject_frame.named_subject.value_counts().head()
#explore_names[explore_names.index.map(len) < 10]
subject_frame.named_subject.unique().__len__()
(subject_frame.named_subject.value_counts() > 1).value_counts()
explore_names = subject_frame.named_subject.value_counts()
explore_names.hist(bins=50)<jupyter_output><empty_output><jupyter_text>number of names we have:<jupyter_code>subject_frame.named_subject.value_counts().count()<jupyter_output><empty_output><jupyter_text>Create named_subject.pkl<jupyter_code>subject_frame[subject_frame.named_subject.notna()].to_pickle('named_subject.pkl')
subject_frame[subject_frame.named_subject.notna()].named_subject.map(len).value_counts()<jupyter_output><empty_output><jupyter_text>how many resources does that correspond to:<jupyter_code>id_identified = subject_frame[subject_frame.named_subject.notna()].id.unique()<jupyter_output><empty_output><jupyter_text>about half of the images belong to known person<jupyter_code>subject_frame[subject_frame.named_subject.notna()].id.unique().__len__()
len(set(subject_frame.id.unique())) # nbr unique ids<jupyter_output><empty_output><jupyter_text>a lot of them don't tell us the name in the source, but it can be found in the title<jupyter_code>len(set(subject_frame.id.unique()).difference(set(id_identified)))
subject_frame[~subject_frame.id.isin(named_subjects)].descriptor.value_counts().head(10)
subject_frame[~subject_frame.id.isin(portrait_ids) & ~subject_frame.id.isin(named_subjects)].descriptor.value_counts().head()
subject_frame[~subject_frame.id.isin(portrait_ids)].head()
df['dc:title'].map(lambda x: len(x) if type(x) == list else 1).value_counts()
df['nbr_title'] = df['dc:title'].map(lambda x: len(x) if type(x) == list else 1)
df[df.nbr_title == 1]['dc:title'].sample(10).tolist()
def find_names_title(name):
    return [x.strip() for x in re.findall('([\S\s ^(\:)]*) \: ', name) if len(x.strip()) > 0]
def extract_from_brackets(name):
    return re.findall('\[([\w \-\,\.]*)\]', name)
def fix_bad_naming(name):
    return re.sub('[\[\]]*', '', name)
text_1 = 'Huguet. Folies-Dramatiques. La petite fronde : [photographie, tirage de démonstration] / [<NAME>]'
text_2 = 'Armand.[lol] Châtelet : [photographie :,'
fix_bad_naming(text_2)
extract_from_brackets(text_1)
df[df.nbr_title == 1]['dc:title'].map(extract_from_brackets).value_counts().head()
df.loc[df.nbr_title == 1, 'first_title'] = df[df.nbr_title == 1]['dc:title'].map(find_names_title)
df.first_title.dropna().map(len).value_counts()
df[df.first_title.map(lambda x: len(x)==0 if type(x) ==list else False)]['dc:title'].tolist()
df.loc[df.first_title.notna(), 'first_title'] = df[
    df.first_title.notna()].first_title.map(
    lambda x: fix_bad_naming(x[0]) if len(x)>0  else x)#.value_counts()
df.count()
df = pd.merge(df, subject_frame[['id', 'named_subject']].dropna(), on='id', how='outer')
df.count()
df.loc[df.first_title.isna() & df['dc:title'].notna(), 'first_title'] = df[    df.first_title.isna( )& df['dc:title'].notna()]['dc:title'].map(lambda x: fix_bad_naming(find_names_title(x[0])[0])).tolist()
df[df.named_subject.isna()]['dc:subject'].value_counts().index.tolist().__len__()
df.named_subject.dropna().map(lambda x: 'Charlaviski' in x).value_counts()
df[df.first_title.notna() & df.named_subject.isna() & df['dc:subject'].map(lambda x: 'collectifs' in x)].first_title<jupyter_output><empty_output><jupyter_text>now that we have extracted names for most people, get years lived where available (from subject)<jupyter_code>def get_years(string):
    x = re.findall('\(([\d{4}  -]+)\)', string)
    if len(x) == 0:
        return None
    return x[0]

def get_years_special(string):
    x = re.findall('\(([\d{4}\d{2}  -\.\?]+)[\S \;]*\)', string)
    if len(x) == 0:
        return None
    return x[0]
subject_frame['years'] = subject_frame.descriptor.map(get_years) 
get_years_special('Febvre, Paul (18..-1928)')<jupyter_output><empty_output><jupyter_text>41 for which we can't find perfect match<jupyter_code>unmatched = subject_frame.years.isna() & subject_frame.descriptor.map(lambda x: '(1' in x)
subject_frame[unmatched].count()
subject_frame[unmatched].descriptor
subject_frame.loc[unmatched, 'years'] = subject_frame[unmatched].descriptor.map(get_years_special)
subject_frame[subject_frame.years.notna()].id.unique().__len__()
id_years = subject_frame[subject_frame.years.notna()].id.unique()
df = pd.merge(df, subject_frame[['id', 'years']].dropna(), on='id', how='outer')<jupyter_output><empty_output><jupyter_text>we never have more than one named individual ... it seems<jupyter_code>(df.groupby(id).named_subject.count() > 1).value_counts()<jupyter_output><empty_output><jupyter_text>we can't get year or name of subject, who are we looking at<jupyter_code>df[df.named_subject.isna() | df.years.isna()].count()
df[
    df.named_subject.isna() |\
    df.years.isna() &\
    df['dc:subject'].map(lambda x: 'Portraits de studio -- 19e siècle' in x) &\
    ~df['dc:subject'].map(lambda x: 'Théâtre' in x)
]['dc:title'].value_counts().sample(10)
matches = df['dc:title'].map(lambda x: re.match('(?!\:)([\s\S]*) :', str(x)))
df[matches.isna()]['dc:title']
matches.dropna().map(lambda x: re.sub('\[|\]|\:', '', x.group(0))).value_counts().head(100).index.tolist()
def find_keywords(x):
    
    if not x:
        return x
    x = str(x)
    l = re.findall('(?!\d)[\w]*', x) # take all words, exclude years
    l = list(set([word for word in l if len(word) > 0])) # drop duplicates in one entry
    return l
tag_list = pd.Series([x for i in df['dc:subject'].map(find_keywords).tolist() for x in i]).value_counts().reset_index()
tag_list_title = pd.Series([x for i in df['dc:title'].map(find_keywords).tolist() for x in i]).value_counts().reset_index()
tag_list_title[tag_list_title['index'].map(lambda x: x.islower() and len(x) > 2)]['index'].head(30).tolist()
tag_list[tag_list['index'].map(lambda x: x.islower() and len(x) > 2)]['index'].tolist()<jupyter_output><empty_output><jupyter_text>associate tags and ids in new dataframe. Remove useless tags. save tag-id association as json file<jupyter_code># clear useless tags 

from itertools import compress


def tag_exclude(tag):
    
    result = False
    if len(tag) < 3:
        result = False
    elif len(tag) == 3:
        if tag.islower() and tag != 'roi':
            result = False
        else:
            result = True
    else:
        # this list may change
        if tag in  ['siècle', 'Portraits', 'studio', 'Scènes', 'collectifs', 'photographie', 'Nadar']:
            result = False
        else:
            result = True
    return result

def tag_list_exclude(tag_list):
    mask = list(map(tag_exclude, tag_list))
    return list(compress(tag_list, mask))
tag_frame = pd.DataFrame(df.id)
tag_frame['tags'] = df['dc:subject'].map(find_keywords) + df['dc:title'].map(find_keywords)
tag_frame['tags'] = tag_frame['tags'].apply(lambda x: list(set(x)))
tag_frame['tags'] = tag_frame.tags.apply(tag_list_exclude)
tag_explode = tag_frame.tags.apply(pd.Series) \
            .merge(tag_frame, right_index=True, left_index=True) \
            .drop(['tags'],axis=1) \
            .melt(id_vars=['id'], value_name='tag') \
            .drop('variable', axis=1) \
tag_ids = tag_explode.groupby(tag_explode.tag)['id'].unique().apply(list)
tag_ids.to_json('tag_ids.json')<jupyter_output><empty_output><jupyter_text>there really is no associated year<jupyter_code>subject_frame[~subject_frame.id.isin(id_years) &\
              subject_frame.named_subject.isna() &\
              ~subject_frame.Portraits].descriptor.value_counts().index
df[df.first_title.notna() & df.named_subject.notna()][['first_title', 'named_subject']]
df[df.first_title.notna() & df.named_subject.notna()].named_subject.value_counts()
df[df.first_title.notna() & df.named_subject.notna()].named_subject.count()
df[df.first_title.notna() & df.named_subject.isna()].first_title.value_counts().head(30).index.tolist()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17653254687170844, 'fraction_numerical': 0.007513517309177726, 'mean_word_length': 6.375453133091662, 'pattern_counts': {'":': 0, '<': 124, '<?xml version=': 0, '>': 134, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '29', 'dir': 'jupyter-structured-clean-dedup', 'id': '488826', 'license': 'MIT', 'n_tokens_mistral': 5330, 'n_tokens_neox': 5187, 'n_words': 838, 'path': 'notebooks/Cleaning  the Metadata.ipynb', 'repo_name': 'liabifano/humans-of-paris-1900', 'size': '14282'}