<jupyter_start><jupyter_text>1. dataset name2. number of publications3. number of combined datasets4. top 5 datasets and their providers5. number of authors6. top 5 authors  test steps for one dataset<jupyter_code># build on existing function to generate metrics
uuid, title, rank, url, provider, publ_list = net.reco_data(net.data['dataset-ae01c2bf3451493f3620'])
def extract_institution(orcid,institutions_dict, institutions_count):
    hit = False

    # get orcid institution
    if len(net.auth[a].view["orcid"]) > 19:
        orcid = net.auth[a].view["orcid"][-19:]

    result = schol.orcid.affiliations(orcid)
    result2 = schol.orcid.funding(orcid)
    if result.meta:
        hit = True
        if isinstance(result.meta, list):
            institution = result.meta[0]['employment:organization']
        else:
            institution = result.meta['employment:organization']
        #print(orcid, institution['common:name'])

        if institution['common:name'] in institutions_dict.keys():
            institutions_count[institution['common:name']] += 1
        else:
            institutions_dict[institution['common:name']] = institution  # TODO potentially show countries?
            institutions_count[institution['common:name']] = 1
    if result2.meta:
        #print("TODO, look at schol.orcid.funding(orcid) for", orcid)
        funding.append(result2.meta)
    return hit
# load all KG known authors
graph = rc_graph.RCGraph()
graph.authors.load_entities("../rcgraph/authors.json") 

# load all Dimensions raw metadata from publications
dimension_pubs = {}
for partition, pub_iter in graph.iter_publications("../rcgraph/bucket_stage"):
    for pub in pub_iter:
        if "Dimensions" in pub:
            if "authors" in pub["Dimensions"]:
                dimension_pubs[pub["title"]] = pub["Dimensions"]["authors"]

def extract_institution_from_dimensions(pub,author_uuid):
    if author_uuid in graph.authors.known.uuid_map:
        if "dimensions" in graph.authors.known.uuid_map[author_uuid]:
            researcher_id = graph.authors.known.uuid_map[author_uuid]["dimensions"]
            if pub in dimension_pubs:
                for dimensions_meta in dimension_pubs[pub]:
                    if dimensions_meta["researcher_id"] == researcher_id:
                        try:
                            institution_name = dimensions_meta["affiliations"][0]["name"]

                            if institution_name in institutions_dict.keys():
                                institutions_count[institution_name] += 1
                            else:
                                institutions_dict[institution_name] = dimensions_meta["affiliations"][0]  # TODO potentially show countries?
                                institutions_count[institution_name] = 1
                        except:
                            pass
                        return True

    return False

funding = []


print(net.data['dataset-ae01c2bf3451493f3620'].view['title'])
print(len(publ_list))
joined_datasets = {}
joined_data_counts = {}
authors_dict = {}
authors_count = {}
institutions_dict = {}
institutions_count = {}
authorsWithInstitution_count = 0
schol = rc_scholapi.ScholInfraAPI(config_file="rc.cfg", logger=None)

for pubid, pub, pubrank in publ_list:
    d_list = net.publ[net.id_list[pubid]].view['datasets'].copy()
    d_list.remove(uuid) # don't include current dataset
    for d in d_list:
        if d in joined_datasets.keys():
            joined_data_counts[d] += 1
        else:
            joined_datasets[d] = net.data[d].view
            joined_data_counts[d] = 1
            joined_datasets[d]['ProvName'] = net.prov[joined_datasets[d]['provider']].view['title']
    a_list = net.publ[net.id_list[pubid]].view['authors'].copy()
    for a in a_list:
        if a in authors_dict.keys():
            authors_count[a] += 1
        else:
            authors_dict[a] = net.auth[a].view
            authors_count[a] = 1
            
            # for each dataset, count the institution only once per author. If the author wrote several publications, it will account for only one institution count.
            if net.auth[a].view["orcid"]:
                hit = extract_institution(net.auth[a].view["orcid"], institutions_dict, institutions_count)
                if hit:
                    authorsWithInstitution_count += 1
print(joined_datasets.keys())
joined_datasets['dataset-17fbd0c3d561e8260ab3']
# to get top 5 authors
sorted(authors_count.items(), key=lambda t: t[1], reverse=-True)[:5]
sorted(joined_data_counts.items(), key=lambda t: t[1], reverse=-True)[:5]
for data_id, data_count in sorted(joined_data_counts.items(), key=lambda t: t[1], reverse=-True)[:5]:
    DataName = joined_datasets[data_id]['title']
    DataProv = net.prov[joined_datasets[data_id]['provider']].view['title']
    print('Dataset {} by {} joined {} times'.format(DataName,DataProv,data_count))
for auth_id, auth_count in sorted(authors_count.items(), key=lambda t: t[1], reverse=-True)[:5]:
    AuthName = authors_dict[auth_id]['title']
    AuthORCID = authors_dict[auth_id]['orcid']
    print('{} | {} | used dataset {} times'.format(AuthName, AuthORCID, auth_count))<jupyter_output><empty_output><jupyter_text>work through specified list of datasetsmetrics to generate for each:- number of publications- number of combined datasets- top 5 datasets and their providers- number of authors- top 5 authors<jupyter_code>funding = []
schol = rc_scholapi.ScholInfraAPI(config_file="rc.cfg", logger=None) # to access ORCID

dataset_metrics = {} # object to hold resulting metrics calculated below

for this_data in data_list:
    # build on existing function to generate metrics
    uuid, title, rank, url, provider, publ_list = net.reco_data(net.data[this_data])
    print('collecting measures for {}'.format(title))
    num_pubs = len(publ_list)
    print('-- used in {} publications'.format(num_pubs))
    joined_datasets = {}
    joined_data_counts = {}
    authors_dict = {}
    authors_count = {}
    institutions_dict = {}
    institutions_count = {}
    authorsWithInstitution_count = 0

    for pubid, pub, pubrank in publ_list:
        # get datasets used in publication
        d_list = net.publ[net.id_list[pubid]].view['datasets'].copy()
        d_list.remove(uuid) # don't include current dataset
        for d in d_list:
            if d in joined_datasets.keys():
                joined_data_counts[d] += 1
            else:
                joined_datasets[d] = net.data[d].view
                joined_data_counts[d] = 1
                joined_datasets[d]['ProvName'] = net.prov[joined_datasets[d]['provider']].view['title']
        a_list = net.publ[net.id_list[pubid]].view['authors'].copy()
        for a in a_list:
            if a in authors_dict.keys():
                authors_count[a] += 1
            else:
                authors_dict[a] = net.auth[a].view
                authors_count[a] = 1
                
                # extract institution first trying with author's ORCID (new pull) and then with Dimensions (existing metadata in RCGraph)
                hit = False
                if net.auth[a].view["orcid"]:
                    hit = extract_institution(net.auth[a].view["orcid"], institutions_dict, institutions_count)
                else:
                    hit = extract_institution_from_dimensions(pub,a)
                
                if hit:
                    authorsWithInstitution_count += 1

    # add results to dataset_metrics dict:
    dataset_metrics[uuid] = {'combinedDatasets': joined_datasets, 'combinedDataCounts': joined_data_counts,
                            'PubAuthors': authors_dict, 'AuthorCounts': authors_count,
                            'TotalPublications': num_pubs, 'PubIDs': publ_list,
                             'AuthorInstitutions':institutions_dict, 'InstitutionCounts':institutions_count,
                             'AuthorsWithInstitutionCounts':authorsWithInstitution_count}
    
    ## print selected gathered info:
    # total datasets
    print('-- {} datasets were combined with this dataset. The top 5 are:'.format(len(joined_datasets.keys())))
    # top 5 datasets
    for data_id, data_count in sorted(joined_data_counts.items(), key=lambda t: t[1], reverse=-True)[:5]:
        DataName = joined_datasets[data_id]['title']
        DataProv = joined_datasets[data_id]['ProvName']
        print('---- {} by {} joined {} times'.format(DataName,DataProv,data_count))
    # total authors:
    print('-- {} authors used this dataset. The top 5 are:'.format(len(authors_dict.keys())))
    # top 5 authors:
    for auth_id, auth_count in sorted(authors_count.items(), key=lambda t: t[1], reverse=-True)[:5]:
        AuthName = authors_dict[auth_id]['title']
        AuthORCID = authors_dict[auth_id]['orcid']
        if AuthORCID=='':
            AuthORCID = 'unknown'
        print('---- {} | ORCID = {} | used dataset {} times'.format(AuthName, AuthORCID, auth_count))
    # total institutions:
    print('-- {} institutions used this dataset. The top 5 are:'.format(len(institutions_dict.keys())))
    # top 5 institutions:
    for institution_name , institution_count in sorted(institutions_count.items(), key=lambda t: t[1], reverse=-True)[:5]:
        InstitutionName = institution_name

        print('---- {} | used dataset {} times'.format(InstitutionName, institution_count))

    print('------ Note: of the {} authors, we were able to find institutions for {} of them'.format(
        len(authors_dict.keys()), authorsWithInstitution_count))
    print('') # add line between each datast summary


print("TODO: review funding found for ",len(funding),"authors")
dataset_metrics.keys()
# can access all results if need to report out differently, eg number of datasets:
len(dataset_metrics['dataset-955eb4bf66b73016354c']['combinedDatasets'].keys())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.117296423519903, 'fraction_numerical': 0.011214386744796929, 'mean_word_length': 2.9235037653587, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '399504', 'license': 'CC0-1.0', 'n_tokens_mistral': 2996, 'n_tokens_neox': 2899, 'n_words': 661, 'path': 'rc_metrics/RC_badge_draft.ipynb', 'repo_name': 'Coleridge-Initiative/rc-ml-corpus', 'size': '9898'}