<commit_before>from esprit import raw
from portality import models
import re

'''
Use a scroll search to update mis-labelled licenses in the DOAJ.
Requires esprit (install from https://github.com/richard-jones/esprit)
'''

license_correct_dict = { "CC by" : "CC BY",
                         "CC by-nc" : "CC BY-NC",
                         "CC by-nc-nd" : "CC BY-NC-ND",
                         "CC by-nc-sa" : "CC BY-NC-SA",
                         "CC-BY-NC-SA" : "CC BY-NC-SA",
                         "CC by-sa" : "CC BY-SA",
                         "CC by-nd" : "CC BY-ND",
                         "not-cc-like" : "Not CC-like"
                        }

# Buld a regex to match any of our targets to change, e.g.:
# '\b(not-cc-like|CC by-nc-sa|CC by-nc-nd|CC-BY-NC-SA|CC by-nd|CC by-sa|CC by-nc|CC by)\b'
keys_by_len = sorted(license_correct_dict.keys(), key=len, reverse=True)
match_licenses = re.compile(r'(' + '|'.join(keys_by_len) + r')')

# Function to update the license in the json (as string)
def update_license_entry(old_string):
    return match_licenses.sub(lambda x: license_correct_dict[x.group()], old_string)

# Connection to the ES index
conn = raw.make_connection(None, 'localhost', 9200, 'doaj')

# Initialise the search and get our first batch of results
resp = raw.initialise_scroll(conn, type='journal', query=None, keepalive='1m')
journals = raw.unpack_result(resp)

write_batch = []
batch_size = 1000
while journals:

    # Process the previous set of journals
    for j in journals:
        try:
            journal_model = models.Journal(_source=j)
            # Change the license
            j_license = journal_model.bibjson().get_license()
            if j_license:
                print "\nold:\t{0}".format(journal_model.data['bibjson']['license'])
                journal_model.data['bibjson']['license'] = '[{0}]'.format(update_license_entry(str(j_license)))
                write_batch.append(journal_model.data)
                print "new:\t{0}".format(journal_model.data['bibjson']['license'])
        except ValueError:
            print "Failed to create a model"
        except KeyError:
            # No license present, pass
            pass

    # When we have enough, do some writing
    if len(write_batch) >= batch_size:
        print "writing ", len(write_batch)
        models.Journal.bulk(write_batch)
        write_batch = []

    # Read the next batch of journals by scrolling again
    try:
        sid = resp.json()['_scroll_id']
        resp = raw.scroll_next(conn, scroll_id=sid)
        journals = raw.unpack_result(resp)
    except ValueError:
        print "Reading JSON failed on: " + resp.text
        print "Aborting - cannae scroll further"
        break

# Write the last part-batch to index
if len(write_batch) > 0:
    print "writing ", len(write_batch)
    print models.Journal.bulk(write_batch)<commit_msg>Much simplified; now the changes occur but there are some records left unchanged. #242<commit_after>from esprit import tasks, raw
from portality import models

'''
Use a scroll search to update mis-labelled licenses in the DOAJ.
Requires esprit (install from https://github.com/richard-jones/esprit)
'''

license_correct_dict = { "CC by" : "CC BY",
                         "CC by-nc" : "CC BY-NC",
                         "CC by-nc-nd" : "CC BY-NC-ND",
                         "CC by-nc-sa" : "CC BY-NC-SA",
                         "CC-BY-NC-SA" : "CC BY-NC-SA",
                         "CC by-sa" : "CC BY-SA",
                         "CC by-nd" : "CC BY-ND",
                         "not-cc-like" : "Not CC-like"
                        }

# Connection to the ES index
conn = raw.make_connection(None, 'localhost', 9200, 'doaj')

write_batch = []
batch_size = 1000

# Process the previous set of journals
for j in tasks.scroll(conn, 'journal'):
    try:
        journal_model = models.Journal(_source=j)
        # Change the license
        j_license = journal_model.bibjson().get_license()
        if j_license:
            j_license['type'] = license_correct_dict[j_license['type']]
            j_license['title'] = license_correct_dict[j_license['title']]
            journal_model.prep()
            write_batch.append(journal_model.data)
    except ValueError:
        print "Failed to create a model"
    except KeyError:
        # No license present, pass
        pass

    # When we have enough, do some writing
    if len(write_batch) >= batch_size:
        print "writing ", len(write_batch)
        models.Journal.bulk(write_batch)
        write_batch = []

# Write the last part-batch to index
if len(write_batch) > 0:
    print "writing ", len(write_batch)
    print models.Journal.bulk(write_batch)
{'fraction_non_alphanumeric': 0.11835954101147471, 'fraction_numerical': 0.0053123671908202295, 'mean_word_length': 2.2529371112646857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '816045', 'n_tokens_mistral': 1480, 'n_tokens_neox': 1415, 'n_words': 387}