<commit_before>from time import time
from app import db
from util import elapsed
import argparse

from models.person import add_or_overwrite_person_from_orcid_id
from models.orcid import clean_orcid
from models.orcid import NoOrcidException

# needs to be imported so the definitions get loaded into the registry
import job_definitions


"""
Call from command line to add ORCID profiles based on IDs in a local CSV.

"""


def load_campaign(filename, campaign_name, limit=None):

    with open("data/" + filename, "r") as f:
        lines = f.read().split("\n")
        print "found {} ORCID lines".format(len(lines))

    if limit:
        lines = lines[:limit]

    for line in lines:
        start = time()

        if "," in line:
            dirty_orcid, campaign_email = line.split(",")
        else:
            dirty_orcid = line
            campaign_email = None

        try:
            orcid_id = clean_orcid(dirty_orcid)
        except NoOrcidException:
            print u"\n\nWARNING: no valid orcid_id in line {}; skipping\n\n".format(line)
            continue

        add_or_overwrite_person_from_orcid_id(orcid_id, campaign_email, campaign_name, high_priority=False)

    print "loaded {} profiles in {}s\n".format(len(lines), elapsed(start))





if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Run stuff.")<commit_msg>handle unicode error in script debug<commit_after>from time import time
from app import db
from util import elapsed
import argparse

from models.person import add_or_overwrite_person_from_orcid_id
from models.orcid import clean_orcid
from models.orcid import NoOrcidException

# needs to be imported so the definitions get loaded into the registry
import job_definitions


"""
Call from command line to add ORCID profiles based on IDs in a local CSV.

"""


def load_campaign(filename, campaign_name, limit=None):

    with open("data/" + filename, "r") as f:
        lines = f.read().split("\n")
        print "found {} ORCID lines".format(len(lines))

    if limit:
        lines = lines[:limit]

    for line in lines:
        start = time()

        if "," in line:
            dirty_orcid, campaign_email = line.split(",")
        else:
            dirty_orcid = line
            campaign_email = None

        try:
            orcid_id = clean_orcid(dirty_orcid)
        except NoOrcidException:
            try:
                print u"\n\nWARNING: no valid orcid_id in line {}; skipping\n\n".format(line)
            except TypeError:
                print u"\n\nWARNING: no valid orcid_id; skipping\n\n"
            continue

        add_or_overwrite_person_from_orcid_id(orcid_id, campaign_email, campaign_name, high_priority=False)

    print "loaded {} profiles in {}s\n".format(len(lines), elapsed(start))





if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Run stuff.")
{'fraction_non_alphanumeric': 0.09021512838306732, 'fraction_numerical': 0.0, 'mean_word_length': 2.88021534320323, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '181887', 'n_tokens_mistral': 871, 'n_tokens_neox': 835, 'n_words': 257}