<commit_before>import tempfile
from path import path
import flask
import requests
import database


def refresh_json():
    app = flask.current_app
    instance_path = path(app.instance_path)
    json_path = path(app.instance_path)/'hartapoliticii-people.json'
    with tempfile.NamedTemporaryFile(dir=instance_path, delete=False) as t:
        r = requests.get("http://hartapoliticii.ro/"
                         "api/search.php?api_key=%s"
                         % app.config['HPOL_API_KEY'])
        for chunk in r.iter_content(65536):
            t.write(chunk)
    path(t.name).rename(json_path)


def update_identities():
    app = flask.current_app
    json_path = path(app.instance_path)/'hartapoliticii-people.json'
    if not json_path.isfile():
        refresh_json()

    hpol_map = {}
    with json_path.open() as f:
        for hpol_person in flask.json.load(f):
            hpol_map[hpol_person['id']] = hpol_person['name']

    for person in database.Person.objects_current().all():
        hpol_id = person.get_meta('hpol_id')
        print "ok:", hpol_id, person.id
        hpol_map.pop(hpol_id, None)

    for hpol_id, hpol_name in hpol_map.iteritems():
        print hpol_id, hpol_name
        person = database.Person(name=hpol_name)
        database.db.session.add(person)
        meta = database.PersonMeta(person=person, key='hpol_id', value=hpol_id)
        database.db.session.add(meta)

    database.db.session.commit()<commit_msg>less noise in logging; print date; quite strings<commit_after>import tempfile
from path import path
import flask
import requests
import database
from datetime import datetime


def refresh_json():
    app = flask.current_app
    instance_path = path(app.instance_path)
    json_path = path(app.instance_path)/'hartapoliticii-people.json'
    with tempfile.NamedTemporaryFile(dir=instance_path, delete=False) as t:
        r = requests.get("http://hartapoliticii.ro/"
                         "api/search.php?api_key=%s"
                         % app.config['HPOL_API_KEY'])
        for chunk in r.iter_content(65536):
            t.write(chunk)
    path(t.name).rename(json_path)


def update_identities():
    app = flask.current_app
    json_path = path(app.instance_path)/'hartapoliticii-people.json'
    if not json_path.isfile():
        refresh_json()

    now = datetime.utcnow().strftime('%Y-%m-%d')

    hpol_map = {}
    with json_path.open() as f:
        for hpol_person in flask.json.load(f):
            hpol_map[hpol_person['id']] = hpol_person['name']

    for person in database.Person.objects_current().all():
        hpol_id = person.get_meta('hpol_id')
        #print "ok:", hpol_id, person.id
        hpol_map.pop(hpol_id, None)

    for hpol_id, hpol_name in hpol_map.iteritems():
        print "importing: %r %r %s" % (hpol_id, hpol_name, now)
        person = database.Person(name=hpol_name)
        database.db.session.add(person)
        meta = database.PersonMeta(person=person, key='hpol_id', value=hpol_id)
        database.db.session.add(meta)

    database.db.session.commit()
{'fraction_non_alphanumeric': 0.11893124796350603, 'fraction_numerical': 0.0032583903551645487, 'mean_word_length': 2.9561855670103094, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3323993', 'n_tokens_mistral': 1044, 'n_tokens_neox': 1007, 'n_words': 187}