<commit_before>import json
from backend.app import app, db
from backend.models import *
from flask import url_for

# read in json redirect dump
with open('data/prod_url_alias.json', 'r') as f:

    redirects = json.loads(f.read())

print len(redirects)

old_urls = []
error_count = 0

for i in range(len(redirects)):
    nid = None
    try:
        nid = int(redirects[i]['nid'])
    except ValueError as e:
        tmp = redirects[i]['nid']
        if not 'user' in tmp:
            tmp = tmp.split('/')
            for item in tmp:
                try:
                    nid = int(item)
                    break
                except ValueError:
                    pass
        url = redirects[i]['url']
        if nid and not url in old_urls:
            redirect = Redirect(nid=nid, old_url=url)
            old_urls.append(url)
            db.session.add(redirect)
        else:
            error_count += 1
            print nid, redirects[i]['url']
    if i % 500 == 0:
        print "saving 500 redirects (" + str(i) + " out of " + str(len(redirects)) + ")"
        db.session.commit()
db.session.commit()

print "Error count:", str(error_count)<commit_msg>Bugfix. Not enough nids populated to DB.<commit_after>import json
from backend.app import app, db
from backend.models import *
from flask import url_for

# read in json redirect dump
with open('data/prod_url_alias.json', 'r') as f:

    redirects = json.loads(f.read())

print len(redirects)

old_urls = []

existing_redirects = Redirect.query.all()
for tmp in existing_redirects:
    old_urls.append(tmp.old_url)

error_count = 0
no_nid_count = 0
duplicate_count = 0

for i in range(len(redirects)):
    nid = None
    try:
        nid = int(redirects[i]['nid'])
    except ValueError as e:
        tmp = redirects[i]['nid']
        if not 'user' in tmp:
            tmp = tmp.split('/')
            for item in tmp:
                try:
                    nid = int(item)
                    break
                except ValueError:
                    pass
    url = redirects[i]['url']
    if not nid:
        no_nid_count += 1
    elif url in old_urls:
        duplicate_count += 1

    else:
        redirect = Redirect(nid=nid, old_url=url)
        old_urls.append(url)
        db.session.add(redirect)
        # print nid, redirects[i]['url'].encode('utf8')
    if i % 500 == 0:
        print "saving 500 redirects (" + str(i) + " out of " + str(len(redirects)) + ")"
        db.session.commit()
db.session.commit()

print duplicate_count, "duplicates found"
print no_nid_count, "nids could not be parsed"
{'fraction_non_alphanumeric': 0.11571207430340558, 'fraction_numerical': 0.008513931888544891, 'mean_word_length': 2.08472553699284, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3691973', 'n_tokens_mistral': 850, 'n_tokens_neox': 822, 'n_words': 229}