<commit_before>#!/usr/bin/python
# -*- coding:Utf-8 -*-

import os
import sys
from datetime import datetime
from json import loads

sys.path += [os.path.abspath(os.path.split(__file__)[0])[:-len("parltrack")] + "apps/"]

from votes.models import RecommendationData

if __name__ == "__main__":
    print "cleaning"
    RecommendationData.objects.all().delete()
    print "read file"
    lines = open("ep_votes.json").readlines()
    a = 1
    for vote in lines:
        jayson = loads(vote)
        #print jayson.get("report", None), jayson["title"], jayson["ts"]["$date"]
        RecommendationData.objects.create(proposal_name=jayson.get("report", jayson["title"]),
                                         title=jayson["title"],
                                         data=vote,
                                         date=datetime.fromtimestamp(int(str(jayson["ts"]["$date"])[:-3])))
        sys.stdout.write("%s/%s\r" % (a, len(lines)))
        sys.stdout.flush()
        a += 1
    sys.stdout.write("\n")

# vim:set shiftwidth=4 tabstop=4 expandtab:<commit_msg>[fix] import data form the new ep_votes.json data structure without eating all the system RAM<commit_after>#!/usr/bin/python
# -*- coding:Utf-8 -*-

import os
import sys
from dateutil.parser import parse
from json import loads, dumps

sys.path += [os.path.abspath(os.path.split(__file__)[0])[:-len("parltrack")] + "apps/"]

from votes.models import RecommendationData
from django.db import transaction

if __name__ == "__main__":
    print "cleaning"
    RecommendationData.objects.all().delete()
    print "read file"
    current_json = ""
    a = 1
    with transaction.commit_on_success():
        for i in open("ep_votes.json", "r"):
            if i in ("[{\n", "{\n"):
                #print "begin doc"
                current_json += "{\n"
            elif "}\n" == i:
                #print "end"
                current_json += "\n}"
                vote = loads(current_json)
                RecommendationData.objects.create(proposal_name=vote.get("report", vote["title"]),
                                                 title=vote["title"],
                                                 data=dumps(vote, indent=4),
                                                 date=parse(vote["ts"])),
                current_json = ""
                sys.stdout.write("%s\r" % a)
                sys.stdout.flush()
                a += 1
            elif i == ",\n":
                pass
            else:
                current_json += i
        sys.stdout.write("\n")

# vim:set shiftwidth=4 tabstop=4 expandtab:
{'fraction_non_alphanumeric': 0.14550366653801622, 'fraction_numerical': 0.005403319181783096, 'mean_word_length': 1.8577728776185225, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4203364', 'n_tokens_mistral': 770, 'n_tokens_neox': 741, 'n_words': 162}