<commit_before># -*- coding: utf-8 -*-

'''Helper functions for importing the nzdc database from the old server.'''

'''This file may be safely removed once all data pre-processing is completed'''

import subprocess
import datetime as dt

from DictionaryOfNewZealandEnglish.user.models import *
from DictionaryOfNewZealandEnglish.headword.models import *
from DictionaryOfNewZealandEnglish.headword.attribute.models import *
from DictionaryOfNewZealandEnglish.headword.citation.models import *





# Processed data was missing various Notes found on the db
# this code should add those notes into the text files
# (which can then be uploaded using existing code).
@manager.command
def addNotesToDb(myfile):
    #print "here"
    head_loc = "./addDbNotes/Headwords_working_copy.txt" # tab delimited file from db
    file_loc = "./addDbNotes/db files Aug 2015/nzdc_export_"
    file_type = ".txt"
    action = myfile.split(":")

    myfile = action[0]
    print ""
    print "##################################################################"
    if myfile == "all":
      for myfile in list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
        print "#### Importing data from file: %s ##" % (file_loc + myfile + file_type)
        writeNotesTotextFile(file_loc, myfile, file_type, head_loc)
    else: 
      print "#### Importing data from file: %s ##" % (file_loc + myfile + file_type)
      writeNotesTotextFile(file_loc, myfile, file_type, head_loc)

    print "#### Copied Notes to %s.txt ..." % myfile
<commit_msg>Fix error in new file<commit_after># -*- coding: utf-8 -*-

'''Helper functions for importing the nzdc database from the old server.'''

'''This file may be safely removed once all data pre-processing is completed'''

import subprocess
import datetime as dt

from DictionaryOfNewZealandEnglish.user.models import *
from DictionaryOfNewZealandEnglish.headword.models import *
from DictionaryOfNewZealandEnglish.headword.attribute.models import *
from DictionaryOfNewZealandEnglish.headword.citation.models import *





# Processed data was missing various Notes found on the db
# this code should add those notes into the text files
# (which can then be uploaded using existing code).
def addNotesToDb(myfile):
    #print "here"
    head_loc = "./addDbNotes/Headwords_working_copy.txt" # tab delimited file from db
    file_loc = "./addDbNotes/db files Aug 2015/nzdc_export_"
    file_type = ".txt"
    action = myfile.split(":")

    myfile = action[0]
    print ""
    print "##################################################################"
    if myfile == "all":
      for myfile in list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
        print "#### Importing data from file: %s ##" % (file_loc + myfile + file_type)
        writeNotesTotextFile(file_loc, myfile, file_type, head_loc)
    else: 
      print "#### Importing data from file: %s ##" % (file_loc + myfile + file_type)
      writeNotesTotextFile(file_loc, myfile, file_type, head_loc)

    print "#### Copied Notes to %s.txt ..." % myfile

{'fraction_non_alphanumeric': 0.1431901431901432, 'fraction_numerical': 0.003996003996003996, 'mean_word_length': 4.1350427350427355, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1882156', 'n_tokens_mistral': 926, 'n_tokens_neox': 871, 'n_words': 277}