<commit_before># -*- coding: utf-8 -*-

'''Helper functions for importing the nzdc database from the old server.'''

import os
import subprocess
import datetime as dt

from DictionaryOfNewZealandEnglish.user.models import *
from DictionaryOfNewZealandEnglish.headword.models import *
from DictionaryOfNewZealandEnglish.headword.attribute.models import *
from DictionaryOfNewZealandEnglish.headword.citation.models import *



### Part 1. Pre-processing ### 
# Convert rtf files into txt files to strip away formating info.
# Cut n paste contents into gedit. Check last listing matches.
# run python manage.py data_import A
# will process text file A.txt and print to terminal
# run python manage.py data_import all
# will process all 26 text files and print to terminal
### Part 2. Load db ###
# inserts "create entries" commands into the script
# run python manage.py data_import A:db
# will process text file A.txt, print to terminal and load data to db
def nzdc_data_import(myfile):
    file_loc = "./db files June 2015/nzdc_export_"
    file_type = ".txt"
    load_db = False
    action = myfile.split(":")
    try:
      if action[1] == "db":
        load_db = True
        # prep database
        print "#### removing old files"
        subprocess.call(["rm", "dev.db"])
        subprocess.call(["rm", "-r", "migrations"])
        print "#### init db"
        subprocess.call(["python", "manage.py", "db", "init"])
        print "#### migrate db"
        subprocess.call(["python", "manage.py", "db", "migrate"])
        print "#### upgrade db"
        subprocess.call(["python", "manage.py", "db", "upgrade"])
        
        # load first users
        initial_db_users()
        # load secondary tables
        seed_tables()
    except:
      # do nothing, try fails if action[1] does not exist, so load_db stays False
      print "Not changing database."

    myfile = action[0]
    print ""
    print "##################################################################"
    if myfile == "all":
      for myfile in list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
        print "#### Importing data from file: %s ##" % (file_loc + myfile + file_type)
        data_import_read_files(file_loc, myfile, file_type, load_db)
    else: 
      print "#### Importing data from file: %s ##" % (file_loc + myfile + file_type)
      data_import_read_files(file_loc, myfile, file_type, load_db)

    if load_db:
      print "#### Loaded database ...???"

def data_import_read_files(file_loc, myfile, file_type, load_db):
    # Files are sets of word data including 0+ Citations
    # Word data starts with a headword and finishes with a blank line after all citations.
    # Citations data starts with "Citations"
    # A Citation starts with "Date: 18/04/2001" and finishes with "Notes: stuff"
    # next citation starts with blank line followed by "Date:"
    # next headword starts with blank line not followed by "Date:"
    with open(file_loc + myfile + file_type) as text:
      currentLine = nextLine = ""
      i = 1
      for line in text:
        currentLine = nextLine
        nextLine = line.strip()
        if currentLine == "":
          continue

        # this is a headword
        currentLine, nextLine, headword, headword_obj = create_headword(currentLine, nextLine, text, i)

        # this is zero or more citations for the headword
        nextLine, headword = create_citation_list(currentLine, 
                                                  nextLine, 
                                                  text, 
                                                  headword,
                                                  headword_obj)
        if nextLine == None:
          break
        i = i + 1
        # limits output for testing
#        print "### " + str(i)<commit_msg>Changed loop counter to be more pythonic with enumerate<commit_after># -*- coding: utf-8 -*-

'''Helper functions for importing the nzdc database from the old server.'''

import subprocess
import datetime as dt

from DictionaryOfNewZealandEnglish.user.models import *
from DictionaryOfNewZealandEnglish.headword.models import *
from DictionaryOfNewZealandEnglish.headword.attribute.models import *
from DictionaryOfNewZealandEnglish.headword.citation.models import *



### Part 1. Pre-processing ### 
# Convert rtf files into txt files to strip away formating info.
# Cut n paste contents into gedit. Check last listing matches.
# run python manage.py data_import A
# will process text file A.txt and print to terminal
# run python manage.py data_import all
# will process all 26 text files and print to terminal
### Part 2. Load db ###
# inserts "create entries" commands into the script
# run python manage.py data_import A:db
# will process text file A.txt, print to terminal and load data to db
def nzdc_data_import(myfile):
    file_loc = "./db files June 2015/nzdc_export_"
    file_type = ".txt"
    load_db = False
    action = myfile.split(":")
    try:
      if action[1] == "db":
        load_db = True
        # prep database
        print "#### removing old files"
        subprocess.call(["rm", "dev.db"])
        subprocess.call(["rm", "-r", "migrations"])
        print "#### init db"
        subprocess.call(["python", "manage.py", "db", "init"])
        print "#### migrate db"
        subprocess.call(["python", "manage.py", "db", "migrate"])
        print "#### upgrade db"
        subprocess.call(["python", "manage.py", "db", "upgrade"])
        
        # load first users
        initial_db_users()
        # load secondary tables
        seed_tables()
    except:
      # do nothing, try fails if action[1] does not exist, so load_db stays False
      print "Not changing database."

    myfile = action[0]
    print ""
    print "##################################################################"
    if myfile == "all":
      for myfile in list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
        print "#### Importing data from file: %s ##" % (file_loc + myfile + file_type)
        data_import_read_files(file_loc, myfile, file_type, load_db)
    else: 
      print "#### Importing data from file: %s ##" % (file_loc + myfile + file_type)
      data_import_read_files(file_loc, myfile, file_type, load_db)

    if load_db:
      print "#### Loaded database ...???"

def data_import_read_files(file_loc, myfile, file_type, load_db):
    # Files are sets of word data including 0+ Citations
    # Word data starts with a headword and finishes with a blank line after all citations.
    # Citations data starts with "Citations"
    # A Citation starts with "Date: 18/04/2001" and finishes with "Notes: stuff"
    # next citation starts with blank line followed by "Date:"
    # next headword starts with blank line not followed by "Date:"
    with open(file_loc + myfile + file_type) as text:
      currentLine = nextLine = ""
      
      for i, line in enumerate(text):
        currentLine = nextLine
        nextLine = line.strip()
        if currentLine == "":
          continue

        # this is a headword
        currentLine, nextLine, headword, headword_obj = create_headword(currentLine, nextLine, text, i)

        # this is zero or more citations for the headword
        nextLine, headword = create_citation_list(currentLine, 
                                                  nextLine, 
                                                  text, 
                                                  headword,
                                                  headword_obj)
        if nextLine == None:
          break
        # limits output for testing
#        print "### " + str(i)
{'fraction_non_alphanumeric': 0.11507570770243582, 'fraction_numerical': 0.005793285055957867, 'mean_word_length': 2.4892053284336244, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3394277', 'n_tokens_mistral': 2151, 'n_tokens_neox': 2062, 'n_words': 725}