<commit_before>#!/usr/bin/python
# coding=utf-8

'''
author: <NAME>
Last Update: 19-October-2014

'''

from __future__ import division

import os
import argparse
import nltk
import re
from lxml import etree
from tempfile import NamedTemporaryFile
from diffMatcher import DiffMatcher

description = "Merges two CorA XML files based on their modern tokenization."
epilog = "The output is a CorA XML file identical to TRANS_XML, but enriched with annotations from the parallel file ANNO_XML."
parser = argparse.ArgumentParser(description=description, epilog=epilog)
parser.add_argument('fileA',
                    metavar='TRANS_XML',
                    type=argparse.FileType('r'),
                    help='XML file with the definitive transcription')
parser.add_argument('fileB',
                    metavar='ANNO_XML',
                    type=argparse.FileType('r'),
                    help='XML file with the definitive annotation')
parser.add_argument('-i', '--ignore',
                    action='store_true',
                    help='If on, then script ignores case-sensitivity')
parser.add_argument('-l', '--log',
                    dest='logfile',
                    metavar='LOGFILE',
                    type=argparse.FileType('w'),
                    help='Write logfile of changes to LOGFILE')

args = parser.parse_args()

log_file = args.logfile
diff_list = []
complete_list = {}
seen = {}

def print_utf8(handle, text):
    try:
        print >> handle, text.encode("utf-8")
    except UnicodeError:
        print >> handle, text

#function reads out ascii data from given file and write them in an output file        
def read_out(root,file_out):
    for mod in root.iter('mod'):
        print_utf8(file_out, mod.attrib['ascii'])

def change_data(number,column, create_log):
    word = diff_list[number][column]
    look = 'token/mod[@ascii="'+word+'"]'
    result_list = []
    #if word is more than one time in file
    if word in seen:
        seen[word]+=1
        #create list for every match with the searched word
        result_list = complete_list[word]
        #take right word
        actual_word = result_list[seen[word]]
        id_b = actual_word.get("id")
        #take sub elements and put them in file A
        if create_log == 'no':
            for match in actual_word.iterfind(".//"):<commit_msg>Add error handler<commit_after>#!/usr/bin/python
# coding=utf-8

'''
author: <NAME>
Last Update: 19-October-2014

'''

from __future__ import division

import os
import argparse
import nltk
import re
from lxml import etree
from tempfile import NamedTemporaryFile
from diffMatcher import DiffMatcher

description = "Merges two CorA XML files based on their modern tokenization."
epilog = "The output is a CorA XML file identical to TRANS_XML, but enriched with annotations from the parallel file ANNO_XML."
parser = argparse.ArgumentParser(description=description, epilog=epilog)
parser.add_argument('fileA',
                    metavar='TRANS_XML',
                    type=argparse.FileType('r'),
                    help='XML file with the definitive transcription')
parser.add_argument('fileB',
                    metavar='ANNO_XML',
                    type=argparse.FileType('r'),
                    help='XML file with the definitive annotation')
parser.add_argument('-i', '--ignore',
                    action='store_true',
                    help='If on, then script ignores case-sensitivity')
parser.add_argument('-l', '--log',
                    dest='logfile',
                    metavar='LOGFILE',
                    type=argparse.FileType('w'),
                    help='Write logfile of changes to LOGFILE')

args = parser.parse_args()

log_file = args.logfile
diff_list = []
complete_list = {}
seen = {}

def print_utf8(handle, text):
    try:
        print >> handle, text.encode("utf-8")
    except UnicodeError:
        print >> handle, text

#function reads out ascii data from given file and write them in an output file
def read_out(root,file_out):
    try:
        for mod in root.iter('mod'):
            print_utf8(file_out, mod.attrib['ascii'])
    except KeyError:
        print("KeyError: mod '{}' has no 'ascii' attribute".format(mod.attrib['id']))
        raise

def change_data(number,column, create_log):
    word = diff_list[number][column]
    look = 'token/mod[@ascii="'+word+'"]'
    result_list = []
    #if word is more than one time in file
    if word in seen:
        seen[word]+=1
        #create list for every match with the searched word
        result_list = complete_list[word]
        #take right word
        actual_word = result_list[seen[word]]
        id_b = actual_word.get("id")
        #take sub elements and put them in file A
        if create_log == 'no':
            for match in actual_word.iterfind(".//"):
{'fraction_non_alphanumeric': 0.10660847880299251, 'fraction_numerical': 0.00457190357439734, 'mean_word_length': 2.5971599402092678, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6773317', 'n_tokens_mistral': 1419, 'n_tokens_neox': 1357, 'n_words': 413}