<commit_before>from collatex import *
from xml.dom import pulldom
import re
import glob
from datetime import datetime, date
# import pytz
# from tzlocal import get_localzone

# today = date.today()
# utc_dt = datetime(today, tzinfo=pytz.utc)
# dateTime = utc_dt.astimezone(get_localzone())
# strDateTime = str(dateTime)

now = datetime.utcnow()
nowStr = str(now)

regexWhitespace = re.compile(r'\s+')
regexNonWhitespace = re.compile(r'\S+')
regexEmptyTag = re.compile(r'/>$')
regexBlankLine = re.compile(r'\n{2,}')
regexLeadingBlankLine = re.compile(r'^\n')
regexPageBreak = re.compile(r'<pb.+?/>')
RE_MARKUP = re.compile(r'<.+?>')
RE_PARA = re.compile(r'<p.+?/>')
RE_MILESTONE = re.compile(r'<milestone.+?/>')
RE_AMP_NSB = re.compile(r'\S&amp;')
RE_AMP_NSE = re.compile(r'&amp;\S')
RE_AMP = re.compile(r'&')
RE_MDEL = re.compile(r'<mdel>.+?/</mdel>')
RE_SHI = re.compile(r'<shi.+?>.+?</shi>')
RE_METAMARK = re.compile(r'<metamark.+?>.+?</metamark>')
RE_HI = re.compile(r'<hi\s.+?/>')
RE_PB = re.compile(r'<pb.+?/>')
RE_LB = re.compile(r'<lb.+?/>')
RE_LG = re.compile(r'<lg.+?/>')
RE_L = re.compile(r'<l\s.+?/>')
RE_CIT = re.compile(r'<cit\s.+?/>')
RE_QUOTE = re.compile(r'<quote\s.+?/>')
RE_GAP = re.compile(r'<gap\s.+?/>')
# ebb: RE_MDEL = those pesky deletions of two letters or less that we want to normalize out of the collation, but preserve in the output.

# Element types: xml, div, head, p, hi, pb, note, lg, l; comment()
# Tags to ignore, with content to keep: xml, comment, anchor
# Structural elements: div, p, lg, l
# Inline elements (empty) retained in normalization: pb, milestone, xi:include
# Inline and block elements (with content) retained in normalization: note, hi, head, ab

# GIs fall into one three classes
# 2017-05-21 ebb: Due to trouble with pulldom parsing XML comments, I have converted these to comment elements,
# 2017-05-21 ebb: to be ignored during collation.
# 2017-05-30 ebb: Determined that comment elements cannot really be ignored when they have text nodes (the text is
# 2017-05-30 ebb: collated but the tags are not). Decision to make the comments into self-closing elements with text
# 2017-05-30 ebb: contents as attribute values, and content such as tags simplified to be legal attribute values.
# 2017-05-22 ebb: I've set anchor elements with @xml:ids to be the indicators of collation "chunks" to process together
ignore = ['sourceDoc', 'xml', 'comment', 'w', 'mod', 'anchor', 'include', 'delSpan', 'addSpan', 'add', 'handShift', 'damage', 'restore', 'zone', 'surface', 'graphic', 'unclear', 'retrace', 'damage', 'restore']
inlineEmpty = ['pb', 'lb', 'gap', 'del', 'p', 'div', 'milestone', 'lg', 'l', 'note', 'cit', 'quote', 'bibl', 'ab', 'hi', 'head']
# 2018-05-12 ebb: I'm setting a white space on either side of the inlineEmpty elements in line 76
# 2018-07-20: ebb: CHECK: are there white spaces on either side of empty elements in the output?
inlineContent = ['metamark', 'mdel', 'shi']
#2018-07-17 ebb: I moved the following list up into inlineEmpty, since they are all now empty elements: blockElement = ['lg', 'l', 'note', 'cit', 'quote', 'bibl']
# ebb: Tried removing 'comment', from blockElement list above, because we don't want these to be collated.

# 10-23-2017 ebb rv:

def normalizeSpace(inText):
    """Replaces all whitespace spans with single space characters"""
    if regexNonWhitespace.search(inText):
        return regexWhitespace.sub('\n', inText)
    else:
        return ''


def extract(input_xml):
    """Process entire input XML document, firing on events"""
    # Start pulling; it continues automatically
    doc = pulldom.parse(input_xml)
    output = ''
    for event, node in doc:
        # elements to ignore: xml
        if event == pulldom.START_ELEMENT and node.localName in ignore:
            continue
        # copy comments intact
        elif event == pulldom.COMMENT:
            doc.expandNode(node)
            output += node.toxml()
        # empty inline elements: pb, milestone, lb, lg, l, p, ab, head, hi
        elif event == pulldom.START_ELEMENT and node.localName in inlineEmpty:
            output += node.toxml()
        # non-empty inline elements: mdel, shi, metamark
        elif event == pulldom.START_ELEMENT and node.localName in inlineContent:
            output += regexEmptyTag.sub('>', node.toxml())
        elif event == pulldom.END_ELEMENT and node.localName in inlineContent:
            output += '</' + node.localName + '>'
        # elif event == pulldom.START_ELEMENT and node.localName in blockElement:
        #    output += '\n<' + node.localName + '>\n'
        #elif event == pulldom.END_ELEMENT and node.localName in blockElement:
        #    output += '\n</' + node.localName + '>'
        elif event == pulldom.CHARACTERS:
            output += normalizeSpace(node.data)
        else:
            continue
    return output


def normalize(inputText):
   return RE_AMP.sub('and',\
        RE_MDEL.sub('', \
        RE_SHI.sub('', \
        RE_HI.sub('', \
        RE_LB.sub('', \
        RE_PB.sub('', \
        RE_PARA.sub('<p/>', \
        RE_LB.sub('', \
        RE_LG.sub('<lg/>', \
        RE_L.sub('<l/>', \
        RE_CIT.sub('', \
        RE_QUOTE.sub('', \
        RE_GAP.sub('', \
        RE_METAMARK.sub('', inputText)))))))))))))).lower()
#    return regexPageBreak('',inputText)
# ebb: The normalize function makes it possible to return normalized tokens that screen out some markup, but not all.

def processToken(inputText):
    return {"t": inputText + ' ', "n": normalize(inputText)}


def processWitness(inputWitness, id):
    return {'id': id, 'tokens': [processToken(token) for token in inputWitness]}


for name in glob.glob('../collationChunks/1818_fullFlat_*'):
    try:
        matchString = name.split("fullFlat_", 1)[1]
        # ebb: above gets C30.xml for example
        matchStr = matchString.split(".", 1)[0]
        # ebb: above strips off the file extension
        with open(name, 'rb') as f1818file, \
                open('../collationChunks/Thomas_fullFlat_' + matchString, 'rb') as fThomasfile, \
                open('../collationChunks/1823_fullFlat_' + matchString, 'rb') as f1823file, \
                open('../collationChunks/1831_fullFlat_' + matchString, 'rb') as f1831file, \
                open('../collationChunks/msColl_' + matchString, 'rb') as fMSfile, \
                open('../Full_xmlOutput/collation_' + matchStr + '.xml', 'w') as outputFile:
                # open('collationChunks/msColl_c56_' + matchString, 'rb') as fMSc56file, \
                # open('collationChunks/msColl_c58_' + matchString, 'rb') as fMSc58file, \
                # open('collationChunks/msColl_c57Frag_' + matchString, 'rb') as fMSc57Fragfile, \<commit_msg>tweaking collation for normalizing SGA paragraph markup<commit_after>from collatex import *
from xml.dom import pulldom
import re
import glob
from datetime import datetime, date
# import pytz
# from tzlocal import get_localzone

# today = date.today()
# utc_dt = datetime(today, tzinfo=pytz.utc)
# dateTime = utc_dt.astimezone(get_localzone())
# strDateTime = str(dateTime)

now = datetime.utcnow()
nowStr = str(now)

regexWhitespace = re.compile(r'\s+')
regexNonWhitespace = re.compile(r'\S+')
regexEmptyTag = re.compile(r'/>$')
regexBlankLine = re.compile(r'\n{2,}')
regexLeadingBlankLine = re.compile(r'^\n')
regexPageBreak = re.compile(r'<pb.+?/>')
RE_MARKUP = re.compile(r'<.+?>')
RE_PARA = re.compile(r'<p.+?/>')
RE_MILESTONE = re.compile(r'<milestone.+?/>')
RE_AMP_NSB = re.compile(r'\S&amp;')
RE_AMP_NSE = re.compile(r'&amp;\S')
RE_AMP = re.compile(r'&')
RE_MDEL = re.compile(r'<mdel>.+?/</mdel>')
RE_SHI = re.compile(r'<shi.+?>.+?</shi>')
RE_METAMARK = re.compile(r'<metamark.+?>.+?</metamark>')
RE_HI = re.compile(r'<hi\s.+?/>')
RE_PB = re.compile(r'<pb.+?/>')
RE_LB = re.compile(r'<lb.+?/>')
RE_LG = re.compile(r'<lg.+?/>')
RE_L = re.compile(r'<l\s.+?/>')
RE_CIT = re.compile(r'<cit\s.+?/>')
RE_QUOTE = re.compile(r'<quote\s.+?/>')
RE_GAP = re.compile(r'<gap\s.+?/>')
# &lt;milestone unit="tei:p"/&gt;
RE_sgaP = re.compile(r'<milestone\s+unit="tei:p".+?/>')
# ebb: RE_MDEL = those pesky deletions of two letters or less that we want to normalize out of the collation, but preserve in the output.

# Element types: xml, div, head, p, hi, pb, note, lg, l; comment()
# Tags to ignore, with content to keep: xml, comment, anchor
# Structural elements: div, p, lg, l
# Inline elements (empty) retained in normalization: pb, milestone, xi:include
# Inline and block elements (with content) retained in normalization: note, hi, head, ab

# GIs fall into one three classes
# 2017-05-21 ebb: Due to trouble with pulldom parsing XML comments, I have converted these to comment elements,
# 2017-05-21 ebb: to be ignored during collation.
# 2017-05-30 ebb: Determined that comment elements cannot really be ignored when they have text nodes (the text is
# 2017-05-30 ebb: collated but the tags are not). Decision to make the comments into self-closing elements with text
# 2017-05-30 ebb: contents as attribute values, and content such as tags simplified to be legal attribute values.
# 2017-05-22 ebb: I've set anchor elements with @xml:ids to be the indicators of collation "chunks" to process together
ignore = ['sourceDoc', 'xml', 'comment', 'w', 'mod', 'anchor', 'include', 'delSpan', 'addSpan', 'add', 'handShift', 'damage', 'restore', 'zone', 'surface', 'graphic', 'unclear', 'retrace', 'damage', 'restore']
inlineEmpty = ['pb', 'lb', 'gap', 'del', 'p', 'div', 'milestone', 'lg', 'l', 'note', 'cit', 'quote', 'bibl', 'ab', 'hi', 'head']
# 2018-05-12 ebb: I'm setting a white space on either side of the inlineEmpty elements in line 76
# 2018-07-20: ebb: CHECK: are there white spaces on either side of empty elements in the output?
inlineContent = ['metamark', 'mdel', 'shi']
#2018-07-17 ebb: I moved the following list up into inlineEmpty, since they are all now empty elements: blockElement = ['lg', 'l', 'note', 'cit', 'quote', 'bibl']
# ebb: Tried removing 'comment', from blockElement list above, because we don't want these to be collated.

# 10-23-2017 ebb rv:

def normalizeSpace(inText):
    """Replaces all whitespace spans with single space characters"""
    if regexNonWhitespace.search(inText):
        return regexWhitespace.sub('\n', inText)
    else:
        return ''


def extract(input_xml):
    """Process entire input XML document, firing on events"""
    # Start pulling; it continues automatically
    doc = pulldom.parse(input_xml)
    output = ''
    for event, node in doc:
        # elements to ignore: xml
        if event == pulldom.START_ELEMENT and node.localName in ignore:
            continue
        # copy comments intact
        elif event == pulldom.COMMENT:
            doc.expandNode(node)
            output += node.toxml()
        # empty inline elements: pb, milestone, lb, lg, l, p, ab, head, hi
        elif event == pulldom.START_ELEMENT and node.localName in inlineEmpty:
            output += node.toxml()
        # non-empty inline elements: mdel, shi, metamark
        elif event == pulldom.START_ELEMENT and node.localName in inlineContent:
            output += regexEmptyTag.sub('>', node.toxml())
        elif event == pulldom.END_ELEMENT and node.localName in inlineContent:
            output += '</' + node.localName + '>'
        # elif event == pulldom.START_ELEMENT and node.localName in blockElement:
        #    output += '\n<' + node.localName + '>\n'
        #elif event == pulldom.END_ELEMENT and node.localName in blockElement:
        #    output += '\n</' + node.localName + '>'
        elif event == pulldom.CHARACTERS:
            output += normalizeSpace(node.data)
        else:
            continue
    return output


def normalize(inputText):
   return RE_AMP.sub('and',\
        RE_MDEL.sub('', \
        RE_SHI.sub('', \
        RE_HI.sub('', \
        RE_LB.sub('', \
        RE_PB.sub('', \
        RE_PARA.sub('<p/>', \
        RE_sgaP.sub('<p/>', \
        RE_LB.sub('', \
        RE_LG.sub('<lg/>', \
        RE_L.sub('<l/>', \
        RE_CIT.sub('', \
        RE_QUOTE.sub('', \
        RE_GAP.sub('', \
        RE_METAMARK.sub('', inputText))))))))))))))).lower()
#    return regexPageBreak('',inputText)
# ebb: The normalize function makes it possible to return normalized tokens that screen out some markup, but not all.

def processToken(inputText):
    return {"t": inputText + ' ', "n": normalize(inputText)}


def processWitness(inputWitness, id):
    return {'id': id, 'tokens': [processToken(token) for token in inputWitness]}


for name in glob.glob('../collationChunks/1818_fullFlat_*'):
    try:
        matchString = name.split("fullFlat_", 1)[1]
        # ebb: above gets C30.xml for example
        matchStr = matchString.split(".", 1)[0]
        # ebb: above strips off the file extension
        with open(name, 'rb') as f1818file, \
                open('../collationChunks/Thomas_fullFlat_' + matchString, 'rb') as fThomasfile, \
                open('../collationChunks/1823_fullFlat_' + matchString, 'rb') as f1823file, \
                open('../collationChunks/1831_fullFlat_' + matchString, 'rb') as f1831file, \
                open('../collationChunks/msColl_' + matchString, 'rb') as fMSfile, \
                open('../Full_xmlOutput/collation_' + matchStr + '.xml', 'w') as outputFile:
                # open('collationChunks/msColl_c56_' + matchString, 'rb') as fMSc56file, \
                # open('collationChunks/msColl_c58_' + matchString, 'rb') as fMSc58file, \
                # open('collationChunks/msColl_c57Frag_' + matchString, 'rb') as fMSc57Fragfile, \
{'fraction_non_alphanumeric': 0.1553612111413243, 'fraction_numerical': 0.018372896303373264, 'mean_word_length': 3.3268680445151033, 'pattern_counts': {'":': 4, '<': 53, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 28}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2646803', 'n_tokens_mistral': 4853, 'n_tokens_neox': 4513, 'n_words': 1291}