<commit_before>"""
ISI flat file format parser.

run test(s) with
```
python -m isiparse path/to/data.isi
```
"""



class EmptyModule(): pass

try:
    import builtins
except ImportError:
    # hack around python2/3 (in)compatibility
    builtins = EmptyModule()
    import codecs
    builtins.open = codecs.open

import sys

from datetime import date
import time
from util import chomp

import logging

class ISIFormatError(Exception):
  pass


#class NumberedISIFormatError(ISIFormatError):
#	def __init__(self, line, *args):
#		self._line #^ this is meant to DRY up the line[%d] shit
#		super()    # but i don't want to think about it right now
	

def strpisimonth(d, fmt):
    """
    parse an ISI format "date" (PD field). This sometimes doesn't exist, sometimes includes a month, and sometimes includes a day.
  
    Known formats:
    Month ("%b")
    Month Day ("%b %d")
    Month-Month ("%b-%b") --- this gets coerced to the first %b, dropping the month range
    Season ("%s") --- this gets coerced to use the first month of the given season
    
    returns just the integer month, since we aren't going to use more resolution than that since the data isn't going to give it to us
    (this is a helper routine for parse_isi_month(); the reason it's not an inner function is poor: it's so I can instrument it with wrapper functions from external modules)
    """
    try:
        # handle the extension formats by a mixture of edits to the data and to fmt
        # then fall back on the standard strptime (whether or not we went through an extension format or not)
        if fmt == "%b-%b":
            # handle a month range e.g. "2006 APR-MAY"
            # which strptime can't handle
            l = d.find("-")
            if not (len(d) == 3 + 1 + 3): #four chars for the year, 3 for a month, 3 for another month
                raise ValueError("time data '%s' has wrong length for format %%b-%%b" % (d,))
            if not (d[3] == "-"): #four chars for the year, 3 for a month, 3 for another month
                raise ValueError("time data '%s' missing hyphen for format %%b-%%b" % (d,))
            end_month = time.strptime(d[l+1:], "%b") #this is a no-op, but it ensures that the format looks like we think it does
            d, fmt = d[:-4], "%b"
        elif fmt == "%s":
            # handle a season e.g. "FAL"
            # remap season to a month
            # this is a kludge, but at least it gives us *something*
            # each season gets three months out of twelve
            seasons = {'SPR': "MAR", 'SUM': "JUN", 'FAL': 'SEP', 'WIN': "DEC"}
            if d in seasons:
                d, fmt = seasons[d], "%b"
            else:
                raise ValueError("unknown ISI season '%s'" % (d,))
        
        return time.strptime(d, fmt).tm_mon
    except ValueError:
        # coerce any error to strptime()'s default message, to hide the real source
        raise ValueError("time data '%s' does not match format '%s'" % (d,fmt))

def parse_month(m):
    """
    """
    # try all the formats, starting with the most restrictive
    # strpisimonth requires an exact match so order shouldn't matter
    for fmt in ["%b %d", "%b", "%b-%b", "%s"]:
        try:
            return strpisimonth(m, fmt) 
        except ValueError:
            # silence the ValueErrors from a failed strpisimonth()
            # however if everything ValueError's out, the else clause will fire
            #print("error parsing '%s' with format '%s'" % (d, fmt))
            #traceback.print_exc() #DEBUG; warning: enabling this will be verrrry noisy
            continue
    else:
        raise ValueError("ISI time data '%s': format unrecognized" % (d,))
    
    assert False, "NOTREACHED"

def parse_year(year):
    # parse out the year
    if not isinstance(year, str):
        raise TypeError
    
    if len(year) != 4:
        raise ISIFormatError("Invalid ISI Year '%s'" % (year,))
    try:
        _year = int(year)
        if not (1900 <= int(_year) <= 3000):
            raise ValueError("year out of range")
        return _year
    except ValueError:
        raise ISIFormatError("Invalid ISI Year '%s'" % (year,))
        
        


def records(isi):
	"""
	read records from an open ISI-format file
	
  	TODO: make into a method on isireader()
	"""
	
	if isi.encoding.lower() != "utf-8-sig":
		logging.warn("%s opened in '%s' instead of BOM-compatible 'utf-8-sig'" % (isi, isi.encoding,))
	
	isi = (chomp(line) for line in isi) #(lazily) autostrip trailing newlines
	isi = ((i+1, line) for i, line in enumerate(isi)) #number isi lines, so we can give good debugging output
	def partition_lines(isi): #this is longer than it needs to be simply because I want to pass 'i' through it for the 'sep' error message
				   #it would be shorter written as a simple map function, without an internal loop
		for i, line in isi:
			#print(i,line) #DEBUG (handy: can see *every line* go past)
			#import IPython; IPython.embed()
			# partition the line
			# (note: EF, ER, and the blank line between records break the pattern, hence the verbose if chain)
			# ( also being stateful and ordering ops choosily means we can reuse 'line' without worrying stomping it before the 'sep' check)
			
			if len(line) > 2:
				sep = line[2]
				if sep != ' ':
					raise ISIFormatError("line[%d]: Malformed ISI line: '%r'" % (i,line))
			tag = line[:2]
			line = line[3:] #subtle: [3:] works even if the line is not that long, it just gives "". This is probably acceptable:
				# on pure-tag lines like "ER" line will end up as "" as expected, and on the blank line, so will tag
			
			yield i, (tag, line)
	
	isi = partition_lines(isi)
	<commit_msg>Handle more observed PD field formats.<commit_after>"""
ISI flat file format parser.

run test(s) with
```
python -m isiparse path/to/data.isi
```
"""

# TODO:
# [ ] rework strpisimonth to be strpisidate, returning a tuple instead of an integer;
#     I *thought* the PD field was only ever month and maybe day, but sometimes it duplicates the year (PY) field too
#     Once this works, write an assertion "not ('PY' in fields and 'PD' in fields and fields['PD'].year is not None) or (fields['PY'] == fields['PD'].year)

class EmptyModule(): pass

try:
    import builtins
except ImportError:
    # hack around python2/3 (in)compatibility
    builtins = EmptyModule()
    import codecs
    builtins.open = codecs.open

import sys

from datetime import date
import time
from util import chomp

import logging

class ISIFormatError(Exception):
  pass


#class NumberedISIFormatError(ISIFormatError):
#	def __init__(self, line, *args):
#		self._line #^ this is meant to DRY up the line[%d] shit
#		super()    # but i don't want to think about it right now
	

def strpisimonth(d, fmt):
    """
    parse an ISI format "date" (PD field). This sometimes doesn't exist, sometimes includes a month, and sometimes includes a day, and sometimes includes a year.
  
    Known formats:
    Month ("%b")
    Month Day ("%b %d")
    Month-Month ("%b-%b") --- this gets coerced to the first %b, dropping the month range
    Season ("%s") --- this gets coerced to use the first month of the given season
    Month Day Year ("%b %d %Y")
    Month Year ("%b %Y")
    
    
    returns a tuple (year, month, day). This is like a datetime object, except that any of the three may be None to indicate missing data.
    """
    try:
        # handle the extension formats by a mixture of edits to the data and to fmt
        # then fall back on the standard strptime (whether or not we went through an extension format or not)
        if fmt == "%b-%b":
            # handle a month range e.g. "2006 APR-MAY"
            # which strptime can't handle
            l = d.find("-")
            if not (len(d) == 3 + 1 + 3): #four chars for the year, 3 for a month, 3 for another month
                raise ValueError("time data '%s' has wrong length for format %%b-%%b" % (d,))
            if not (d[3] == "-"): #four chars for the year, 3 for a month, 3 for another month
                raise ValueError("time data '%s' missing hyphen for format %%b-%%b" % (d,))
            end_month = time.strptime(d[l+1:], "%b") #this is a no-op, but it ensures that the format looks like we think it does
            d, fmt = d[:-4], "%b"
        elif fmt == "%s":
            # handle a season e.g. "FAL"
            # remap season to a month
            # this is a kludge, but at least it gives us *something*
            # each season gets three months out of twelve
            seasons = {'SPR': "MAR", 'SUM': "JUN", 'FAL': 'SEP', 'WIN': "DEC"}
            if d in seasons:
                d, fmt = seasons[d], "%b"
            else:
                raise ValueError("unknown ISI season '%s'" % (d,))
        
        return time.strptime(d, fmt).tm_mon
    except ValueError:
        # coerce any error to strptime()'s default message, to hide the real source
        raise ValueError("time data '%s' does not match format '%s'" % (d,fmt))

def parse_month(m):
    """
    """
    # try all the formats, starting with the most restrictive
    # strpisimonth requires an exact match so order shouldn't matter
    for fmt in ["%b %d", "%b", "%b-%b", "%s", "%b %d %Y", "%b %Y"]:
        try:
            return strpisimonth(m, fmt) 
        except ValueError:
            # silence the ValueErrors from a failed strpisimonth()
            # however if everything ValueError's out, the else clause will fire
            #print("error parsing '%s' with format '%s'" % (d, fmt))
            #traceback.print_exc() #DEBUG; warning: enabling this will be verrrry noisy
            continue
    else:
        raise ValueError("ISI time data '%s': format unrecognized" % (d,))
    
    assert False, "NOTREACHED"

def parse_year(year):
    # parse out the year
    if not isinstance(year, str):
        raise TypeError
    
    if len(year) != 4:
        raise ISIFormatError("Invalid ISI Year '%s'" % (year,))
    try:
        _year = int(year)
        if not (1900 <= int(_year) <= 3000):
            raise ValueError("year out of range")
        return _year
    except ValueError:
        raise ISIFormatError("Invalid ISI Year '%s'" % (year,))
        
        


def records(isi):
	"""
	read records from an open ISI-format file
	
  	TODO: make into a method on isireader()
	"""
	
	if isi.encoding.lower() != "utf-8-sig":
		logging.warn("%s opened in '%s' instead of BOM-compatible 'utf-8-sig'" % (isi, isi.encoding,))
	
	isi = (chomp(line) for line in isi) #(lazily) autostrip trailing newlines
	isi = ((i+1, line) for i, line in enumerate(isi)) #number isi lines, so we can give good debugging output
	def partition_lines(isi): #this is longer than it needs to be simply because I want to pass 'i' through it for the 'sep' error message
				   #it would be shorter written as a simple map function, without an internal loop
		for i, line in isi:
			#print(i,line) #DEBUG (handy: can see *every line* go past)
			#import IPython; IPython.embed()
			
			# partition the line
			# (note: EF, ER, and the blank line between records break the pattern, hence the verbose if-chain)
			# ( also being stateful and ordering ops choosily means we can reuse 'line' without worrying stomping it before the 'sep' check)
			if len(line) > 2:
				sep = line[2]
				if sep != ' ':
					raise ISIFormatError("line[%d]: Malformed ISI line: '%r'" % (i,line))
			tag = line[:2]
			line = line[3:] #subtle: [3:] works even if the line is not that long, it just gives "". This is probably acceptable:
				# on pure-tag lines like "ER" line will end up as "" as expected, and on the blank line, so will tag
			
			yield i, (tag, line)
	
	isi = partition_lines(isi)
	
{'fraction_non_alphanumeric': 0.11283147989734817, 'fraction_numerical': 0.005645851154833191, 'mean_word_length': 2.6105620753551575, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2673012', 'n_tokens_mistral': 3715, 'n_tokens_neox': 3520, 'n_words': 1526}