<commit_before>"""
MagPy
IAGA02 input filter
Written by <NAME> 2012
- contains test, read and write function
"""

from stream import *


def isNOAAACE(filename):
    """
    Checks whether a file is NOAA ACE format.
    """
    try:
        tempf = open(filename, 'rt')
        temp1= tempf.readline()
        temp2= tempf.readline()
        temp3= tempf.readline()
    except:
        return False
    if not temp1.startswith(':'):
        return False
    if not 'NOAA' in temp3:
        return False
    return True



def readNOAAACE(filename, headonly=False, **kwargs):
    """
    Reading NOAA Ace format data.
    """
    starttime = kwargs.get('starttime')
    endtime = kwargs.get('endtime')
    getfile = True

    fh = open(filename, 'rt')
    # read file and split text into channels
    stream = DataStream()
    # Check whether header infromation is already present
    if stream.header is None:
        headers = {}
    else:
        headers = stream.header
    data = []
    key = None

    # get day from filename (platform independent)
    splitpath = os.path.split(filename)
    try:
        splitname = splitpath[1].split('_')
        if len(splitname) == 4:
            tmpdaystring = splitpath[1].split('_')[0]
        elif len(splitname) == 5:
            tmpdaystring = splitpath[1].split('_')[1]
        elif len(splitname) == 6:
            tmpdaystring = splitpath[1].split('_')[2]
        daystring = re.findall(r'\d+',tmpdaystring)[0]
        datatype = splitpath[1].split('_')[2]
        day = datetime.strftime(datetime.strptime(daystring, "%Y%m%d"),"%Y-%m-%d")
        # Select only files within eventually defined time range
        if starttime:
            if not datetime.strptime(day,'%Y-%m-%d') >= datetime.strptime(datetime.strftime(stream._testtime(starttime),'%Y-%m-%d'),'%Y-%m-%d'):
                getfile = False
        if endtime:
            if not datetime.strptime(day,'%Y-%m-%d') <= datetime.strptime(datetime.strftime(stream._testtime(endtime),'%Y-%m-%d'),'%Y-%m-%d'):
                getfile = False
    except:
        if splitpath[0] == '/tmp':
            # internet file accessed
            pass 
        else:
            logging.warning("Could not identify typical NOAA date in %s. Reading all ..." % daystring)
        getfile = True
<commit_msg>Correction to NOAA data reading.<commit_after>"""
MagPy
IAGA02 input filter
Written by <NAME> June 2012
- contains test, read and write function
"""

from stream import *


def isNOAAACE(filename):
    """
    Checks whether a file is NOAA ACE format.
    """
    try:
        tempf = open(filename, 'rt')
        temp1= tempf.readline()
        temp2= tempf.readline()
        temp3= tempf.readline()
    except:
        return False
    if not temp1.startswith(':'):
        return False
    if not 'NOAA' in temp3:
        return False
    return True



def readNOAAACE(filename, headonly=False, **kwargs):
    """
    Reading NOAA Ace format data.
    """
    starttime = kwargs.get('starttime')
    endtime = kwargs.get('endtime')
    getfile = True

    fh = open(filename, 'rt')
    # read file and split text into channels
    stream = DataStream()
    # Check whether header infromation is already present
    if stream.header is None:
        headers = {}
    else:
        headers = stream.header
    data = []
    key = None

    # get day from filename (platform independent)
    splitpath = os.path.split(filename)
    try:
        splitname = splitpath[1].split('_')
	print splitname
	if len(splitname) == 3: # file is current file
	    tmpdaystring = datetime.strftime(datetime.utcnow(),'%Y%m%d')
            datatype = splitpath[1].split('_')[1]
        elif len(splitname) == 4:
            tmpdaystring = splitpath[1].split('_')[0]
            datatype = splitpath[1].split('_')[2]
        elif len(splitname) == 5:
            tmpdaystring = splitpath[1].split('_')[1]
            datatype = splitpath[1].split('_')[2]
        elif len(splitname) == 6:
            tmpdaystring = splitpath[1].split('_')[2]
            datatype = splitpath[1].split('_')[2]
        daystring = re.findall(r'\d+',tmpdaystring)[0]
        day = datetime.strftime(datetime.strptime(daystring, "%Y%m%d"),"%Y-%m-%d")
        # Select only files within eventually defined time range
        if starttime:
            if not datetime.strptime(day,'%Y-%m-%d') >= datetime.strptime(datetime.strftime(stream._testtime(starttime),'%Y-%m-%d'),'%Y-%m-%d'):
                getfile = False
        if endtime:
            if not datetime.strptime(day,'%Y-%m-%d') <= datetime.strptime(datetime.strftime(stream._testtime(endtime),'%Y-%m-%d'),'%Y-%m-%d'):
                getfile = False
    except:
        if splitpath[0] == '/tmp':
            # internet file accessed
            pass 
        else:
            logging.warning("Could not identify typical NOAA date in %s. Reading all ..." % daystring)
        getfile = True

{'fraction_non_alphanumeric': 0.13812715823684746, 'fraction_numerical': 0.01157830591102986, 'mean_word_length': 2.372602739726027, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6575784', 'n_tokens_mistral': 1572, 'n_tokens_neox': 1526, 'n_words': 380}