<commit_before>"""
MagPy
Intermagnet ImagCDF input filter
(based on cdflib)
Written by <NAME> October 2019
- contains test, read and write functions for
        ImagCDF
- supports python >= 3.5
- currently requires cdflib<=0.3.18
"""

from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from io import open

from magpy.stream import *

import cdflib

#import ciso8601 ## fast datetime parser  ciso8601.parse_datetime should be 10 times faster than datetime.strftime

import sys
import logging
logger = logging.getLogger(__name__)

HEADTRANSLATE = {'FormatDescription':'DataFormat', 'IagaCode':'StationID', 'ElementsRecorded':'DataComponents', 'ObservatoryName':'StationName', 'Latitude':'DataAcquisitionLatitude', 'Longitude':'DataAcquisitionLongitude', 'Institution':'StationInstitution', 'VectorSensOrient':'DataSensorOrientation', 'TermsOfUse':'DataTerms','UniqueIdentifier':'DataID','ParentIdentifiers':'SensorID','ReferenceLinks':'StationWebInfo', 'FlagRulesetType':'FlagRulesetType','FlagRulesetVersion':'FlagRulesetVersion'}


def isIMAGCDF(filename):
    """
    Checks whether a file is ImagCDF format
    """
    try:
        temp = cdflib.CDF(filename)
    except:
        return False
    try:
        form = temp.globalattsget().get('FormatDescription')
        if not form.startswith('INTERMAGNET'):
            return False
    except:
        return False

    print ("Running new ImagCDF import filter")

    logger.debug("isIMAGCDF: Found INTERMAGNET CDF data - using cdflib")
    return True

def readIMAGCDF(filename, headonly=False, **kwargs):
    """
    Reading Intermagnet CDF format (1.0,1.1,1.2)
    """

    debug = kwargs.get('debug')

    headers={}
    arraylist = []
    array = [[] for elem in KEYLIST]
    multipletimedict = {}
    newdatalist = []
    tllist = []
    indexarray = np.asarray([])

    cdfdat = cdflib.CDF(filename)

    if debug:
        print ("Reading ImagCDF with cdflib")

    for att in cdfdat.globalattsget():
        value = cdfdat.globalattsget().get(att)
        try:
            if isinstance(list(value), list):
                if len(value) == 1:
                    value = value[0]
        except:
            pass
        if not att in HEADTRANSLATE:
            attname = 'Data'+att
        else:
            attname = HEADTRANSLATE[att] 
        headers[attname] = value

    #Some specials:
    headers['StationIAGAcode'] = headers.get('StationID')
    headers['DataFormat'] = headers.get('DataFormat') + '; ' + cdfdat.globalattsget().get('FormatVersion')

    try:
        try:
            pubdate = cdflib.cdfepoch.to_datetime(cdflib.cdfepoch,headers.get('DataPublicationDate'))
        except TypeError:
            pubdate = cdflib.cdfepoch.to_datetime(headers.get('DataPublicationDate'))
        headers['DataPublicationDate'] = pubdate[0]
        #pubdate = cdflib.cdfepoch.unixtime(headers.get('DataPublicationDate'))
        #headers['DataPublicationDate'] = datetime.utcfromtimestamp(pubdate[0]) 
    except:
        print ("imagcdf warning: Publication date is not provided as tt_2000")
        try:
            pubdate = DataStream()._testtime(headers.get('DataPublicationDate'))
            headers['DataPublicationDate'] = pubdate
        except:
            pass

    if debug:
        logger.info("readIMAGCDF: FOUND IMAGCDF file created with version {}".format(headers.get('DataFormat')))

    headers['DataLeapSecondUpdated'] = cdfdat.cdf_info().get('LeapSecondUpdated')
    if debug:
        print ("LEAP seconds updated:", cdfdat.cdf_info().get('LeapSecondUpdated'))
    # Get all available Variables - ImagCDF usually uses only zVariables
    datalist = cdfdat.cdf_info().get('zVariables')

    # New in 0.3.99 - provide a SensorID as well consisting of IAGA code, min/sec 
    # and numerical publevel

    #  IAGA code
    if headers.get('SensorID','') == '':
        try:
            #TODO determine resolution
            headers['SensorID'] = "{}_{}_{}".format(headers.get('StationIAGAcode','xxx').upper()+'sec',headers.get('DataPublicationLevel','0'),'0001')
        except:
            pass


    # #########################################################
    # 1. Now getting individual data and check time columns 
    # #########################################################
    zpos = KEYLIST.index('z') # used for idf records
    for elem in datalist:
        if elem.endswith('Times') and not elem.startswith('Flag'):
            try:
                #if elem in ['GeomagneticVectorTimes','GeomagneticTimes','GeomagneticScalarTimes']:
                tl = int(cdfdat.varinq(elem).get('Last_Rec'))
                tllist.append([tl,elem])
            except:
                pass

    if len(tllist) < 1:
        """
        No time column identified
        -> Check for starttime and sampling rate in header
        """
        if  cdfdat.globalattsget().get('StartTime','') and cdfdat.globalattsget().get('SamplingPeriod',''):
            # TODO Write that function
            st = cdfdat.globalattsget().get('StartTime','')
            sr = cdfdat.globalattsget().get('SamplingPeriod','')
            # get length of f or x 
        else:
            logger.error("readIMAGCDF: No Time information available - aborting")
            return
    elif len(tllist) > 1:
        tl = [el[0] for el in tllist]
        if not max(tl) == min(tl):
            if debug:
                print ("Time columns of different length")
            logger.warning("readIMAGCDF: Time columns of different length. Choosing longest as basis")
            newdatalist.append(['time',max(tllist)[1]])
            datnumar1 = date2num(np.asarray([datetime.utcfromtimestamp(el) for el in cdflib.cdfepoch.unixtime(cdfdat.varget(max(tllist)[1]))]))
            datnumar2 = date2num(np.asarray([datetime.utcfromtimestamp(el) for el in cdflib.cdfepoch.unixtime(cdfdat.varget(min(tllist)[1]))]))
            try:
                indexarray = np.nonzero(np.in1d(datnumar1,datnumar2))[0]
            except:
                indexarray = np.asarray([])
            # create a dictionary with time column name and indexarray
            multipletimedict = {min(tllist)[1]:indexarray}
        else:
            logger.info("readIMAGCDF: Equal length time axes found - assuming identical time")
            if 'GeomagneticVectorTimes' in datalist:
                newdatalist.append(['time','GeomagneticVectorTimes'])
            else:
                newdatalist.append(['time',tllist[0][1]]) # Take the first one
    else:
        #"Single time axis found in file"
        newdatalist.append(['time',tllist[0][1]])

    def Ruleset2Flaglist(flagginglist,rulesettype,rulesetversion):
        if rulesettype in ['Conrad', 'conrad', 'MagPy','magpy']:
            if rulesetversion in ['1.0','1',1]:
                flagcolsconrad = [flagginglist[0],flagginglist[1],flagginglist[3],flagginglist[4],flagginglist[5],flagginglist[6],flagginglist[2]]
                flaglisttmp = []
                for elem in flagcolsconrad:
                    flaglisttmp.append(cdfdat[elem][...])
                flaglisttmp[0] = cdflib.cdfepoch.to_datetime(cdflib.cdfepoch,flaglisttmp[0])
                flaglisttmp[1] = cdflib.cdfepoch.to_datetime(cdflib.cdfepoch,flaglisttmp[1])
                flaglisttmp[-1] = cdflib.cdfepoch.to_datetime(cdflib.cdfepoch,flaglisttmp[-1])
                flaglist = np.transpose(flaglisttmp)
                flaglist = [list(elem) for elem in flaglist]
                return list(flaglist)
        else:
            logger.warning("readIMAGCDF: Could  not interprete Ruleset")

    if not headers.get('FlagRulesetType','') == '':
        if debug:
            print ("readIMAGCDF: Found flagging ruleset {} vers.{} - extracting flagging information".format(headers.get('FlagRulesetType',''),headers.get('FlagRulesetVersion','')))
        logger.info("readIMAGCDF: Found flagging ruleset {} vers.{} - extracting flagging information".format(headers.get('FlagRulesetType',''),headers.get('FlagRulesetVersion','')))
        flagginglist = [elem for elem in datalist if elem.startswith('Flag')]
        flaglist = Ruleset2Flaglist(flagginglist,headers.get('FlagRulesetType',''),headers.get('FlagRulesetVersion',''))

    datalist = [elem for elem in datalist if not elem.endswith('Times') and not elem.startswith('Flag')]

    # #########################################################
    # 2. Sort the datalist according to KEYLIST
    # #########################################################
    for key in KEYLIST:
        possvals = [key]
        if key == 'x':
            possvals.extend(['h','i'])<commit_msg>corrected read issue for IMAGCDF vers 1.3 with flagging<commit_after>"""
MagPy
Intermagnet ImagCDF input filter
(based on cdflib)
Written by <NAME> October 2019
- contains test, read and write functions for
        ImagCDF
- supports python >= 3.5
- currently requires cdflib<=0.3.18
"""

from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from io import open

from magpy.stream import *

import cdflib

#import ciso8601 ## fast datetime parser  ciso8601.parse_datetime should be 10 times faster than datetime.strftime

import sys
import logging
logger = logging.getLogger(__name__)

HEADTRANSLATE = {'FormatDescription':'DataFormat', 'IagaCode':'StationID', 'ElementsRecorded':'DataComponents', 'ObservatoryName':'StationName', 'Latitude':'DataAcquisitionLatitude', 'Longitude':'DataAcquisitionLongitude', 'Institution':'StationInstitution', 'VectorSensOrient':'DataSensorOrientation', 'TermsOfUse':'DataTerms','UniqueIdentifier':'DataID','ParentIdentifiers':'SensorID','ReferenceLinks':'StationWebInfo', 'FlagRulesetType':'FlagRulesetType','FlagRulesetVersion':'FlagRulesetVersion'}


def isIMAGCDF(filename):
    """
    Checks whether a file is ImagCDF format
    """
    try:
        temp = cdflib.CDF(filename)
    except:
        return False
    try:
        form = temp.globalattsget().get('FormatDescription')
        if not form.startswith('INTERMAGNET'):
            return False
    except:
        return False

    print ("Running new ImagCDF import filter")

    logger.debug("isIMAGCDF: Found INTERMAGNET CDF data - using cdflib")
    return True

def readIMAGCDF(filename, headonly=False, **kwargs):
    """
    Reading Intermagnet CDF format (1.0,1.1,1.2)
    """

    debug = kwargs.get('debug')

    headers={}
    arraylist = []
    array = [[] for elem in KEYLIST]
    multipletimedict = {}
    newdatalist = []
    tllist = []
    indexarray = np.asarray([])

    cdfdat = cdflib.CDF(filename)

    if debug:
        print ("Reading ImagCDF with cdflib")

    for att in cdfdat.globalattsget():
        value = cdfdat.globalattsget().get(att)
        try:
            if isinstance(list(value), list):
                if len(value) == 1:
                    value = value[0]
        except:
            pass
        if not att in HEADTRANSLATE:
            attname = 'Data'+att
        else:
            attname = HEADTRANSLATE[att] 
        headers[attname] = value

    #Some specials:
    headers['StationIAGAcode'] = headers.get('StationID')
    headers['DataFormat'] = headers.get('DataFormat') + '; ' + cdfdat.globalattsget().get('FormatVersion')

    try:
        try:
            pubdate = cdflib.cdfepoch.to_datetime(cdflib.cdfepoch,headers.get('DataPublicationDate'))
        except TypeError:
            pubdate = cdflib.cdfepoch.to_datetime(headers.get('DataPublicationDate'))
        headers['DataPublicationDate'] = pubdate[0]
        #pubdate = cdflib.cdfepoch.unixtime(headers.get('DataPublicationDate'))
        #headers['DataPublicationDate'] = datetime.utcfromtimestamp(pubdate[0]) 
    except:
        print ("imagcdf warning: Publication date is not provided as tt_2000")
        try:
            pubdate = DataStream()._testtime(headers.get('DataPublicationDate'))
            headers['DataPublicationDate'] = pubdate
        except:
            pass

    if debug:
        logger.info("readIMAGCDF: FOUND IMAGCDF file created with version {}".format(headers.get('DataFormat')))

    headers['DataLeapSecondUpdated'] = cdfdat.cdf_info().get('LeapSecondUpdated')
    if debug:
        print ("LEAP seconds updated:", cdfdat.cdf_info().get('LeapSecondUpdated'))
    # Get all available Variables - ImagCDF usually uses only zVariables
    datalist = cdfdat.cdf_info().get('zVariables')

    # New in 0.3.99 - provide a SensorID as well consisting of IAGA code, min/sec 
    # and numerical publevel

    #  IAGA code
    if headers.get('SensorID','') == '':
        try:
            #TODO determine resolution
            headers['SensorID'] = "{}_{}_{}".format(headers.get('StationIAGAcode','xxx').upper()+'sec',headers.get('DataPublicationLevel','0'),'0001')
        except:
            pass


    # #########################################################
    # 1. Now getting individual data and check time columns 
    # #########################################################
    zpos = KEYLIST.index('z') # used for idf records
    for elem in datalist:
        if elem.endswith('Times') and not elem.startswith('Flag'):
            try:
                #if elem in ['GeomagneticVectorTimes','GeomagneticTimes','GeomagneticScalarTimes']:
                tl = int(cdfdat.varinq(elem).get('Last_Rec'))
                tllist.append([tl,elem])
            except:
                pass

    if len(tllist) < 1:
        """
        No time column identified
        -> Check for starttime and sampling rate in header
        """
        if  cdfdat.globalattsget().get('StartTime','') and cdfdat.globalattsget().get('SamplingPeriod',''):
            # TODO Write that function
            st = cdfdat.globalattsget().get('StartTime','')
            sr = cdfdat.globalattsget().get('SamplingPeriod','')
            # get length of f or x 
        else:
            logger.error("readIMAGCDF: No Time information available - aborting")
            return
    elif len(tllist) > 1:
        tl = [el[0] for el in tllist]
        if not max(tl) == min(tl):
            if debug:
                print ("Time columns of different length")
            logger.warning("readIMAGCDF: Time columns of different length. Choosing longest as basis")
            newdatalist.append(['time',max(tllist)[1]])
            datnumar1 = date2num(np.asarray([datetime.utcfromtimestamp(el) for el in cdflib.cdfepoch.unixtime(cdfdat.varget(max(tllist)[1]))]))
            datnumar2 = date2num(np.asarray([datetime.utcfromtimestamp(el) for el in cdflib.cdfepoch.unixtime(cdfdat.varget(min(tllist)[1]))]))
            try:
                indexarray = np.nonzero(np.in1d(datnumar1,datnumar2))[0]
            except:
                indexarray = np.asarray([])
            # create a dictionary with time column name and indexarray
            multipletimedict = {min(tllist)[1]:indexarray}
        else:
            logger.info("readIMAGCDF: Equal length time axes found - assuming identical time")
            if 'GeomagneticVectorTimes' in datalist:
                newdatalist.append(['time','GeomagneticVectorTimes'])
            else:
                newdatalist.append(['time',tllist[0][1]]) # Take the first one
    else:
        #"Single time axis found in file"
        newdatalist.append(['time',tllist[0][1]])

    def Ruleset2Flaglist(flagginglist,rulesettype,rulesetversion):
        if rulesettype in ['Conrad', 'conrad', 'MagPy','magpy']:
            if rulesetversion in ['1.0','1',1]:
                flagcolsconrad = [flagginglist[0],flagginglist[1],flagginglist[3],flagginglist[4],flagginglist[5],flagginglist[6],flagginglist[2]]
                flaglisttmp = []
                for elem in flagcolsconrad:
                    flaglisttmp.append(cdfdat[elem][...])
                try:
                    flaglisttmp[0] = cdflib.cdfepoch.to_datetime(cdflib.cdfepoch,flaglisttmp[0])
                except:
                    flaglisttmp[0] = cdflib.cdfepoch.to_datetime(flaglisttmp[0])
                try:
                    flaglisttmp[1] = cdflib.cdfepoch.to_datetime(cdflib.cdfepoch,flaglisttmp[1])
                except:
                    flaglisttmp[1] = cdflib.cdfepoch.to_datetime(flaglisttmp[1])
                try:
                    flaglisttmp[-1] = cdflib.cdfepoch.to_datetime(cdflib.cdfepoch,flaglisttmp[-1])
                except:
                    flaglisttmp[-1] = cdflib.cdfepoch.to_datetime(flaglisttmp[-1])
                flaglist = np.transpose(flaglisttmp)
                flaglist = [list(elem) for elem in flaglist]
                return list(flaglist)
            else:
                return []
        else:
            print ("readIMAGCDF: Could  not interprete flags ruleset")
            logger.warning("readIMAGCDF: Could  not interprete Ruleset")
            return []


    if not headers.get('FlagRulesetType','') == '':
        if debug:
            print ("readIMAGCDF: Found flagging ruleset {} vers.{} - extracting flagging information".format(headers.get('FlagRulesetType',''),headers.get('FlagRulesetVersion','')))
        logger.info("readIMAGCDF: Found flagging ruleset {} vers.{} - extracting flagging information".format(headers.get('FlagRulesetType',''),headers.get('FlagRulesetVersion','')))
        flagginglist = [elem for elem in datalist if elem.startswith('Flag')]
        flaglist = Ruleset2Flaglist(flagginglist,headers.get('FlagRulesetType',''),headers.get('FlagRulesetVersion',''))

    datalist = [elem for elem in datalist if not elem.endswith('Times') and not elem.startswith('Flag')]

    # #########################################################
    # 2. Sort the datalist according to KEYLIST
    # #########################################################
    for key in KEYLIST:
        possvals = [key]
        if key == 'x':
            possvals.extend(['h','i'])
{'fraction_non_alphanumeric': 0.14415916416583305, 'fraction_numerical': 0.009669889963321108, 'mean_word_length': 2.8640755851406485, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3683142', 'n_tokens_mistral': 5458, 'n_tokens_neox': 5265, 'n_words': 1129}