<commit_before>'''
List of supported filetypes and associated parameters
Whips.py will consider all classes in this module with names 
ending in "_filetype" to be valid filetype definitions

Each class in this file has, at minimum, an attribute "parser"
that specifies the associated parser for that class and an 
attribute "doutf" that specifies the default output function.
(List the name of the output function, not including "_out_func")

Each class may also contain any other variables that will be 
used for output function attributes if the output function
accepts those attributes

In some cases, output function attributes may consist of lists
associated with field names for that particular filetype.  In 
these cases, the variables for these attributes will be 
dictionaries keyed off the field name strings.

Some parsers require additional attributes (beyond the 
filename, subtype, and extension). These attributes may be
listed in the ParserParms dictionary, and whips.py will 
require valid values for these arguments before processing
files of this type. Entries in the ParserParms dictionary
should follow the form
   'RequiredAttrName':("AttrDescription",'AttrCastType')
where AttrCastType is one of the cast types for OutFuncAttrs
   [None, 'int', 'posint', 'decimal', 'posdecimal',
    'time', 'bool', 'list', 'listoflists']
'''

class OMI_NO2_KNMI_HDF_v2_0_preFeb2006_filetype():
    parser = 'HDFknmiomil2'
    parserParms = {}
    doutf = 'OMNO2e_netCDF_avg'
    overallQualFlag = 'TroposphericColumnFlag'
    cloudFrac = 'CloudFraction'
    solarZenithAngle = 'SolarZenithAngle'
    time = 'Time'
    longitude = 'Longitude'
    pixIndXtrackAxis = '1'
    outUnits = { 'AirMassFactor' : 'Unitless',
                 'AirMassFactorGeometric' : 'Unitless',
                 'AirMassFactorTropospheric' : 'Unitless',
                 'AssimilatedStratosphericSlantColumn' : 'Molecules cm^-2 x 10^-15',
                 'AssimilatedStratosphericVerticalColumn' : 'Molecules cm^-2 x 10^-15',
                 'AveragingKernel' : 'Unitless x 1000',
                 'CloudFraction' : 'Unitless x 1000',
                 'CloudFractionStd' : 'Unitless x 1000',
                 'CloudPressure' : 'hPa',
                 'CloudPressureStd' : 'hPa',
                 'CloudRadianceFraction' : 'Percent x 100',
                 'GhostColumn' : 'Molecules cm^-2 x 10^-15',
                 'InstrumentConfigurationId' : 'Unitless',
                 'MeasurementQualityFlags' : 'Unitless',
                 'SlantColumnAmountNO2' : 'Molecules cm^-2 x 10^-15',
                 'SlantColumnAmountNO2Std' : 'Molecules cm^-2 x 10^-15',
                 'Surface Albedo' : 'Unitless x 10,000',
                 'TM4PressurelevelA' : 'Pa',
                 'TM4PressurelevelB' : 'Unitless', 
                 'TM4SurfacePressure' : 'hPa',
                 'TM4TerrainHeight' : 'meters',
                 'TM4TropoPauseLevel' : 'Unitless',
                 'TerrainHeight' : 'meters',
                 'TotalVerticalColumn' : 'Molecules cm^-2 x 10^-15',
                 'TotalVerticalColumnError' : 'Molecules cm^-2 x 10^-15',
                 'TroposphericColumnFlag' : 'Unitless',
                 'TroposphericVerticalColumn' : 'Molecules cm^-2 x 10^-15',
                 'TroposphericVerticalColumnError' : 'Molecules cm^-2 x 10^-15',
                 'TroposphericVerticalColumnModel' : 'Molecules cm^-2 x 10^-15',
                 'VCDErrorUsingAvKernel' : 'Molecules cm^-2 x 10^-15',
                 'VCDTropErrorUsingAvKernel' : 'Molecules cm^-2 x 10^-15',
                 'GroundPixelQualityFlag' : 'Unitless',
                 'Latitude' : 'Degrees',
                 'LatitudeCornerpoints' : 'Degrees',
                 'Longitude' : 'Degrees',
                 'LongitudeCornerpoints' : 'Degrees',
                 'SolarAzimuthAngle' : 'Degrees',
                 'SolarZenithAngle' : 'Degrees',
                 'Time' : 'seconds',
                 'ViewingAzimuthAngle' : 'Degrees',
                 'ViewingZenithAngle' : 'Degrees' }

    extraDimLabel = { 'AirMassFactor' : 'none',
                      'AirMassFactorGeometric' : 'none',
                      'AirMassFactorTropospheric' : 'none',
                      'AssimilatedStratosphericSlantColumn' : 'none',
                      'AssimilatedStratosphericVerticalColumn' : 'none',
                      'AveragingKernel' : 'layer',
                      'CloudFraction' : 'none',
                      'CloudFractionStd' : 'none',
                      'CloudPressure' : 'none',
                      'CloudPressureStd' : 'none',
                      'CloudRadianceFraction' : 'none',
                      'GhostColumn' : 'none',
                      'InstrumentConfigurationId' : 'none',
                      'MeasurementQualityFlags' : 'none',
                      'SlantColumnAmountNO2' : 'none',
                      'SlantColumnAmountNO2Std' : 'none',
                      'SurfaceAlbedo' : 'none',
                      'TM4PressurelevelA' : 'layer',
                      'TM4PressurelevelB' : 'layer',
                      'TM4SurfacePressure' : 'none',
                      'TM4TerrainHeight' : 'none',
                      'TM4TropoPauseLevel' : 'none',
                      'TerrainHeight' : 'none',
                      'TotalVerticalColumn' : 'none',
                      'TotalVerticalColumnError' : 'none',
                      'TroposphericColumnFlag' : 'none',
                      'TroposphericVerticalColumn' : 'none',
                      'TroposphericVerticalColumnError' : 'none',
                      'TroposphericVerticalColumnModel' : 'none',
                      'VCDErrorUsingAvKernel' : 'none',
                      'VCDTropErrorUsingAvKernel' : 'none',
                      'GroundPixelQualityFlag' : 'none',
                      'Latitude' : 'none',
                      'LatitudeCornerPoints': 'corner',
                      'Longitude' : 'none',
                      'LongitudeCornerPoints' : 'corner',
                      'SolarAzimuthAngle' : 'none',
                      'SolarZenithAngle' : 'none',
                      'Time' : 'none',
                      'ViewingAzimuthAngle' : 'none',
                      'ViewingZenithAngle' : 'none' }

    extraDimSize = { 'AirMassFactor' : 0,
                     'AirMassFactorGeometric' : 0,
                     'AirMassFactorTropospheric' : 0,
                     'AssimilatedStratosphericSlantColumn' : 0,
                     'AssimilatedStratosphericVerticalColumn' : 0,
                     'AveragingKernel' : 35,
                     'CloudFraction' : 0,
                     'CloudFractionStd' : 0,
                     'CloudPressure' : 0,
                     'CloudPressureStd' : 0,
                     'CloudRadianceFraction' : 0,
                     'GhostColumn' : 0,
                     'InstrumentConfigurationId' : 0,
                     'MeasurementQualityFlags' : 0,
                     'SlantColumnAmountNO2' : 0,
                     'SlantColumnAmountNO2Std' : 0,
                     'SurfaceAlbedo' : 0,
                     'TM4PressurelevelA' : 35,
                     'TM4PressurelevelB' : 35,
                     'TM4SurfacePressure' : 0,
                     'TM4TerrainHeight' : 0,
                     'TM4TropoPauseLevel' : 0,
                     'TerrainHeight' : 0,
                     'TotalVerticalColumn' : 0,
                     'TotalVerticalColumnError' : 0,
                     'TroposphericColumnFlag' : 0,
                     'TroposphericVerticalColumn' : 0,
                     'TroposphericVerticalColumnError' : 0,
                     'TroposphericVerticalColumnModel' : 0,
                     'VCDErrorUsingAvKernel' : 0,
                     'VCDTropErrorUsingAvKernel' : 0, 
                     'GroundPixelQualityFlag' : 0,
                     'Latitude' : 0,
                     'LatitudeCornerPoints' : 4,
                     'Longitude' : 0,
                     'LongitudeCornerPoints' : 4,
                     'SolarAzimuthAngle' : 0,
                     'SolarZenithAngle' : 0,
                     'Time' : 0,
                     'ViewingAzimuthAngle' : 0,
                     'ViewingZenithAngle' : 0 }

class OMI_NO2_KNMI_HDF_v2_0_postFeb2006_filetype():
    parser = 'HDFknmiomil2'
    parserParms = {}
    doutf = 'OMNO2e_netCDF_avg'
    overallQualFlag = 'TroposphericColumnFlag'
    cloudFrac = 'CloudFraction'
    solarZenithAngle = 'SolarZenithAngle'
    time = 'Time'
    longitude = 'Longitude'
    pixIndXtrackAxis = '1'
    outUnits = { 'AirMassFactor' : 'Unitless',
                 'AirMassFactorGeometric' : 'Unitless',
                 'AirMassFactorTropospheric' : 'Unitless',
                 'AssimilatedStratosphericSlantColumn' : 'Molecules cm^-2 x 10^-15',
                 'AssimilatedStratosphericVerticalColumn' : 'Molecules cm^-2 x 10^-15',
                 'AveragingKernel' : 'Unitless x 1000',
                 'CloudFraction' : 'Unitless x 1000',
                 'CloudFractionStd' : 'Unitless x 1000',
                 'CloudPressure' : 'hPa',<commit_msg>Fixed casting issue in filetypes.py<commit_after>'''
List of supported filetypes and associated parameters
Whips.py will consider all classes in this module with names 
ending in "_filetype" to be valid filetype definitions

Each class in this file has, at minimum, an attribute "parser"
that specifies the associated parser for that class and an 
attribute "doutf" that specifies the default output function.
(List the name of the output function, not including "_out_func")

Each class may also contain any other variables that will be 
used for output function attributes if the output function
accepts those attributes

In some cases, output function attributes may consist of lists
associated with field names for that particular filetype.  In 
these cases, the variables for these attributes will be 
dictionaries keyed off the field name strings.

Some parsers require additional attributes (beyond the 
filename, subtype, and extension). These attributes may be
listed in the ParserParms dictionary, and whips.py will 
require valid values for these arguments before processing
files of this type. Entries in the ParserParms dictionary
should follow the form

   'RequiredAttrName':("AttrDescription",'AttrCastType')

where AttrCastType is one of the cast types for OutFuncAttrs

   [None, 'int', 'posint', 'decimal', 'posdecimal',
    'time', 'bool', 'list', 'listoflists']
'''

class OMI_NO2_KNMI_HDF_v2_0_preFeb2006_filetype():
    parser = 'HDFknmiomil2'
    parserParms = {}
    doutf = 'OMNO2e_netCDF_avg'
    overallQualFlag = 'TroposphericColumnFlag'
    cloudFrac = 'CloudFraction'
    solarZenithAngle = 'SolarZenithAngle'
    time = 'Time'
    longitude = 'Longitude'
    pixIndXtrackAxis = 1
    outUnits = { 'AirMassFactor' : 'Unitless',
                 'AirMassFactorGeometric' : 'Unitless',
                 'AirMassFactorTropospheric' : 'Unitless',
                 'AssimilatedStratosphericSlantColumn' : 'Molecules cm^-2 x 10^-15',
                 'AssimilatedStratosphericVerticalColumn' : 'Molecules cm^-2 x 10^-15',
                 'AveragingKernel' : 'Unitless x 1000',
                 'CloudFraction' : 'Unitless x 1000',
                 'CloudFractionStd' : 'Unitless x 1000',
                 'CloudPressure' : 'hPa',
                 'CloudPressureStd' : 'hPa',
                 'CloudRadianceFraction' : 'Percent x 100',
                 'GhostColumn' : 'Molecules cm^-2 x 10^-15',
                 'InstrumentConfigurationId' : 'Unitless',
                 'MeasurementQualityFlags' : 'Unitless',
                 'SlantColumnAmountNO2' : 'Molecules cm^-2 x 10^-15',
                 'SlantColumnAmountNO2Std' : 'Molecules cm^-2 x 10^-15',
                 'Surface Albedo' : 'Unitless x 10,000',
                 'TM4PressurelevelA' : 'Pa',
                 'TM4PressurelevelB' : 'Unitless', 
                 'TM4SurfacePressure' : 'hPa',
                 'TM4TerrainHeight' : 'meters',
                 'TM4TropoPauseLevel' : 'Unitless',
                 'TerrainHeight' : 'meters',
                 'TotalVerticalColumn' : 'Molecules cm^-2 x 10^-15',
                 'TotalVerticalColumnError' : 'Molecules cm^-2 x 10^-15',
                 'TroposphericColumnFlag' : 'Unitless',
                 'TroposphericVerticalColumn' : 'Molecules cm^-2 x 10^-15',
                 'TroposphericVerticalColumnError' : 'Molecules cm^-2 x 10^-15',
                 'TroposphericVerticalColumnModel' : 'Molecules cm^-2 x 10^-15',
                 'VCDErrorUsingAvKernel' : 'Molecules cm^-2 x 10^-15',
                 'VCDTropErrorUsingAvKernel' : 'Molecules cm^-2 x 10^-15',
                 'GroundPixelQualityFlag' : 'Unitless',
                 'Latitude' : 'Degrees',
                 'LatitudeCornerpoints' : 'Degrees',
                 'Longitude' : 'Degrees',
                 'LongitudeCornerpoints' : 'Degrees',
                 'SolarAzimuthAngle' : 'Degrees',
                 'SolarZenithAngle' : 'Degrees',
                 'Time' : 'seconds',
                 'ViewingAzimuthAngle' : 'Degrees',
                 'ViewingZenithAngle' : 'Degrees' }

    extraDimLabel = { 'AirMassFactor' : 'none',
                      'AirMassFactorGeometric' : 'none',
                      'AirMassFactorTropospheric' : 'none',
                      'AssimilatedStratosphericSlantColumn' : 'none',
                      'AssimilatedStratosphericVerticalColumn' : 'none',
                      'AveragingKernel' : 'layer',
                      'CloudFraction' : 'none',
                      'CloudFractionStd' : 'none',
                      'CloudPressure' : 'none',
                      'CloudPressureStd' : 'none',
                      'CloudRadianceFraction' : 'none',
                      'GhostColumn' : 'none',
                      'InstrumentConfigurationId' : 'none',
                      'MeasurementQualityFlags' : 'none',
                      'SlantColumnAmountNO2' : 'none',
                      'SlantColumnAmountNO2Std' : 'none',
                      'SurfaceAlbedo' : 'none',
                      'TM4PressurelevelA' : 'layer',
                      'TM4PressurelevelB' : 'layer',
                      'TM4SurfacePressure' : 'none',
                      'TM4TerrainHeight' : 'none',
                      'TM4TropoPauseLevel' : 'none',
                      'TerrainHeight' : 'none',
                      'TotalVerticalColumn' : 'none',
                      'TotalVerticalColumnError' : 'none',
                      'TroposphericColumnFlag' : 'none',
                      'TroposphericVerticalColumn' : 'none',
                      'TroposphericVerticalColumnError' : 'none',
                      'TroposphericVerticalColumnModel' : 'none',
                      'VCDErrorUsingAvKernel' : 'none',
                      'VCDTropErrorUsingAvKernel' : 'none',
                      'GroundPixelQualityFlag' : 'none',
                      'Latitude' : 'none',
                      'LatitudeCornerPoints': 'corner',
                      'Longitude' : 'none',
                      'LongitudeCornerPoints' : 'corner',
                      'SolarAzimuthAngle' : 'none',
                      'SolarZenithAngle' : 'none',
                      'Time' : 'none',
                      'ViewingAzimuthAngle' : 'none',
                      'ViewingZenithAngle' : 'none' }

    extraDimSize = { 'AirMassFactor' : 0,
                     'AirMassFactorGeometric' : 0,
                     'AirMassFactorTropospheric' : 0,
                     'AssimilatedStratosphericSlantColumn' : 0,
                     'AssimilatedStratosphericVerticalColumn' : 0,
                     'AveragingKernel' : 35,
                     'CloudFraction' : 0,
                     'CloudFractionStd' : 0,
                     'CloudPressure' : 0,
                     'CloudPressureStd' : 0,
                     'CloudRadianceFraction' : 0,
                     'GhostColumn' : 0,
                     'InstrumentConfigurationId' : 0,
                     'MeasurementQualityFlags' : 0,
                     'SlantColumnAmountNO2' : 0,
                     'SlantColumnAmountNO2Std' : 0,
                     'SurfaceAlbedo' : 0,
                     'TM4PressurelevelA' : 35,
                     'TM4PressurelevelB' : 35,
                     'TM4SurfacePressure' : 0,
                     'TM4TerrainHeight' : 0,
                     'TM4TropoPauseLevel' : 0,
                     'TerrainHeight' : 0,
                     'TotalVerticalColumn' : 0,
                     'TotalVerticalColumnError' : 0,
                     'TroposphericColumnFlag' : 0,
                     'TroposphericVerticalColumn' : 0,
                     'TroposphericVerticalColumnError' : 0,
                     'TroposphericVerticalColumnModel' : 0,
                     'VCDErrorUsingAvKernel' : 0,
                     'VCDTropErrorUsingAvKernel' : 0, 
                     'GroundPixelQualityFlag' : 0,
                     'Latitude' : 0,
                     'LatitudeCornerPoints' : 4,
                     'Longitude' : 0,
                     'LongitudeCornerPoints' : 4,
                     'SolarAzimuthAngle' : 0,
                     'SolarZenithAngle' : 0,
                     'Time' : 0,
                     'ViewingAzimuthAngle' : 0,
                     'ViewingZenithAngle' : 0 }

class OMI_NO2_KNMI_HDF_v2_0_postFeb2006_filetype():
    parser = 'HDFknmiomil2'
    parserParms = {}
    doutf = 'OMNO2e_netCDF_avg'
    overallQualFlag = 'TroposphericColumnFlag'
    cloudFrac = 'CloudFraction'
    solarZenithAngle = 'SolarZenithAngle'
    time = 'Time'
    longitude = 'Longitude'
    pixIndXtrackAxis = 1
    outUnits = { 'AirMassFactor' : 'Unitless',
                 'AirMassFactorGeometric' : 'Unitless',
                 'AirMassFactorTropospheric' : 'Unitless',
                 'AssimilatedStratosphericSlantColumn' : 'Molecules cm^-2 x 10^-15',
                 'AssimilatedStratosphericVerticalColumn' : 'Molecules cm^-2 x 10^-15',
                 'AveragingKernel' : 'Unitless x 1000',
                 'CloudFraction' : 'Unitless x 1000',
                 'CloudFractionStd' : 'Unitless x 1000',
                 'CloudPressure' : 'hPa',
{'fraction_non_alphanumeric': 0.0986329720603453, 'fraction_numerical': 0.020369260933500354, 'mean_word_length': 1.6959330494787843, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '205527', 'n_tokens_mistral': 5505, 'n_tokens_neox': 4954, 'n_words': 1107}