<commit_before>#
# Copyright 2017-2020 European Centre for Medium-Range Weather Forecasts (ECMWF).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Authors:
#   <NAME> - B-Open - https://bopen.eu
#   <NAME> - B-Open - https://bopen.eu
#   <NAME> - B-Open - https://bopen.eu
#

import itertools
import logging
import typing as T  # noqa
import warnings

import numpy as np  # type: ignore
import xarray as xr

from . import cfmessage, dataset

LOGGER = logging.getLogger(__name__)


DEFAULT_GRIB_KEYS = {
    "centre": 255,  # missing value, see: http://apps.ecmwf.int/codes/grib/format/grib1/centre/0/
    "typeOfLevel": "surface",
}
TYPE_OF_LEVELS_SFC = ["surface", "meanSea", "cloudBase", "cloudTop"]
TYPE_OF_LEVELS_PL = ["isobaricInhPa", "isobaricInPa"]
TYPE_OF_LEVELS_ML = ["hybrid"]
ALL_TYPE_OF_LEVELS = TYPE_OF_LEVELS_SFC + TYPE_OF_LEVELS_PL + TYPE_OF_LEVELS_ML
GRID_TYPES = [
    "polar_stereographic",
    "reduced_gg",
    "reduced_ll",
    "regular_gg",
    "regular_ll",
    "rotated_gg",
    "rotated_ll",
    "sh",
]
MESSAGE_DEFINITION_KEYS = [
    # for the GRIB 2 sample we must set this before setting 'totalNumber'<commit_msg>Code stye<commit_after>#
# Copyright 2017-2020 European Centre for Medium-Range Weather Forecasts (ECMWF).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Authors:
#   <NAME> - B-Open - https://bopen.eu
#   <NAME> - B-Open - https://bopen.eu
#   <NAME> - B-Open - https://bopen.eu
#

import itertools
import logging
import typing as T
import warnings

import numpy as np  # type: ignore
import xarray as xr

from . import cfmessage, dataset

LOGGER = logging.getLogger(__name__)


DEFAULT_GRIB_KEYS = {
    "centre": 255,  # missing value, see: http://apps.ecmwf.int/codes/grib/format/grib1/centre/0/
    "typeOfLevel": "surface",
}
TYPE_OF_LEVELS_SFC = ["surface", "meanSea", "cloudBase", "cloudTop"]
TYPE_OF_LEVELS_PL = ["isobaricInhPa", "isobaricInPa"]
TYPE_OF_LEVELS_ML = ["hybrid"]
ALL_TYPE_OF_LEVELS = TYPE_OF_LEVELS_SFC + TYPE_OF_LEVELS_PL + TYPE_OF_LEVELS_ML
GRID_TYPES = [
    "polar_stereographic",
    "reduced_gg",
    "reduced_ll",
    "regular_gg",
    "regular_ll",
    "rotated_gg",
    "rotated_ll",
    "sh",
]
MESSAGE_DEFINITION_KEYS = [
    # for the GRIB 2 sample we must set this before setting 'totalNumber'
{'fraction_non_alphanumeric': 0.10910757946210269, 'fraction_numerical': 0.011002444987775062, 'mean_word_length': 4.106084243369735, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6505072', 'n_tokens_mistral': 1133, 'n_tokens_neox': 1088, 'n_words': 339}