<commit_before>import datetime
import re
import urllib
import urllib.parse
import requests

from itertools import chain

class IdxField(object):
    def __init__(self, idx_line):
        self.idx_data = idx_line.strip().split(':')

        self.index = int(self.idx_data[0])
        self.bytes_start = int(self.idx_data[1])
        self.reftime = datetime.datetime.strptime(
            self.idx_data[2].split('=')[1], '%Y%m%d%H'
        )
        self.varname = self.idx_data[3]
        self.level = self.idx_data[4]
        self.interval = self.idx_data[5]

    def __str__(self):
        return ':'.join([
            str(self.index), str(self.bytes_start),
            self.reftime.strftime('d=%Y%m%d%H'), self.varname,
            self.level, self.interval
        ])

    def __repr__(self):
        return '{!s}({!r})'.format(
            type(self).__name__,
            str(self)
        )


class IdxCollection(object):
    def __init__(self):
        self.i_to_idx = dict()
        self.idx_to_i = dict()

    def add_idx(self, idx):
        self.i_to_idx[idx.index] = idx
        self.idx_to_i[idx] = idx.index

    def byterange(self, idx):
        bytes_start = idx.bytes_start
        try:
            bytes_end = self.i_to_idx[idx.index+1].bytes_start - 1
        except KeyError:
            bytes_end = ''

        return (bytes_start, bytes_end)

    def filter(self, regex):
        matches = list()
        regex = re.compile(regex)

        for idx in self.i_to_idx.values():
            if regex.match(str(idx)) is not None:
                matches.append(idx)

        return matches





class NomadsDownloader(object):

    SERVER = 'www.ftp.ncep.noaa.gov'
    BASE_URL = '/data/nccf/com/gfs/prod/'
    STEPS = {<commit_msg>Removed function definitions for empty dicts & lists where applicable.<commit_after>import datetime
import re
import urllib
import urllib.parse
import requests

from itertools import chain

class IdxField(object):
    def __init__(self, idx_line):
        self.idx_data = idx_line.strip().split(':')

        self.index = int(self.idx_data[0])
        self.bytes_start = int(self.idx_data[1])
        self.reftime = datetime.datetime.strptime(
            self.idx_data[2].split('=')[1], '%Y%m%d%H'
        )
        self.varname = self.idx_data[3]
        self.level = self.idx_data[4]
        self.interval = self.idx_data[5]

    def __str__(self):
        return ':'.join([
            str(self.index), str(self.bytes_start),
            self.reftime.strftime('d=%Y%m%d%H'), self.varname,
            self.level, self.interval
        ])

    def __repr__(self):
        return '{!s}({!r})'.format(
            type(self).__name__,
            str(self)
        )


class IdxCollection(object):
    def __init__(self):
        self.i_to_idx = {}
        self.idx_to_i = {}

    def add_idx(self, idx):
        self.i_to_idx[idx.index] = idx
        self.idx_to_i[idx] = idx.index

    def byterange(self, idx):
        bytes_start = idx.bytes_start
        try:
            bytes_end = self.i_to_idx[idx.index+1].bytes_start - 1
        except KeyError:
            bytes_end = ''

        return (bytes_start, bytes_end)

    def filter(self, regex):
        matches = []
        regex = re.compile(regex)

        for idx in self.i_to_idx.values():
            if regex.match(str(idx)) is not None:
                matches.append(idx)

        return matches





class NomadsDownloader(object):

    SERVER = 'www.ftp.ncep.noaa.gov'
    BASE_URL = '/data/nccf/com/gfs/prod/'
    STEPS = {
{'fraction_non_alphanumeric': 0.12976022566995768, 'fraction_numerical': 0.005077574047954866, 'mean_word_length': 2.1297440423654015, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3541199', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1176, 'n_words': 219}