<commit_before>import csv
import gzip


class FccCsv(object):
    """ This object wraps the FCC license CSV dataset.
    Currently, we're only looking for CL type licenses."""
    def __init__(self, csv_gzfile):
        self.csv_gzfile = csv_gzfile
        self.fields = self.get_fields()
        self.radio_service_code = "CL"

    def __iter__(self):
        with gzip.open(self.csv_gzfile) as gz_feed:
            feed = csv.DictReader(gz_feed, fieldnames=self.fields)
            for row in feed:
                if row["RADIO_SERVICE_CODE"] == self.radio_service_code:
                    yield row

    def get_fields(self):
        with gzip.open(self.csv_gzfile) as gz_feed:
            topline = gz_feed.readline()
            fields = self.get_fields_from_topline(topline)
            return fields

    def get_fields_from_topline(self, topline):
        cleaned_up = topline.replace('#', '', 1).replace('\n', '')
        fields = cleaned_up.split(',')
        return fields<commit_msg>filter for more than just "CL". solves issue of only AT&T/VZW towers being available in the FCC dataset.<commit_after>import csv
import gzip


class FccCsv(object):
    """
    This object wraps the FCC license CSV dataset. The license
    dataset includes _all_ FCC licensees, including things like TV
    and broadcast radio. So we filter down to a few types specific
    to cellular communications.
    """
    def __init__(self, csv_gzfile):
        self.csv_gzfile = csv_gzfile
        self.fields = self.get_fields()
        self.radio_service_code = frozenset({
            # legacy cellular service bands https://www.fcc.gov/wireless/bureau-divisions/mobility-division/cellular-service
            #   - 850MHz "CLR"
            #   - at&t, vzw
            "CL",
            # PCS https://www.fcc.gov/wireless/bureau-divisions/mobility-division/broadband-personal-communications-service-pcs
            #   - 1900MHz
            #   - at&t, sprint, t-mobile, vzw
            "CW",
            "CY",
            # AWS https://www.fcc.gov/wireless/bureau-divisions/mobility-division/advanced-wireless-services-aws
            #   - 1700/2100MHz
            #   - at&t, t-mobile, vzw
            "AW",
            "AD",
            # auctioned SMR https://www.fcc.gov/wireless/bureau-divisions/broadband-division/specialized-mobile-radio-service-smr
            #   - 800MHz
            #   - sprint "ESMR"
            #   - LTE band 27
            "YC",
            "YD",
            "YH",
            ### per the data dictionary, a few other things http://data.fcc.gov/download/license-view/fcc-license-view-data-dictionary.doc
            # BRS 2500MHz sprint (LTE B41)
            # https://www.fcc.gov/tags/broadband-radio-service-brs
            "BR",
            # 700MHz Lower Bands; at&t, t-mobile, vzw
            "WU",
            "WX",
            "WY",
            "WZ",
            # WCS 2300MHz; at&t; LTE band 30
            "WS"
            # ? CBRS 3600MHz (3550-3700MHz) https://www.fcc.gov/rulemaking/12-354
            "NN",
        })


    def __iter__(self):
        with gzip.open(self.csv_gzfile) as gz_feed:
            feed = csv.DictReader(gz_feed, fieldnames=self.fields)
            for row in feed:
                if row["RADIO_SERVICE_CODE"] in self.radio_service_code:
                    yield row

    def get_fields(self):
        with gzip.open(self.csv_gzfile) as gz_feed:
            topline = gz_feed.readline()
            fields = self.get_fields_from_topline(topline)
            return fields

    def get_fields_from_topline(self, topline):
        cleaned_up = topline.replace('#', '', 1).replace('\n', '')
        fields = cleaned_up.split(',')
        return fields
{'fraction_non_alphanumeric': 0.10842076943771858, 'fraction_numerical': 0.014527845036319613, 'mean_word_length': 1.9183673469387754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4119154', 'n_tokens_mistral': 1234, 'n_tokens_neox': 1137, 'n_words': 254}