<commit_before>#!/usr/bin/env python3

import sqlite3
import xml.etree.ElementTree as ET
import requests

def fetch_grantees():
    r = requests.get('https://fcc.io/grantees')
    print(dir(r))
    print(r.headers['content-type'])
    print(r.encoding)

attributes = [
    'grantee_code',
    'grantee_name',
    'mailing_address',
    'po_box',
    'city',
    'state',
    'country',
    'zip_code',
    'contact_name',
    'date_received',
]

def parse_grantees():
    global attributes
    rows = []
    tree = ET.parse('grantees.xml')
    root = tree.getroot()
    for row in root.iter('Row'):
        grantee = []
        for attribute in attributes:
            x = row.find(attribute)
            if hasattr(x, 'text'):
                grantee.append(x.text)
        rows.append(grantee)
    print("Found %d grantees" % len(rows))
    return rows

def write_db(granteeTest):
    conn = sqlite3.connect('FCCgrantees.db')
    c = conn.cursor()

    c.execute("""drop table if exists grantees""")
    conn.commit()

    c.execute('''CREATE TABLE grantees
                (grantee_code int primary key not NULL,  
                grantee_name text,
                mailing_address text,
                po_box text,
                city text,
                state text,
                country text,
                zip_code text,
                contact_name text,
                date_received text)''')

    c.executemany('INSERT INTO grantees VALUES (?,?,?,?,?,?,?,?,?,?)', granteeTest)
    conn.commit()

    for row in c:
        print(row)

    c.close()
    print("Table Created")


if __name__ == "__main__":
    grantees = parse_grantees()
    write_db(grantees)<commit_msg>Added functionality to allow for both remotely and locally accessing grantee data<commit_after>#!/usr/bin/env python3

import sqlite3
import xml.etree.ElementTree as ET
import requests
#from sys import argv
import argparse

# saves all grantees from url to an xml file
def fetch_grantees_xml():
    r = requests.get('https://apps.fcc.gov/eas/GetEntityDownload.xml?type=G')
    with open('grantees.xml', 'w') as f:
        f.write(r.text)

attributes = [
    'grantee_code',
    'grantee_name',
    'mailing_address',
    'po_box',
    'city',
    'state',
    'country',
    'zip_code',
    'contact_name',
    'date_received',
]

def parse_grantees():
    global attributes
    rows = []
    tree = ET.parse('grantees.xml')
    root = tree.getroot()
    for row in root.iter('Row'):
        grantee = []
        for attribute in attributes:
            x = row.find(attribute)
            if hasattr(x, 'text'):
                grantee.append(x.text)
        rows.append(grantee)
    print("Found %d grantees" % len(rows))
    return rows

def write_db(granteeTest):
    conn = sqlite3.connect('FCCgrantees.db')
    c = conn.cursor()

    c.execute("""DROP TABLE IF EXISTS grantees""")
    conn.commit()

    c.execute('''CREATE TABLE grantees
                (grantee_code int primary key not NULL,  
                grantee_name text,
                mailing_address text,
                po_box text,
                city text,
                state text,
                country text,
                zip_code text,
                contact_name text,
                date_received text)''')

    c.executemany('INSERT INTO grantees VALUES (?,?,?,?,?,?,?,?,?,?)', granteeTest)
    conn.commit()

    #for row in c:
    #   print(row)

    c.close()
    print("Table Created in FCCgrantees.db")


if __name__ == "__main__": 
    parser = argparse.ArgumentParser()
    parser.add_argument("-r", "--remote", help="Download grantee data from FCC website", action="store_true")
    parser.add_argument("-l", "--local", help="Create grantee database from local file", action="store_true")
    args = parser.parse_args()
    #if argv[1] == '-r':
    if args.remote:
        print("Downloading grantee data...")
        fetch_grantees_xml()
    if args.local:
        try:
            grantees = parse_grantees()
        except FileNotFoundError:
            print("No local xml file found.")
            print("Use argument --help for help")
            print("Downloading...")
            fetch_grantees_xml()
            grantees = parse_grantees()
    write_db(grantees)
{'fraction_non_alphanumeric': 0.12702512326837287, 'fraction_numerical': 0.001643578304766377, 'mean_word_length': 2.2150943396226417, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7086585', 'n_tokens_mistral': 1387, 'n_tokens_neox': 1284, 'n_words': 316}