<commit_before>import requests
import pandas as pd
from zipfile import ZipFile
from StringIO import StringIO
from eduextractor.config import secrets

NWEA_USERNAME = secrets['nwea_map']['username'] 
NWEA_PASSWORD = secrets['nwea_map']['secret']

def load_map_files():
## send a GET request to the API endpoint
    print 'Connecting to NWEA...'
    cdf_url = 'https://api.mapnwea.org/services/reporting/dex'
    r = requests.get(cdf_url, auth=(NWEA_USERNAME,NWEA_PASSWORD))

    print r.reason
## if the request is successful...
    if r.status_code == 200:
        print 'Downloading CDF...'
        ## convert the response content into a zipfile
        z = ZipFile(StringIO(r.content))
        zfiles = z.namelist()

        for i, f in enumerate(zfiles):    
            print 'Extracting', f
            ## open/extract file from zip
            cdf = z.open(f)
            ## read into dataframe
            df = pd.read_csv(cdf)        
            ## define the table name        
            df.to_csv('/tmp/map.csv')
    print 'Done!'

if __name__ == '__main__':
    load_map_files()<commit_msg>more map stuff<commit_after>import requests
import pandas as pd
from zipfile import ZipFile
from StringIO import StringIO
from eduextractor.config import secrets

NWEA_USERNAME = secrets['nwea_map']['username'] 
NWEA_PASSWORD = secrets['nwea_map']['secret']


def load_map_files():
    # send a GET request to the API endpoint
    print 'Connecting to NWEA...'
    cdf_url = "https://api.mapnwea.org/services/reporting/dex"
    r = requests.get(cdf_url, auth=(NWEA_USERNAME, NWEA_PASSWORD))

    print r.reason
# if the request is successful...
    if r.status_code == 200:
        print 'Downloading CDF...'
        # convert the response content into a zipfile
        z = ZipFile(StringIO(r.content))
        zfiles = z.namelist()

        for i, f in enumerate(zfiles):    
            print 'Extracting', f
            # open/extract file from zip
            cdf = z.open(f)

            # read into dataframe
            df = pd.read_csv(cdf)        
            df.to_csv('/tmp/map.csv')
    print 'Done!'

if __name__ == '__main__':
    load_map_files()
{'fraction_non_alphanumeric': 0.10664819944598337, 'fraction_numerical': 0.002770083102493075, 'mean_word_length': 2.5008077544426492, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '633150', 'n_tokens_mistral': 702, 'n_tokens_neox': 675, 'n_words': 188}