<commit_before># Adapted in part from <NAME>
# https://github.com/olivierhagolle/LANDSAT-Download
import os
import re
import requests
from future.standard_library import install_aliases
import tarfile
from pandas import read_csv

install_aliases()
from urllib.parse import urlencode
from urllib.request import urlopen, install_opener, build_opener, HTTPCookieProcessor, Request


class StationNotFoundError(Exception):
    pass


class InvalidSatelliteError(Exception):
    pass


class BadRequestsResponse(Exception):
    pass


class InvalidCredentialsResponse(Exception):
    pass


def download_image(url, output_dir, image, creds):
    cookies = HTTPCookieProcessor()
    opener = build_opener(cookies)
    install_opener(opener)

    data = urlopen("https://ers.cr.usgs.gov").read().decode('utf-8')
    m = re.search(r'<input .*?name="csrf_token".*?value="(.*?)"', data)
    if m:
        token = m.group(1)
    else:
        print("Error : CSRF_Token not found")

    params = urlencode(dict(username=creds['account'], password=<PASSWORD>['<PASSWORD>'], csrf_token=token))
    params = params.encode('ascii')
    request = Request("https://ers.cr.usgs.gov/login", params, headers={})
    f = urlopen(request)

    data = f.read().decode('utf-8')
    f.close()
    if data.find('Invalid username/password') > 0:
        print("Authentification failed")
        raise InvalidCredentialsResponse('Check your credentials, '
                                         'they were rejected by USGS')

    req = urlopen(url)

    uri = req.url
    response = requests.get(uri)

    if response.status_code == 200:
        with open(os.path.join(output_dir, image), 'wb') as f:
            for chunk in response.iter_content(chunk_size=1024 * 1024 * 8):
                f.write(chunk)

    elif response.status_code > 399:
        print('Code {}'.format(response.status_code))
        raise BadRequestsResponse(Exception)


def unzip_image(tgzfile, outputdir):
    target_tgz = os.path.join(outputdir, tgzfile)
    if os.path.exists(target_tgz):
        print('found tgs: {} \nunzipping...'.format(target_tgz))
        tfile = tarfile.open(target_tgz, 'r:gz')
        tfile.extractall(outputdir)
        print('unzipped\ndeleting tgz: {}'.format(target_tgz))
        os.remove(target_tgz)
    else:
        raise NotImplementedError('Did not find download output directory to unzip...')
    return None
<commit_msg>changed tarzip unzip to context managed<commit_after># Adapted in part from <NAME>
# https://github.com/olivierhagolle/LANDSAT-Download
import os
import re
import requests
from future.standard_library import install_aliases
import tarfile
from pandas import read_csv

install_aliases()
from urllib.parse import urlencode
from urllib.request import urlopen, install_opener, build_opener, HTTPCookieProcessor, Request


class StationNotFoundError(Exception):
    pass


class InvalidSatelliteError(Exception):
    pass


class BadRequestsResponse(Exception):
    pass


class InvalidCredentialsResponse(Exception):
    pass


def download_image(url, output_dir, image, creds):
    cookies = HTTPCookieProcessor()
    opener = build_opener(cookies)
    install_opener(opener)

    data = urlopen("https://ers.cr.usgs.gov").read().decode('utf-8')
    m = re.search(r'<input .*?name="csrf_token".*?value="(.*?)"', data)
    if m:
        token = m.group(1)
    else:
        print("Error : CSRF_Token not found")

    params = urlencode(dict(username=creds['account'], password=<PASSWORD>['<PASSWORD>'], csrf_token=token))
    params = params.encode('ascii')
    request = Request("https://ers.cr.usgs.gov/login", params, headers={})
    f = urlopen(request)

    data = f.read().decode('utf-8')
    f.close()
    if data.find('Invalid username/password') > 0:
        print("Authentification failed")
        raise InvalidCredentialsResponse('Check your credentials, '
                                         'they were rejected by USGS')

    req = urlopen(url)

    uri = req.url
    response = requests.get(uri)

    if response.status_code == 200:
        with open(os.path.join(output_dir, image), 'wb') as f:
            for chunk in response.iter_content(chunk_size=1024 * 1024 * 8):
                f.write(chunk)

    elif response.status_code > 399:
        print('Code {}'.format(response.status_code))
        raise BadRequestsResponse(Exception)


def unzip_image(tgzfile, outputdir):
    target_tgz = os.path.join(outputdir, tgzfile)
    if os.path.exists(target_tgz):
        print('found tgs: {} \nunzipping...'.format(target_tgz))
        with tarfile.open(target_tgz, 'r:gz') as tgz:
            tgz.extractall(outputdir)
            print('unzipped\ndeleting tgz: {}'.format(target_tgz))
        os.remove(target_tgz)
    else:
        raise NotImplementedError('Did not find download output directory to unzip...')
    return None

{'fraction_non_alphanumeric': 0.12183576867668244, 'fraction_numerical': 0.00782053920559786, 'mean_word_length': 3.3587443946188342, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5736733', 'n_tokens_mistral': 1558, 'n_tokens_neox': 1488, 'n_words': 333}