<commit_before>import requests
import json
import os
import zipfile

API_TOKEN = os.environ.get('GITHUB_API_TOKEN')
params = {'API_TOKEN': API_TOKEN}

url = "https://api.github.com/search/repositories?q={}".format(input())


def urlretrieve(url, path):
    with open(path, 'wb') as f:
        r = requests.get(url, stream=True)
        r.raise_for_status()  # Replace this with better error handling.

        for chunk in r.iter_content(1024):
            f.write(chunk)


def extractfiles(zipf):
    with zipfile.ZipFile(zipf, "r") as z:
        z.extractall()


def main():
    # request = urllib.request.Request(url)
    # request.add_header('Authorization', 'token %s' % API_TOKEN)
    response = requests.get(url, params=params).text
    search_results = json.loads(response)
    first_result = search_results['items'][0]
    download_url = first_result['html_url'] + '/archive/master.zip'
    repo_name = first_result['name']
    print(download_url)
    urlretrieve(download_url, "{}.zip".format(repo_name))
    extractfiles("{}.zip".format(repo_name))
    os.rename("{}-master".format(repo_name), "{}".format(repo_name))
    os.unlink("{}.zip".format(repo_name))

if __name__ == "__main__":
    main()<commit_msg>add docstrings and comments and remove redundant code<commit_after>import requests
import json
import os
import zipfile

API_TOKEN = os.environ.get('GITHUB_API_TOKEN')
params = {'API_TOKEN': API_TOKEN}  # create a dict to be passed by the request

url = "https://api.github.com/search/repositories?q={}".format(input())


def urlretrieve(url, path):
    """Retrieves a zipfile and writes it to a local disk"""
    with open(path, 'wb') as f:
        r = requests.get(url, stream=True)
        for chunk in r.iter_content(1024):
            f.write(chunk)


def extractfiles(zipf):
    """Extract a zipfile to the current directory"""
    with zipfile.ZipFile(zipf, "r") as z:
        z.extractall()


def main():
    response = requests.get(url, params=params).text
    search_results = json.loads(response)
    first_result = search_results['items'][0]  # takes out the first result
    download_url = first_result['html_url'] + '/archive/master.zip'
    repo_name = first_result['name']  # stores the repository name
    print(download_url)
    urlretrieve(download_url, "{}.zip".format(repo_name))
    extractfiles("{}.zip".format(repo_name))
    os.rename("{}-master".format(repo_name), "{}".format(repo_name))
    os.unlink("{}.zip".format(repo_name))

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.14825119236883944, 'fraction_numerical': 0.00397456279809221, 'mean_word_length': 4.01394422310757, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '567034', 'n_tokens_mistral': 839, 'n_tokens_neox': 839, 'n_words': 182}