<commit_before>#!/usr/bin/python3
# -*- coding: UTF-8 -*-
from typing import Optional
import argparse
import sys
import requests
import urllib.parse
import os
import json
import pathlib
import zipfile
import shutil
import logging

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
logger.setLevel(logging.DEBUG)
# disable logging from the package used by requests
logging.getLogger("urllib3").setLevel(logging.WARNING)


class UrlDescriptor(object):
    GITHUB_LATEST_TEMPLATE = "https://api.github.com/repos/{}/releases/latest"

    def __init__(self, url: str):
        self._url = url
        self._github_profile = None
        self._github_repo = None
        self._github_latest_release_url = None
        self._filename = None

        if self._parse_github_repo_url():
            logger.info("URL : '{}' is a Github repo URL.".format(self._url))
            return

        logger.info("URL: {} is standard URL.".format(self._url))
        self.parse_url()

    def __repr__(self):
        return "[gh: {}][fname: {}] {}".format(
            self.is_github_repo_url, self._filename, self._url)

    def get_github_package_url(self) -> Optional[str]:
        profile_and_repo = "{}/{}".format(self._github_profile,
                                          self._github_repo)

        url = self.GITHUB_LATEST_TEMPLATE.format(profile_and_repo)
        response = requests.get(url)
        if response.status_code != 200:
            logger.error("Github URL response code was: {}"
                         .format(response.status_code))
            return None

        response_text = response.text
        response_json = json.loads(response_text)
        if "assets" not in response_json:
            logger.error("No \"assets\" in github response.")
            return None

        assets = response_json["assets"]
        if not isinstance(assets, list):
            logger.error("\"assets\" in github response is not a list.")
            return None

        if 'browser_download_url' not in assets[0]:
            logger.error("No \"browser_download_url\" in assets[0].")
            return None

        github_latest_release_url = assets[0]["browser_download_url"]
        if not github_latest_release_url:<commit_msg>try sending my username to prevent a 403 from github API.<commit_after>#!/usr/bin/python3
# -*- coding: UTF-8 -*-
from typing import Optional
import argparse
import sys
import requests
import urllib.parse
import os
import json
import pathlib
import zipfile
import shutil
import logging

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
logger.setLevel(logging.DEBUG)
# disable logging from the package used by requests
logging.getLogger("urllib3").setLevel(logging.WARNING)

HTTP_HEADERS = {
    'User-Agent': 'neitsa',
}


class UrlDescriptor(object):
    GITHUB_LATEST_TEMPLATE = "https://api.github.com/repos/{}/releases/latest"

    def __init__(self, url: str):
        self._url = url
        self._github_profile = None
        self._github_repo = None
        self._github_latest_release_url = None
        self._filename = None

        if self._parse_github_repo_url():
            logger.info("URL : '{}' is a Github repo URL.".format(self._url))
            return

        logger.info("URL: {} is standard URL.".format(self._url))
        self.parse_url()

    def __repr__(self):
        return "[gh: {}][fname: {}] {}".format(
            self.is_github_repo_url, self._filename, self._url)

    def get_github_package_url(self) -> Optional[str]:
        profile_and_repo = "{}/{}".format(self._github_profile,
                                          self._github_repo)

        url = self.GITHUB_LATEST_TEMPLATE.format(profile_and_repo)
        response = requests.get(url, headers=HTTP_HEADERS)
        if response.status_code != 200:
            logger.error("Github URL response code was: {}"
                         .format(response.status_code))
            return None

        response_text = response.text
        response_json = json.loads(response_text)
        if "assets" not in response_json:
            logger.error("No \"assets\" in github response.")
            return None

        assets = response_json["assets"]
        if not isinstance(assets, list):
            logger.error("\"assets\" in github response is not a list.")
            return None

        if 'browser_download_url' not in assets[0]:
            logger.error("No \"browser_download_url\" in assets[0].")
            return None

        github_latest_release_url = assets[0]["browser_download_url"]
        if not github_latest_release_url:
{'fraction_non_alphanumeric': 0.1070198105081826, 'fraction_numerical': 0.004521963824289405, 'mean_word_length': 2.5593869731800765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5203326', 'n_tokens_mistral': 1350, 'n_tokens_neox': 1299, 'n_words': 313}