<commit_before>#!/usr/bin/env python3
import sys
import json
import argparse
import base64
import requests
from requests.auth import HTTPBasicAuth
from dateutil import parser
import re
from datetime import datetime
from pathlib import Path

user = None
token = None


def printProgressBar(iteration, total, prefix = '', length = 60, fill = '█'):
    filledLength = int(length * iteration // total)
    bar = (fill * filledLength) + ('-' * (length - filledLength))
    percent = 100 * (iteration / float(total))
    fmt = f"\r{prefix} |{bar}| {percent:.1f}%"
    sys.stdout.write(fmt)
    # Print New Line on Complete
    if iteration == total:
        print()


def getfile(url):
    return requests.get(url, auth=HTTPBasicAuth(user, token))


def getPluginJson(plugin):
    if "site" in plugin:
        print("We only currently support github projects")
        return

    site = "https://github.com/"
    userAndProject = plugin["name"]
    userName = plugin["name"].split("/")[0]
    projectUrl = f"https://api.github.com/repos/{userAndProject}"
    releasesUrl = f"{projectUrl}/releases/tags"
    tagsUrl = f"{projectUrl}/tags"

    releaseData = None

    if 'auto_update' in plugin and plugin['auto_update']:
        releaseList = f"{projectUrl}/releases"
        try:
            releaseData = sorted(getfile(releaseList).json(), key=lambda x: parser.parse(x['published_at']), reverse=True)[0]
            if "message" in releaseData and releaseData["message"] == "Not Found":
                print(f"\n\nERROR: {plugin['name']}, Couldn't get release information. Likely the user created a tag but no associated release.\n")
                return None
            plugin['tag'] = releaseData['tag_name']
        except requests.exceptions.HTTPError:
            print(f" Unable get get url {releaseList}")
            return None
    else:

        releases = f"{releasesUrl}/{plugin['tag']}"
        try:
            releaseData = getfile(releases).json()
            if "message" in releaseData and releaseData["message"] == "Not Found":
                print(f"\n\nERROR: {plugin['name']}, Couldn't get release information. Likely the user created a tag but no associated release.\n")
                print(f"Tried to use URL: {releases}")
                return None
        except requests.exceptions.HTTPError:
            print(f" Unable get get url {releases}")
            return None

    commit = None
    zipUrl = None<commit_msg>use latest API to avoid GH awful sorting issues<commit_after>#!/usr/bin/env python3
import sys
import json
import argparse
import base64
import requests
from requests.auth import HTTPBasicAuth
from dateutil import parser
import re
from datetime import datetime
from pathlib import Path

user = None
token = None


def printProgressBar(iteration, total, prefix = '', length = 60, fill = '█'):
    filledLength = int(length * iteration // total)
    bar = (fill * filledLength) + ('-' * (length - filledLength))
    percent = 100 * (iteration / float(total))
    fmt = f"\r{prefix} |{bar}| {percent:.1f}%"
    sys.stdout.write(fmt)
    # Print New Line on Complete
    if iteration == total:
        print()


def getfile(url):
    return requests.get(url, auth=HTTPBasicAuth(user, token))


def getPluginJson(plugin):
    if "site" in plugin:
        print("We only currently support github projects")
        return

    site = "https://github.com/"
    userAndProject = plugin["name"]
    userName = plugin["name"].split("/")[0]
    projectUrl = f"https://api.github.com/repos/{userAndProject}"
    releasesUrl = f"{projectUrl}/releases/tags"
    tagsUrl = f"{projectUrl}/tags"

    releaseData = None

    if 'auto_update' in plugin and plugin['auto_update']:
        latestRelease = f"{projectUrl}/releases/latest"
        try:
            releaseData = getfile(latestRelease).json()
            if "message" in releaseData and releaseData["message"] == "Not Found":
                print(f"\n\nERROR: {plugin['name']}, Couldn't get release information. Likely the user created a tag but no associated release.\n")
                return None
            plugin['tag'] = releaseData['tag_name']
        except requests.exceptions.HTTPError:
            print(f" Unable get get url {latestRelease}")
            return None
    else:

        releases = f"{releasesUrl}/{plugin['tag']}"
        try:
            releaseData = getfile(releases).json()
            if "message" in releaseData and releaseData["message"] == "Not Found":
                print(f"\n\nERROR: {plugin['name']}, Couldn't get release information. Likely the user created a tag but no associated release.\n")
                print(f"Tried to use URL: {releases}")
                return None
        except requests.exceptions.HTTPError:
            print(f" Unable get get url {releases}")
            return None

    commit = None
    zipUrl = None
{'fraction_non_alphanumeric': 0.12036087758868157, 'fraction_numerical': 0.004305925774041419, 'mean_word_length': 2.7843289371605895, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3552667', 'n_tokens_mistral': 1419, 'n_tokens_neox': 1344, 'n_words': 415}