<commit_before>#! /usr/bin/env python3
"""Download and install renode release package."""

import argparse
import os
import re
import sys
import time
import tarfile
import urllib

from pathlib import Path

import wget


def download_artifact(url, artifact_name, out_dir):
    """Download the artifact from url."""
    if not os.path.isdir(out_dir):
        os.makedirs(out_dir)
    out_file = os.path.join(out_dir, artifact_name)
    download_url = os.path.join(url, artifact_name)
    num_retries = 3
    for i in range(num_retries + 1):
        try:
            wget.download(download_url, out=out_file)
            break
        except urllib.error.HTTPError as exception:
            if i == num_retries:
                raise
            print(f"{exception}\nDownload failed. Retrying...")
            time.sleep(5)
    return out_file


def strip_top_directory(tar):
    """Renode packages the release with a top directory. Need to strip it."""
    members = tar.getmembers()
    out_members = []
    for member in members[1:]:
        if member.path.startswith(members[0].name + "/"):
            member.path = os.path.join(
                *(member.name.split(os.path.sep)[1:]))
            out_members.append(member)

    tag = re.search("(.+?)_portable", members[0].name).group(1)
    return tag, out_members


def main():
    """Download and install renode release package."""
    pin_toolchains = os.getenv('PIN_TOOLCHAINS', '').lower().split(' ')
    if "renode" in pin_toolchains:
        print()
        print("****************************************************")
        print("*                                                  *")
        print("*  PIN_TOOLCHAINS includes renode! Skipping the    *")
        print("*  download of the latest Renode binaries.         *")
        print("*  PLEASE DON'T file bugs for Renode mis-behavior! *")
        print("*                                                  *")
        print("****************************************************")
        print()
        sys.exit(0)

    out_dir = os.getenv("OUT")
    if not out_dir:
        out_dir = "/tmp"

    parser = argparse.ArgumentParser(
        description="Download Renode from Antmicro releases")
    parser.add_argument(
        "--release_name", action="store", default="",
        help="Release to be downloaded. If not set, download the latest")
    parser.add_argument(
        "--release_url",
        action="store",
        default="https://dl.antmicro.com/projects/renode/builds/?C=M;O=D",
        help=("URL to check the IREE release."
              "(default: https://dl.antmicro.com/projects/renode/builds/)"))
    parser.add_argument(
        "--renode_dir", action="store", required=True,
        default="",
        help=("Renode installed directory")
    )

    args = parser.parse_args()

    with urllib.request.urlopen(args.release_url) as website:
        html_list = website.read().decode("utf-8")
    files = re.findall(r'href="(renode-.*git.*\.linux-portable\.tar\.gz)"',
                       html_list)

    release_found = False
    release_name = None
    if args.release_name:
        for file in files:
            if args.release_name in file:
                release_found = True
                release_name = re.search(
                    "(.+?).linux-portable.tar.gz", file).group(1)
                break
    else:
        # The latest release is the symlink of the second entry.
        release_found = True
        release_name = re.search(
            "(.+?).linux-portable.tar.gz", files[1]).group(1)

    if not release_found:
        print(f"!!!!!Renode can't be found with release {args.release_name}, "
              "please try a different release!!!!!")
        sys.exit(1)

    print(f"Release: {release_name}")
    commit_sha = re.search("git(.+?)$", release_name).group(1)<commit_msg>renode: Move query args outside of URL param<commit_after>#! /usr/bin/env python3
"""Download and install renode release package."""

import argparse
import os
import re
import sys
import time
import tarfile
import urllib

from pathlib import Path

import wget


def download_artifact(url, artifact_name, out_dir):
    """Download the artifact from url."""
    if not os.path.isdir(out_dir):
        os.makedirs(out_dir)
    out_file = os.path.join(out_dir, artifact_name)
    download_url = os.path.join(url, artifact_name)
    num_retries = 3
    for i in range(num_retries + 1):
        try:
            wget.download(download_url, out=out_file)
            break
        except urllib.error.HTTPError as exception:
            if i == num_retries:
                raise
            print(f"{exception}\nDownload failed. Retrying...")
            time.sleep(5)
    return out_file


def strip_top_directory(tar):
    """Renode packages the release with a top directory. Need to strip it."""
    members = tar.getmembers()
    out_members = []
    for member in members[1:]:
        if member.path.startswith(members[0].name + "/"):
            member.path = os.path.join(
                *(member.name.split(os.path.sep)[1:]))
            out_members.append(member)

    tag = re.search("(.+?)_portable", members[0].name).group(1)
    return tag, out_members


def main():
    """Download and install renode release package."""
    pin_toolchains = os.getenv('PIN_TOOLCHAINS', '').lower().split(' ')
    if "renode" in pin_toolchains:
        print()
        print("****************************************************")
        print("*                                                  *")
        print("*  PIN_TOOLCHAINS includes renode! Skipping the    *")
        print("*  download of the latest Renode binaries.         *")
        print("*  PLEASE DON'T file bugs for Renode mis-behavior! *")
        print("*                                                  *")
        print("****************************************************")
        print()
        sys.exit(0)

    out_dir = os.getenv("OUT")
    if not out_dir:
        out_dir = "/tmp"

    parser = argparse.ArgumentParser(
        description="Download Renode from Antmicro releases")
    parser.add_argument(
        "--release_name", action="store", default="",
        help="Release to be downloaded. If not set, download the latest")
    parser.add_argument(
        "--release_url",
        action="store",
        default="https://dl.antmicro.com/projects/renode/builds/",
        help=("URL to check the IREE release."
              "(default: https://dl.antmicro.com/projects/renode/builds/)"))
    parser.add_argument(
        "--renode_dir", action="store", required=True,
        default="",
        help=("Renode installed directory")
    )

    args = parser.parse_args()

    with urllib.request.urlopen(args.release_url + '?C=M;O=D') as website:
        html_list = website.read().decode("utf-8")
    files = re.findall(r'href="(renode-.*git.*\.linux-portable\.tar\.gz)"',
                       html_list)

    release_found = False
    release_name = None
    if args.release_name:
        for file in files:
            if args.release_name in file:
                release_found = True
                release_name = re.search(
                    "(.+?).linux-portable.tar.gz", file).group(1)
                break
    else:
        # The latest release is the symlink of the second entry.
        release_found = True
        release_name = re.search(
            "(.+?).linux-portable.tar.gz", files[1]).group(1)

    if not release_found:
        print(f"!!!!!Renode can't be found with release {args.release_name}, "
              "please try a different release!!!!!")
        sys.exit(1)

    print(f"Release: {release_name}")
    commit_sha = re.search("git(.+?)$", release_name).group(1)
{'fraction_non_alphanumeric': 0.15322997416020673, 'fraction_numerical': 0.004134366925064599, 'mean_word_length': 2.2281067556296916, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '872438', 'n_tokens_mistral': 2320, 'n_tokens_neox': 2225, 'n_words': 541}