<commit_before>#!/usr/bin/env python3

import argparse
import logging
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import typing
from contextlib import contextmanager
from pathlib import Path
from sys import stdout

logging.basicConfig(level=logging.INFO, format="%(message)s")
logger = logging.getLogger(__file__)

ARTIFACTS_LOCATION = Path("C:/_/artifacts")
RDEPS_REGEX = re.compile(
    r"Required By\s*:\s*(?P<rdeps>[\s\S]*)Optional For\s*:\s*"
)


def convert_win2unix_path(winpath: typing.Union[Path, str]):
    p = subprocess.run(
        ["cygpath", "-u", str(winpath)],
        capture_output=True,
        text=True,
        check=True,
    )
    return p.stdout.strip()


def install_package(pkg: typing.Union[Path, str], local: bool) -> None:
    logger.info("Installing %s", pkg)
    if local:
        command = [
            "pacman",
            "--noprogressbar",
            "--upgrade",
            "--noconfirm",
            convert_win2unix_path(pkg),
        ]
    else:
        command = [
            "pacman",
            "-S",
            "--noconfirm",
            pkg,
        ]
    subprocess.run(
        command,
        check=True,
    )


def get_rdeps(pkg: str) -> typing.List[str]:
    p = subprocess.run(
        ["pacman", "-Sii", pkg],
        capture_output=True,
        text=True,
        check=True,
    )
    ret = p.stdout
    rdeps_re = RDEPS_REGEX.search(ret)
    if rdeps_re:
        pkgs_str = rdeps_re.group("rdeps")
        pkgs = [i.strip() for i in pkgs_str.split() if i != "None"]
        return pkgs
    else:
        raise ValueError("Unable to parse Pacman output")


def run_pip_check(pkg: str) -> None:
    p = subprocess.run(
        [sys.executable, "-m", "pip", "check"],
        capture_output=True,
        text=True,
    )
    if p.returncode != 0:
        logger.error("Pip Check Failed for %s", pkg)<commit_msg>ci-check: query the local package<commit_after>#!/usr/bin/env python3

import argparse
import logging
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import typing
from contextlib import contextmanager
from pathlib import Path
from sys import stdout

logging.basicConfig(level=logging.INFO, format="%(message)s")
logger = logging.getLogger(__file__)

ARTIFACTS_LOCATION = Path("C:/_/artifacts")
RDEPS_REGEX = re.compile(
    r"Required By\s*:\s*(?P<rdeps>[\s\S]*)Optional For\s*:\s*"
)


def convert_win2unix_path(winpath: typing.Union[Path, str]):
    p = subprocess.run(
        ["cygpath", "-u", str(winpath)],
        capture_output=True,
        text=True,
        check=True,
    )
    return p.stdout.strip()


def install_package(pkg: typing.Union[Path, str], local: bool) -> None:
    logger.info("Installing %s", pkg)
    if local:
        command = [
            "pacman",
            "--noprogressbar",
            "--upgrade",
            "--noconfirm",
            convert_win2unix_path(pkg),
        ]
    else:
        command = [
            "pacman",
            "-S",
            "--noconfirm",
            pkg,
        ]
    subprocess.run(
        command,
        check=True,
    )


def get_rdeps(pkg: str) -> typing.List[str]:
    p = subprocess.run(
        ["pacman", "-Qi", pkg],
        capture_output=True,
        text=True,
        check=True,
    )
    ret = p.stdout
    rdeps_re = RDEPS_REGEX.search(ret)
    if rdeps_re:
        pkgs_str = rdeps_re.group("rdeps")
        pkgs = [i.strip() for i in pkgs_str.split() if i != "None"]
        return pkgs
    else:
        raise ValueError("Unable to parse Pacman output")


def run_pip_check(pkg: str) -> None:
    p = subprocess.run(
        [sys.executable, "-m", "pip", "check"],
        capture_output=True,
        text=True,
    )
    if p.returncode != 0:
        logger.error("Pip Check Failed for %s", pkg)
{'fraction_non_alphanumeric': 0.1345804104962328, 'fraction_numerical': 0.0020784619381657575, 'mean_word_length': 2.2849829351535837, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1624576', 'n_tokens_mistral': 1338, 'n_tokens_neox': 1269, 'n_words': 289}