<commit_before>#
# dsextras.py - Extra classes and utilities
#
# TODO:
# Make it possible to import codegen from another dir
#

from distutils.command.build_ext import build_ext
from distutils.command.install_lib import install_lib
from distutils.extension import Extension
import fnmatch
import os
import string
import sys

GLOBAL_INC = []
GLOBAL_MACROS = []

def getoutput(cmd):
    """Return output (stdout or stderr) of executing cmd in a shell."""
    return getstatusoutput(cmd)[1]

def getstatusoutput(cmd):
    """Return (status, output) of executing cmd in a shell."""
    if sys.platform == 'win32':
        pipe = os.popen(cmd, 'r')
        text = pipe.read()
        sts = pipe.close() or 0
        if text[-1] == '\n':
            text = text[:-1]
        return sts, text
    else:
        from commands import getstatusoutput
        return getstatusoutput(cmd)

def have_pkgconfig():
    """Checks for the existence of pkg-config"""
    if (sys.platform == 'win32' and
        os.system('pkg-config --version > NUL') == 0):
        return 1
    else:
        if getstatusoutput('pkg-config')[0] == 256:
            return 1

def list_files(dir):
    """List all files in a dir, with filename match support:
    for example: glade/*.glade will return all files in the glade directory
    that matches *.glade. It also looks up the full path"""
    if dir.find(os.sep) != -1:
        parts = dir.split(os.sep)
        dir = string.join(parts[:-1], os.sep)
        pattern = parts[-1]<commit_msg>Small typo.<commit_after>#
# dsextras.py - Extra classes and utilities
#
# TODO:
# Make it possible to import codegen from another dir
#

from distutils.command.build_ext import build_ext
from distutils.command.install_lib import install_lib
from distutils.extension import Extension
import fnmatch
import os
import string
import sys

GLOBAL_INC = []
GLOBAL_MACROS = []

def getoutput(cmd):
    """Return output (stdout or stderr) of executing cmd in a shell."""
    return getstatusoutput(cmd)[1]

def getstatusoutput(cmd):
    """Return (status, output) of executing cmd in a shell."""
    if sys.platform == 'win32':
        pipe = os.popen(cmd, 'r')
        text = pipe.read()
        sts = pipe.close() or 0
        if text[-1:] == '\n':
            text = text[:-1]
        return sts, text
    else:
        from commands import getstatusoutput
        return getstatusoutput(cmd)

def have_pkgconfig():
    """Checks for the existence of pkg-config"""
    if (sys.platform == 'win32' and
        os.system('pkg-config --version > NUL') == 0):
        return 1
    else:
        if getstatusoutput('pkg-config')[0] == 256:
            return 1

def list_files(dir):
    """List all files in a dir, with filename match support:
    for example: glade/*.glade will return all files in the glade directory
    that matches *.glade. It also looks up the full path"""
    if dir.find(os.sep) != -1:
        parts = dir.split(os.sep)
        dir = string.join(parts[:-1], os.sep)
        pattern = parts[-1]
{'fraction_non_alphanumeric': 0.11733510109380178, 'fraction_numerical': 0.011932383162081539, 'mean_word_length': 2.7677902621722845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5139187', 'n_tokens_mistral': 939, 'n_tokens_neox': 911, 'n_words': 316}