<commit_before>#!/usr/bin/env python3

# Copyright 2012-2016 The Meson development team

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools
import os
import subprocess
import shutil
import sys
import signal
from io import StringIO
from ast import literal_eval
from enum import Enum
import tempfile
from pathlib import Path, PurePath
from mesonbuild import build
from mesonbuild import environment
from mesonbuild import mesonlib
from mesonbuild import mlog
from mesonbuild import mtest
from mesonbuild.mesonlib import stringlistify, Popen_safe
from mesonbuild.coredata import backendlist
import argparse
import xml.etree.ElementTree as ET
import time
import multiprocessing
from concurrent.futures import ProcessPoolExecutor
import re
from run_tests import get_fake_options, run_configure, get_meson_script
from run_tests import get_backend_commands, get_backend_args_for_dir, Backend
from run_tests import ensure_backend_detects_changes


class BuildStep(Enum):
    configure = 1
    build = 2
    test = 3
    install = 4
    clean = 5
    validate = 6


class TestResult:
    def __init__(self, msg, step, stdo, stde, mlog, conftime=0, buildtime=0, testtime=0):
        self.msg = msg
        self.step = step
        self.stdo = stdo
        self.stde = stde
        self.mlog = mlog
        self.conftime = conftime
        self.buildtime = buildtime
        self.testtime = testtime


class AutoDeletedDir:
    def __init__(self, d):
        self.dir = d

    def __enter__(self):
        os.makedirs(self.dir, exist_ok=True)
        return self.dir

    def __exit__(self, _type, value, traceback):
        # We don't use tempfile.TemporaryDirectory, but wrap the
        # deletion in the AutoDeletedDir class because
        # it fails on Windows due antivirus programs
        # holding files open.
        mesonlib.windows_proof_rmtree(self.dir)

failing_logs = []
print_debug = 'MESON_PRINT_TEST_OUTPUT' in os.environ
under_ci = not {'TRAVIS', 'APPVEYOR'}.isdisjoint(os.environ)
do_debug = under_ci or print_debug
no_meson_log_msg = 'No meson-log.txt found.'

system_compiler = None

class StopException(Exception):
    def __init__(self):
        super().__init__('Stopped by user')

stop = False
def stop_handler(signal, frame):
    global stop
    stop = True
signal.signal(signal.SIGINT, stop_handler)
signal.signal(signal.SIGTERM, stop_handler)

def setup_commands(optbackend):
    global do_debug, backend, backend_flags
    global compile_commands, clean_commands, test_commands, install_commands, uninstall_commands
    backend = optbackend
    msbuild_exe = shutil.which('msbuild')
    # Auto-detect backend if unspecified
    if backend is None:
        if msbuild_exe is not None:
            backend = 'vs' # Meson will auto-detect VS version to use
        else:
            backend = 'ninja'
    # Set backend arguments for Meson
    if backend.startswith('vs'):
        backend_flags = ['--backend=' + backend]
        backend = Backend.vs
    elif backend == 'xcode':
        backend_flags = ['--backend=xcode']
        backend = Backend.xcode
    elif backend == 'ninja':
        backend_flags = ['--backend=ninja']
        backend = Backend.ninja
    else:
        raise RuntimeError('Unknown backend: {!r}'.format(backend))
    compile_commands, clean_commands, test_commands, install_commands, \
        uninstall_commands = get_backend_commands(backend, do_debug)

def get_relative_files_list_from_dir(fromdir):
    paths = []
    for (root, _, files) in os.walk(fromdir):
        reldir = os.path.relpath(root, start=fromdir)
        for f in files:
            path = os.path.join(reldir, f).replace('\\', '/')
            if path.startswith('./'):
                path = path[2:]
            paths.append(path)
    return paths

def platform_fix_name(fname, compiler, env):
    if '?lib' in fname:
        if mesonlib.for_cygwin(env.is_cross_build(), env):
            fname = re.sub(r'lib/\?lib(.*)\.so$', r'bin/cyg\1.dll', fname)
            fname = re.sub(r'\?lib(.*)\.dll$', r'cyg\1.dll', fname)
        else:
            fname = re.sub(r'\?lib', 'lib', fname)

    if fname.endswith('?exe'):
        fname = fname[:-4]
        if mesonlib.for_windows(env.is_cross_build(), env) or mesonlib.for_cygwin(env.is_cross_build(), env):
            return fname + '.exe'

    if fname.startswith('?msvc:'):
        fname = fname[6:]
        if compiler != 'cl':
            return None

    if fname.startswith('?gcc:'):
        fname = fname[5:]
        if compiler == 'cl':
            return None

    return fname

def validate_install(srcdir, installdir, compiler, env):
    # List of installed files
    info_file = os.path.join(srcdir, 'installed_files.txt')
    # If this exists, the test does not install any other files
    noinst_file = 'usr/no-installed-files'
    expected = {}
    ret_msg = ''
    # Generate list of expected files
    if os.path.exists(os.path.join(installdir, noinst_file)):
        expected[noinst_file] = False
    elif os.path.exists(info_file):
        with open(info_file) as f:
            for line in f:
                line = platform_fix_name(line.strip(), compiler, env)
                if line:
                    expected[line] = False
    # Check if expected files were found
    for fname in expected:
        file_path = os.path.join(installdir, fname)
        if os.path.exists(file_path) or os.path.islink(file_path):
            expected[fname] = True
    for (fname, found) in expected.items():
        if not found:
            # Ignore missing PDB files if we aren't using cl
            if fname.endswith('.pdb') and compiler != 'cl':
                continue
            ret_msg += 'Expected file {0} missing.\n'.format(fname)
    # Check if there are any unexpected files
    found = get_relative_files_list_from_dir(installdir)
    for fname in found:
        # Windows-specific tests check for the existence of installed PDB
        # files, but common tests do not, for obvious reasons. Ignore any
        # extra PDB files found.
        if fname not in expected and not fname.endswith('.pdb') and compiler == 'cl':
            ret_msg += 'Extra file {0} found.\n'.format(fname)
    return ret_msg

def log_text_file(logfile, testdir, stdo, stde):
    global stop, executor, futures
    logfile.write('%s\nstdout\n\n---\n' % testdir.as_posix())
    logfile.write(stdo)
    logfile.write('\n\n---\n\nstderr\n\n---\n')
    logfile.write(stde)
    logfile.write('\n\n---\n\n')
    if print_debug:
        try:
            print(stdo)
        except UnicodeError:
            sanitized_out = stdo.encode('ascii', errors='replace').decode()
            print(sanitized_out)
        try:
            print(stde, file=sys.stderr)
        except UnicodeError:
            sanitized_err = stde.encode('ascii', errors='replace').decode()
            print(sanitized_err, file=sys.stderr)
    if stop:
        print("Aborting..")
        for f in futures:
            f[2].cancel()
        executor.shutdown()
        raise StopException()


def bold(text):
    return mlog.bold(text).get_text(mlog.colorize_console)<commit_msg>run_projects_tests: Fail if we install extra files even if not in cl<commit_after>#!/usr/bin/env python3

# Copyright 2012-2016 The Meson development team

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools
import os
import subprocess
import shutil
import sys
import signal
from io import StringIO
from ast import literal_eval
from enum import Enum
import tempfile
from pathlib import Path, PurePath
from mesonbuild import build
from mesonbuild import environment
from mesonbuild import mesonlib
from mesonbuild import mlog
from mesonbuild import mtest
from mesonbuild.mesonlib import stringlistify, Popen_safe
from mesonbuild.coredata import backendlist
import argparse
import xml.etree.ElementTree as ET
import time
import multiprocessing
from concurrent.futures import ProcessPoolExecutor
import re
from run_tests import get_fake_options, run_configure, get_meson_script
from run_tests import get_backend_commands, get_backend_args_for_dir, Backend
from run_tests import ensure_backend_detects_changes


class BuildStep(Enum):
    configure = 1
    build = 2
    test = 3
    install = 4
    clean = 5
    validate = 6


class TestResult:
    def __init__(self, msg, step, stdo, stde, mlog, conftime=0, buildtime=0, testtime=0):
        self.msg = msg
        self.step = step
        self.stdo = stdo
        self.stde = stde
        self.mlog = mlog
        self.conftime = conftime
        self.buildtime = buildtime
        self.testtime = testtime


class AutoDeletedDir:
    def __init__(self, d):
        self.dir = d

    def __enter__(self):
        os.makedirs(self.dir, exist_ok=True)
        return self.dir

    def __exit__(self, _type, value, traceback):
        # We don't use tempfile.TemporaryDirectory, but wrap the
        # deletion in the AutoDeletedDir class because
        # it fails on Windows due antivirus programs
        # holding files open.
        mesonlib.windows_proof_rmtree(self.dir)

failing_logs = []
print_debug = 'MESON_PRINT_TEST_OUTPUT' in os.environ
under_ci = not {'TRAVIS', 'APPVEYOR'}.isdisjoint(os.environ)
do_debug = under_ci or print_debug
no_meson_log_msg = 'No meson-log.txt found.'

system_compiler = None

class StopException(Exception):
    def __init__(self):
        super().__init__('Stopped by user')

stop = False
def stop_handler(signal, frame):
    global stop
    stop = True
signal.signal(signal.SIGINT, stop_handler)
signal.signal(signal.SIGTERM, stop_handler)

def setup_commands(optbackend):
    global do_debug, backend, backend_flags
    global compile_commands, clean_commands, test_commands, install_commands, uninstall_commands
    backend = optbackend
    msbuild_exe = shutil.which('msbuild')
    # Auto-detect backend if unspecified
    if backend is None:
        if msbuild_exe is not None:
            backend = 'vs' # Meson will auto-detect VS version to use
        else:
            backend = 'ninja'
    # Set backend arguments for Meson
    if backend.startswith('vs'):
        backend_flags = ['--backend=' + backend]
        backend = Backend.vs
    elif backend == 'xcode':
        backend_flags = ['--backend=xcode']
        backend = Backend.xcode
    elif backend == 'ninja':
        backend_flags = ['--backend=ninja']
        backend = Backend.ninja
    else:
        raise RuntimeError('Unknown backend: {!r}'.format(backend))
    compile_commands, clean_commands, test_commands, install_commands, \
        uninstall_commands = get_backend_commands(backend, do_debug)

def get_relative_files_list_from_dir(fromdir):
    paths = []
    for (root, _, files) in os.walk(fromdir):
        reldir = os.path.relpath(root, start=fromdir)
        for f in files:
            path = os.path.join(reldir, f).replace('\\', '/')
            if path.startswith('./'):
                path = path[2:]
            paths.append(path)
    return paths

def platform_fix_name(fname, compiler, env):
    if '?lib' in fname:
        if mesonlib.for_cygwin(env.is_cross_build(), env):
            fname = re.sub(r'lib/\?lib(.*)\.so$', r'bin/cyg\1.dll', fname)
            fname = re.sub(r'\?lib(.*)\.dll$', r'cyg\1.dll', fname)
        else:
            fname = re.sub(r'\?lib', 'lib', fname)

    if fname.endswith('?exe'):
        fname = fname[:-4]
        if mesonlib.for_windows(env.is_cross_build(), env) or mesonlib.for_cygwin(env.is_cross_build(), env):
            return fname + '.exe'

    if fname.startswith('?msvc:'):
        fname = fname[6:]
        if compiler != 'cl':
            return None

    if fname.startswith('?gcc:'):
        fname = fname[5:]
        if compiler == 'cl':
            return None

    return fname

def validate_install(srcdir, installdir, compiler, env):
    # List of installed files
    info_file = os.path.join(srcdir, 'installed_files.txt')
    # If this exists, the test does not install any other files
    noinst_file = 'usr/no-installed-files'
    expected = {}
    ret_msg = ''
    # Generate list of expected files
    if os.path.exists(os.path.join(installdir, noinst_file)):
        expected[noinst_file] = False
    elif os.path.exists(info_file):
        with open(info_file) as f:
            for line in f:
                line = platform_fix_name(line.strip(), compiler, env)
                if line:
                    expected[line] = False
    # Check if expected files were found
    for fname in expected:
        file_path = os.path.join(installdir, fname)
        if os.path.exists(file_path) or os.path.islink(file_path):
            expected[fname] = True
    for (fname, found) in expected.items():
        if not found:
            # Ignore missing PDB files if we aren't using cl
            if fname.endswith('.pdb') and compiler != 'cl':
                continue
            ret_msg += 'Expected file {0} missing.\n'.format(fname)
    # Check if there are any unexpected files
    found = get_relative_files_list_from_dir(installdir)
    for fname in found:
        # Windows-specific tests check for the existence of installed PDB
        # files, but common tests do not, for obvious reasons. Ignore any
        # extra PDB files found.
        if fname not in expected:
            if not (fname.endswith('.pdb') and compiler != 'cl'):
                ret_msg += 'Extra file {0} found.\n'.format(fname)
    return ret_msg

def log_text_file(logfile, testdir, stdo, stde):
    global stop, executor, futures
    logfile.write('%s\nstdout\n\n---\n' % testdir.as_posix())
    logfile.write(stdo)
    logfile.write('\n\n---\n\nstderr\n\n---\n')
    logfile.write(stde)
    logfile.write('\n\n---\n\n')
    if print_debug:
        try:
            print(stdo)
        except UnicodeError:
            sanitized_out = stdo.encode('ascii', errors='replace').decode()
            print(sanitized_out)
        try:
            print(stde, file=sys.stderr)
        except UnicodeError:
            sanitized_err = stde.encode('ascii', errors='replace').decode()
            print(sanitized_err, file=sys.stderr)
    if stop:
        print("Aborting..")
        for f in futures:
            f[2].cancel()
        executor.shutdown()
        raise StopException()


def bold(text):
    return mlog.bold(text).get_text(mlog.colorize_console)
{'fraction_non_alphanumeric': 0.09977146588312112, 'fraction_numerical': 0.004048318641854391, 'mean_word_length': 2.7649950835791546, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4585343', 'n_tokens_mistral': 4800, 'n_tokens_neox': 4652, 'n_words': 1348}