<commit_before>#!/usr/bin/env python3

# Copyright 2018 The GraphicsFuzz Project Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import filecmp
import os
import shutil
import struct
import subprocess
import time
import platform
import json
import sys
from typing import Any, IO, Optional, Union, List

################################################################################
# Help messages for options that are duplicated elsewhere

FORCE_OPTION_HELP = (
    'Do not wait for the device\'s screen to be on; just continue.  '
    'Only allowed if \'target\' is \'android\'. '
)

LEGACY_OPTION_HELP = 'Render using legacy Vulkan worker.'

SERIAL_OPTION_HELP = (
    'Android device serial number. Only allowed if \'target\' is \'android\'.  '
    'Run "adb devices -l" to list available serial numbers.  '
    'A serial number need not be provided if only one device is connected.  '
    'The serial number will have the form "IP:port" if using adb over TCP.  '
    'See: https://developer.android.com/studio/command-line/adb ')

SKIP_RENDER_OPTION_HELP = 'Compile shaders but do not actually run them.'

TARGET_HELP = 'One of \'host\' (run on host machine) or \'android\' (run on Android device).'

################################################################################
# Constants

HERE = os.path.abspath(__file__)

LOGFILE_NAME = 'vklog.txt'
SSBO_JSON_FILENAME = 'ssbo.json'
TIMEOUT_RUN = 30
NUM_RENDER = 3
BUSY_WAIT_SLEEP_SLOW = 1.0
BUSY_WAIT_SLEEP_FAST = 0.1
AMBER_FENCE_TIMEOUT_MS = 60000

################################################################################
# Common

orig_print = print


# noinspection PyShadowingBuiltins
def print(s):
    orig_print(s, flush=True)


def convert_and_print_stdout_stderr_returncode(result, verbose=False):

    print('RETURNCODE:' + str(result.returncode))

    if result.stdout is not None:
        result.stdout = result.stdout.decode(encoding='utf-8', errors='ignore')
        if verbose:
            print('STDOUT:')
            print(result.stdout)
            print('')
    else:
        if verbose:
            print('STDOUT not captured.')

    if result.stderr is not None:
        result.stderr = result.stderr.decode(encoding='utf-8', errors='ignore')
        if verbose:
            print('STDERR:')
            print(result.stderr)
            print('')
    else:
        if verbose:
            print('STDERR not captured.')


def subprocess_helper(
    cmd: List[str],
    check=True,
    timeout=None,
    stdout: Union[None, int, IO[Any]]=subprocess.PIPE,
    stderr: Union[None, int, IO[Any]]=subprocess.PIPE,
    verbose=False
):

    # We capture stdout and stderr by default so we have something to report if the command fails.

    # Note: "encoding=" and "errors=" are Python 3.6.
    # We manually decode to utf-8 instead of using "universal_newlines=" so we have full control.
    # text= is Python 3.6.
    # Do not use shell=True.

    try:
        print('Exec: ' + str(cmd))

        result = subprocess.run(
            cmd,
            check=check,
            timeout=timeout,
            stdout=stdout,
            stderr=stderr
        )
    except subprocess.CalledProcessError as ex:
        convert_and_print_stdout_stderr_returncode(ex, verbose=verbose)
        raise ex

    convert_and_print_stdout_stderr_returncode(result, verbose=verbose)

    return result


def open_helper(file: str, mode: str):
    return open(file, mode, encoding='utf-8', errors='ignore')


def get_platform():
    host = platform.system()
    if host == 'Linux' or host == 'Windows':
        return host
    elif host == 'Darwin':
        return 'Mac'
    else:
        raise AssertionError('Unsupported platform: {}'.format(host))


def get_bin_dir():
    # graphics-fuzz/python/drivers/
    bin_dir = os.path.dirname(HERE)
    # graphics-fuzz/
    bin_dir = os.path.join(bin_dir, os.path.pardir, os.path.pardir)
    # e.g. graphics-fuzz/bin/Linux
    bin_dir = os.path.join(bin_dir, 'bin', get_platform())
    return bin_dir


BIN_DIR = get_bin_dir()


# A distinguished exception for the case where a tool is not on the path.
class ToolNotOnPathError(Exception):
    pass


def tool_on_path(tool: str) -> str:
    result = shutil.which(tool)
    if result is None:
        raise ToolNotOnPathError('Could not find {} on PATH. Please add to PATH.'.format(tool))
    return result


def glslang_path():
    glslang = os.path.join(BIN_DIR, 'glslangValidator')
    if os.path.isfile(glslang):
        return glslang
    return tool_on_path('glslangValidator')


def spirvas_path():
    spirvas = os.path.join(BIN_DIR, 'spirv-as')
    if os.path.isfile(spirvas):
        return spirvas
    return tool_on_path('spirv-as')


def spirvdis_path():
    spirvas = os.path.join(BIN_DIR, 'spirv-dis')
    if os.path.isfile(spirvas):
        return spirvas
    return tool_on_path('spirv-dis')


def maybe_add_catchsegv(cmd: List[str]) -> None:
    # Add catchsegv to cmd if it is available
    try:
        cmd.append(tool_on_path('catchsegv'))
    except ToolNotOnPathError:
        # Didn't find catchsegv on path; that's OK
        pass


def adb_path():
    if 'ANDROID_HOME' in os.environ:
        adb = os.path.join(os.environ['ANDROID_HOME'], 'platform-tools', 'adb')
        if os.path.isfile(adb):
            return adb
    tool_on_path('adb')


def remove_end(str_in: str, str_end: str):
    assert str_in.endswith(str_end), 'Expected {} to end with {}'.format(str_in, str_end)
    return str_in[:-len(str_end)]


def prepare_shader(output_dir: str, shader: Optional[str]):
    """
    Translates a shader to binary SPIR-V.

    None -> None
    shader.frag -> shader.frag.spv
    shader.vert -> shader.vert.spv
    shader.comp -> shader.comp.spv
    shader.frag.asm -> shader.frag.spv
    shader.vert.asm -> shader.vert.spv
    shader.comp.asm -> shader.comp.spv

    :param output_dir: output directory
    :param shader: e.g. shader.frag, shader.vert, shader.frag.asm
    :return: the resulting .spv file<commit_msg> runspv: Fix bug in adb finder code and improve Windows support (#335)<commit_after>#!/usr/bin/env python3

# Copyright 2018 The GraphicsFuzz Project Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import filecmp
import os
import shutil
import struct
import subprocess
import time
import platform
import json
import sys
from typing import Any, IO, Optional, Union, List

################################################################################
# Help messages for options that are duplicated elsewhere

FORCE_OPTION_HELP = (
    'Do not wait for the device\'s screen to be on; just continue.  '
    'Only allowed if \'target\' is \'android\'. '
)

LEGACY_OPTION_HELP = 'Render using legacy Vulkan worker.'

SERIAL_OPTION_HELP = (
    'Android device serial number. Only allowed if \'target\' is \'android\'.  '
    'Run "adb devices -l" to list available serial numbers.  '
    'A serial number need not be provided if only one device is connected.  '
    'The serial number will have the form "IP:port" if using adb over TCP.  '
    'See: https://developer.android.com/studio/command-line/adb ')

SKIP_RENDER_OPTION_HELP = 'Compile shaders but do not actually run them.'

TARGET_HELP = 'One of \'host\' (run on host machine) or \'android\' (run on Android device).'

################################################################################
# Constants

HERE = os.path.abspath(__file__)

LOGFILE_NAME = 'vklog.txt'
SSBO_JSON_FILENAME = 'ssbo.json'
TIMEOUT_RUN = 30
NUM_RENDER = 3
BUSY_WAIT_SLEEP_SLOW = 1.0
BUSY_WAIT_SLEEP_FAST = 0.1
AMBER_FENCE_TIMEOUT_MS = 60000

################################################################################
# Common

orig_print = print


# noinspection PyShadowingBuiltins
def print(s):
    orig_print(s, flush=True)


def convert_and_print_stdout_stderr_returncode(result, verbose=False):

    print('RETURNCODE:' + str(result.returncode))

    if result.stdout is not None:
        result.stdout = result.stdout.decode(encoding='utf-8', errors='ignore')
        if verbose:
            print('STDOUT:')
            print(result.stdout)
            print('')
    else:
        if verbose:
            print('STDOUT not captured.')

    if result.stderr is not None:
        result.stderr = result.stderr.decode(encoding='utf-8', errors='ignore')
        if verbose:
            print('STDERR:')
            print(result.stderr)
            print('')
    else:
        if verbose:
            print('STDERR not captured.')


def subprocess_helper(
    cmd: List[str],
    check=True,
    timeout=None,
    stdout: Union[None, int, IO[Any]]=subprocess.PIPE,
    stderr: Union[None, int, IO[Any]]=subprocess.PIPE,
    verbose=False
):

    assert cmd[0] is not None and isinstance(cmd[0], str)

    # We capture stdout and stderr by default so we have something to report if the command fails.

    # Note: "encoding=" and "errors=" are Python 3.6.
    # We manually decode to utf-8 instead of using "universal_newlines=" so we have full control.
    # text= is Python 3.6.
    # Do not use shell=True.

    try:
        print('Exec: ' + str(cmd))

        result = subprocess.run(
            cmd,
            check=check,
            timeout=timeout,
            stdout=stdout,
            stderr=stderr
        )
    except subprocess.CalledProcessError as ex:
        convert_and_print_stdout_stderr_returncode(ex, verbose=verbose)
        raise ex

    convert_and_print_stdout_stderr_returncode(result, verbose=verbose)

    return result


def open_helper(file: str, mode: str):
    return open(file, mode, encoding='utf-8', errors='ignore')


def get_platform():
    host = platform.system()
    if host == 'Linux' or host == 'Windows':
        return host
    elif host == 'Darwin':
        return 'Mac'
    else:
        raise AssertionError('Unsupported platform: {}'.format(host))


def get_bin_dir():
    # graphics-fuzz/python/drivers/
    bin_dir = os.path.dirname(HERE)
    # graphics-fuzz/
    bin_dir = os.path.join(bin_dir, os.path.pardir, os.path.pardir)
    # e.g. graphics-fuzz/bin/Linux
    bin_dir = os.path.join(bin_dir, 'bin', get_platform())
    return bin_dir


BIN_DIR = get_bin_dir()


# A distinguished exception for the case where a tool is not on the path.
class ToolNotOnPathError(Exception):
    pass


def tool_on_path(tool: str) -> str:
    result = shutil.which(tool)
    if result is None:
        raise ToolNotOnPathError('Could not find {} on PATH. Please add to PATH.'.format(tool))
    return result


def glslang_path():
    glslang = os.path.join(BIN_DIR, 'glslangValidator')
    if os.path.isfile(glslang):
        return glslang
    return tool_on_path('glslangValidator')


def spirvas_path():
    spirvas = os.path.join(BIN_DIR, 'spirv-as')
    if os.path.isfile(spirvas):
        return spirvas
    return tool_on_path('spirv-as')


def spirvdis_path():
    spirvas = os.path.join(BIN_DIR, 'spirv-dis')
    if os.path.isfile(spirvas):
        return spirvas
    return tool_on_path('spirv-dis')


def maybe_add_catchsegv(cmd: List[str]) -> None:
    # Add catchsegv to cmd if it is available
    try:
        cmd.append(tool_on_path('catchsegv'))
    except ToolNotOnPathError:
        # Didn't find catchsegv on path; that's OK
        pass


def adb_path():
    if 'ANDROID_HOME' in os.environ:
        platform_tools_path = os.path.join(os.environ['ANDROID_HOME'], 'platform-tools')
        adb = shutil.which('adb', path=platform_tools_path)
        if adb is not None:
            return adb
    return tool_on_path('adb')


def remove_end(str_in: str, str_end: str):
    assert str_in.endswith(str_end), 'Expected {} to end with {}'.format(str_in, str_end)
    return str_in[:-len(str_end)]


def prepare_shader(output_dir: str, shader: Optional[str]):
    """
    Translates a shader to binary SPIR-V.

    None -> None
    shader.frag -> shader.frag.spv
    shader.vert -> shader.vert.spv
    shader.comp -> shader.comp.spv
    shader.frag.asm -> shader.frag.spv
    shader.vert.asm -> shader.vert.spv
    shader.comp.asm -> shader.comp.spv

    :param output_dir: output directory
    :param shader: e.g. shader.frag, shader.vert, shader.frag.asm
    :return: the resulting .spv file
{'fraction_non_alphanumeric': 0.13899788711137942, 'fraction_numerical': 0.004753999396317537, 'mean_word_length': 3.207301587301587, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5988356', 'n_tokens_mistral': 4269, 'n_tokens_neox': 4080, 'n_words': 1201}