<commit_before>#!/usr/bin/env python
#
# Licensed to the .NET Foundation under one or more agreements.
# The .NET Foundation licenses this file to you under the MIT license.
# See the LICENSE file in the project root for more information.
#
##
# Title               :run-pmi-diffs.py
#
# Notes:
#
# TODO: Instead of downloading and extracting the dotnet CLI, can we convert
# to using init-tools.cmd/sh and the Tools/dotnetcli "last known good"
# version? (This maybe should be done for format.py as well.)
#
# Script to automate running PMI diffs on a pull request
#
##########################################################################
##########################################################################

import argparse
import distutils.dir_util
import os
import re
import shutil
import subprocess
import urllib
import sys
import tarfile
import zipfile

# Version specific imports
if sys.version_info.major < 3:
    import urllib
else:
    import urllib.request

sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "scripts"))
from coreclr_arguments import *

##########################################################################
# Globals
##########################################################################

testing = False

Coreclr_url = 'https://github.com/dotnet/coreclr.git'
Jitutils_url = 'https://github.com/dotnet/jitutils.git'

# The Docker file and possibly options should be hoisted out to a text file to be shared between scripts.

Docker_name_arm32 = 'microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420'
Docker_opts_arm32 = '-e ROOTFS_DIR=/crossrootfs/arm'

Docker_name_arm64 = 'microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921'
Docker_opts_arm64 = '-e ROOTFS_DIR=/crossrootfs/arm64'

# This should be factored out of build.sh
Unix_name_map = {
    'Linux': 'Linux',
    'Darwin': 'OSX',
    'FreeBSD': 'FreeBSD',
    'OpenBSD': 'OpenBSD',
    'NetBSD': 'NetBSD',
    'SunOS': 'SunOS'
}

Is_windows = (os.name == 'nt')
Clr_os = 'Windows_NT' if Is_windows else Unix_name_map[os.uname()[0]]

##########################################################################
# Delete protocol
##########################################################################

def del_rw(action, name, exc):
    os.chmod(name, 0o651)
    os.remove(name)

##########################################################################
# Argument Parser
##########################################################################

description = 'Tool to generate JIT assembly diffs from the CoreCLR repo'

parser = argparse.ArgumentParser(description=description)

# base_root is normally expected to be None, in which case we'll clone the
# coreclr tree and build it. If base_root is passed, we'll use it, and not
# clone or build the base.

parser.add_argument('-arch', dest='arch', default='x64')
parser.add_argument('-ci_arch', dest='ci_arch', default=None)
parser.add_argument('-build_type', dest='build_type', default='Checked')
parser.add_argument('-base_root', dest='base_root', default=None)
parser.add_argument('-diff_root', dest='diff_root', default=None)
parser.add_argument('-scratch_root', dest='scratch_root', default=None)
parser.add_argument('--skip_baseline_build', dest='skip_baseline_build', action='store_true', default=False)
parser.add_argument('--skip_diffs', dest='skip_diffs', action='store_true', default=False)
parser.add_argument('-target_branch', dest='target_branch', default='master')
parser.add_argument('-commit_hash', dest='commit_hash', default=None)

##########################################################################
# Class to change the current directory, and automatically restore the
# directory back to what it used to be, on exit.
##########################################################################

class ChangeDir:
    def __init__(self, dir):
        self.dir = dir
        self.cwd = None

    def __enter__(self):
        self.cwd = os.getcwd()
        log('[cd] %s' % self.dir)
        if not testing:
            os.chdir(self.dir)

    def __exit__(self, exc_type, exc_val, exc_tb):
        log('[cd] %s' % self.cwd)
        if not testing:
            os.chdir(self.cwd)

##########################################################################
# Helper Functions
##########################################################################

def validate_args(args):
    """ Validate all of the arguments parsed.
    Args:
        args (argparser.ArgumentParser) : Args parsed by the argument parser.
    Returns:
        args (CoreclrArguments)         : Args parsed
    Notes:
        If the arguments are valid then return them all in a tuple. If not, 
        raise an exception stating x argument is incorrect.
    """

    coreclr_setup_args = CoreclrArguments(args, 
                                          require_built_test_dir=False, 
                                          require_built_core_root=True, 
                                          require_built_product_dir=False)

    coreclr_setup_args.verify(args,
                              "base_root",
                              lambda directory: os.path.isdir(directory) if directory is not None else True,
                              "Base root is not a valid directory")

    coreclr_setup_args.verify(args,
                              "diff_root",
                              lambda directory: os.path.isdir(directory) if directory is not None else True,
                              "Base root is not a valid directory",
                              modify_arg=lambda directory: nth_dirname(os.path.abspath(sys.argv[0]), 3) if directory is None else os.path.abspath(directory))

    coreclr_setup_args.verify(args,
                              "scratch_root",
                              lambda directory: os.path.isdir(directory) if directory is not None else True,
                              "Base root is not a valid directory",
                              modify_arg=lambda directory: nth_dirname(os.path.abspath(sys.argv[0]), 3) if directory is None else os.path.abspath(directory))

    coreclr_setup_args.verify(args,
                              "skip_baseline_build",
                              lambda unused: True,
                              "Error setting baseline build")
    
    coreclr_setup_args.verify(args,
                              "skip_diffs",
                              lambda unused: True,
                              "Error setting skip_diffs")

    coreclr_setup_args.verify(args,
                              "target_branch",
                              lambda unused: True,
                              "Error setting target_branch")
<commit_msg>Fix PMI asm diffs<commit_after>#!/usr/bin/env python
#
# Licensed to the .NET Foundation under one or more agreements.
# The .NET Foundation licenses this file to you under the MIT license.
# See the LICENSE file in the project root for more information.
#
##
# Title               :run-pmi-diffs.py
#
# Notes:
#
# TODO: Instead of downloading and extracting the dotnet CLI, can we convert
# to using init-tools.cmd/sh and the Tools/dotnetcli "last known good"
# version? (This maybe should be done for format.py as well.)
#
# Script to automate running PMI diffs on a pull request
#
##########################################################################
##########################################################################

import argparse
import distutils.dir_util
import os
import re
import shutil
import subprocess
import urllib
import sys
import tarfile
import zipfile

# Version specific imports
if sys.version_info.major < 3:
    import urllib
else:
    import urllib.request

sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "scripts"))
from coreclr_arguments import *

##########################################################################
# Globals
##########################################################################

testing = False

Coreclr_url = 'https://github.com/dotnet/coreclr.git'
Jitutils_url = 'https://github.com/dotnet/jitutils.git'

# The Docker file and possibly options should be hoisted out to a text file to be shared between scripts.

Docker_name_arm32 = 'microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420'
Docker_opts_arm32 = '-e ROOTFS_DIR=/crossrootfs/arm'

Docker_name_arm64 = 'microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921'
Docker_opts_arm64 = '-e ROOTFS_DIR=/crossrootfs/arm64'

# This should be factored out of build.sh
Unix_name_map = {
    'Linux': 'Linux',
    'Darwin': 'OSX',
    'FreeBSD': 'FreeBSD',
    'OpenBSD': 'OpenBSD',
    'NetBSD': 'NetBSD',
    'SunOS': 'SunOS'
}

Is_windows = (os.name == 'nt')
Clr_os = 'Windows_NT' if Is_windows else Unix_name_map[os.uname()[0]]

##########################################################################
# Delete protocol
##########################################################################

def del_rw(action, name, exc):
    os.chmod(name, 0o651)
    os.remove(name)

##########################################################################
# Argument Parser
##########################################################################

description = 'Tool to generate JIT assembly diffs from the CoreCLR repo'

parser = argparse.ArgumentParser(description=description)

# base_root is normally expected to be None, in which case we'll clone the
# coreclr tree and build it. If base_root is passed, we'll use it, and not
# clone or build the base.

parser.add_argument('-arch', dest='arch', default='x64')
parser.add_argument('-ci_arch', dest='ci_arch', default=None)
parser.add_argument('-build_type', dest='build_type', default='Checked')
parser.add_argument('-base_root', dest='base_root', default=None)
parser.add_argument('-diff_root', dest='diff_root', default=None)
parser.add_argument('-scratch_root', dest='scratch_root', default=None)
parser.add_argument('--skip_baseline_build', dest='skip_baseline_build', action='store_true', default=False)
parser.add_argument('--skip_diffs', dest='skip_diffs', action='store_true', default=False)
parser.add_argument('-target_branch', dest='target_branch', default='master')
parser.add_argument('-commit_hash', dest='commit_hash', default=None)

##########################################################################
# Class to change the current directory, and automatically restore the
# directory back to what it used to be, on exit.
##########################################################################

class ChangeDir:
    def __init__(self, dir):
        self.dir = dir
        self.cwd = None

    def __enter__(self):
        self.cwd = os.getcwd()
        log('[cd] %s' % self.dir)
        if not testing:
            os.chdir(self.dir)

    def __exit__(self, exc_type, exc_val, exc_tb):
        log('[cd] %s' % self.cwd)
        if not testing:
            os.chdir(self.cwd)

##########################################################################
# Helper Functions
##########################################################################

def validate_args(args):
    """ Validate all of the arguments parsed.
    Args:
        args (argparser.ArgumentParser) : Args parsed by the argument parser.
    Returns:
        args (CoreclrArguments)         : Args parsed
    Notes:
        If the arguments are valid then return them all in a tuple. If not, 
        raise an exception stating x argument is incorrect.
    """

    coreclr_setup_args = CoreclrArguments(args, 
                                          require_built_test_dir=False, 
                                          require_built_core_root=True, 
                                          require_built_product_dir=False)

    coreclr_setup_args.verify(args,
                              "base_root",
                              lambda directory: os.path.isdir(directory) if directory is not None else True,
                              "Base root is not a valid directory")

    coreclr_setup_args.verify(args,
                              "diff_root",
                              lambda directory: os.path.isdir(directory) if directory is not None else True,
                              "Diff root is not a valid directory",
                              modify_arg=lambda directory: nth_dirname(os.path.abspath(sys.argv[0]), 3) if directory is None else os.path.abspath(directory))

    coreclr_setup_args.verify(args,
                              "scratch_root",
                              lambda unused: True,
                              "Error setting scratch_root",
                              modify_arg=lambda directory: os.path.join(coreclr_setup_args.diff_root, '_', 'pmi') if directory is None else os.path.abspath(directory))

    coreclr_setup_args.verify(args,
                              "skip_baseline_build",
                              lambda unused: True,
                              "Error setting baseline build")
    
    coreclr_setup_args.verify(args,
                              "skip_diffs",
                              lambda unused: True,
                              "Error setting skip_diffs")

    coreclr_setup_args.verify(args,
                              "target_branch",
                              lambda unused: True,
                              "Error setting target_branch")

{'fraction_non_alphanumeric': 0.216070372560615, 'fraction_numerical': 0.010053222945002957, 'mean_word_length': 3.043335325762104, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1661212', 'n_tokens_mistral': 3750, 'n_tokens_neox': 3552, 'n_words': 940}