<commit_before># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
From a system-installed copy of the toolchain, packages all the required bits
into a .zip file.

It assumes default install locations for tools, in particular:
- C:\Program Files (x86)\Microsoft Visual Studio 12.0\...
- C:\Program Files (x86)\Windows Kits\10\...

1. Start from a fresh Win7 VM image.
2. Install VS Pro. Deselect everything except MFC.
3. Install Windows 10 SDK. Select only the Windows SDK and Debugging Tools for
Windows.
4. Run this script, which will build a <sha1>.zip.

Express is not yet supported by this script, but patches welcome (it's not too
useful as the resulting zip can't be redistributed, and most will presumably
have a Pro license anyway).
"""

import collections
import glob
import json
import optparse
import os
import platform
import shutil
import subprocess
import sys
import tempfile
import zipfile

import get_toolchain_if_necessary


VS_VERSION = None
WIN_VERSION = None
VC_TOOLS = None


def GetVSPath():
  if VS_VERSION == '2015':
    return r'C:\Program Files (x86)\Microsoft Visual Studio 14.0'
  elif VS_VERSION == '2017':
    # Use vswhere to find the VS 2017 installation. This handles preview
    # versions automatically. This assumes that only one version is installed.
    command = (r'C:\Program Files (x86)\Microsoft Visual Studio\Installer'
               r'\vswhere.exe')
    marker = 'installationPath: '
    for line in subprocess.check_output(command).splitlines():
      if line.startswith(marker):
        return line[len(marker):]
    raise Exception('VS 2017 path not found in vswhere output')
  else:
    raise ValueError(VS_VERSION)


def ExpandWildcards(root, sub_dir):
  # normpath is needed to change '/' to '\\' characters.
  path = os.path.normpath(os.path.join(root, sub_dir))
  matches = glob.glob(path)
  if len(matches) != 1:
    raise Exception('%s had %d matches - should be one' % (path, len(matches)))
  return matches[0]


def BuildFileList(override_dir):
  result = []

  # Subset of VS corresponding roughly to VC.
  paths = [
      'DIA SDK/bin',
      'DIA SDK/idl',
      'DIA SDK/include',
      'DIA SDK/lib',
      VC_TOOLS + '/atlmfc',
      VC_TOOLS + '/crt',
      'VC/redist',
  ]

  if override_dir:
    paths += [
        (os.path.join(override_dir, 'bin'), VC_TOOLS + '/bin'),
        (os.path.join(override_dir, 'include'), VC_TOOLS + '/include'),
        (os.path.join(override_dir, 'lib'), VC_TOOLS + '/lib'),
    ]
  else:
    paths += [
        VC_TOOLS + '/bin',
        VC_TOOLS + '/include',
        VC_TOOLS + '/lib',
    ]

  if VS_VERSION == '2015':
    paths += [
        ('VC/redist/x86/Microsoft.VC140.CRT', 'sys32'),
        ('VC/redist/x86/Microsoft.VC140.CRT', 'win_sdk/bin/x86'),
        ('VC/redist/x86/Microsoft.VC140.MFC', 'sys32'),
        ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugCRT', 'sys32'),
        ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugMFC', 'sys32'),
        ('VC/redist/x64/Microsoft.VC140.CRT', 'sys64'),
        ('VC/redist/x64/Microsoft.VC140.CRT', 'VC/bin/amd64_x86'),
        ('VC/redist/x64/Microsoft.VC140.CRT', 'VC/bin/amd64'),
        ('VC/redist/x64/Microsoft.VC140.CRT', 'win_sdk/bin/x64'),
        ('VC/redist/x64/Microsoft.VC140.MFC', 'sys64'),
        ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugCRT', 'sys64'),
        ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugMFC', 'sys64'),
    ]
  elif VS_VERSION == '2017':
    paths += [
        ('VC/redist/MSVC/14.*.*/x86/Microsoft.VC150.CRT', 'sys32'),
        ('VC/redist/MSVC/14.*.*/x86/Microsoft.VC150.CRT', 'win_sdk/bin/x86'),
        ('VC/redist/MSVC/14.*.*/x86/Microsoft.VC150.MFC', 'sys32'),
        ('VC/redist/MSVC/14.*.*/debug_nonredist/x86/Microsoft.VC150.DebugCRT', 'sys32'),
        ('VC/redist/MSVC/14.*.*/debug_nonredist/x86/Microsoft.VC150.DebugMFC', 'sys32'),
        ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC150.CRT', 'sys64'),
        ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC150.CRT', 'VC/bin/amd64_x86'),
        ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC150.CRT', 'VC/bin/amd64'),
        ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC150.CRT', 'win_sdk/bin/x64'),
        ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC150.MFC', 'sys64'),
        ('VC/redist/MSVC/14.*.*/debug_nonredist/x64/Microsoft.VC150.DebugCRT', 'sys64'),
        ('VC/redist/MSVC/14.*.*/debug_nonredist/x64/Microsoft.VC150.DebugMFC', 'sys64'),
    ]
  else:
    raise ValueError('VS_VERSION %s' % VS_VERSION)

  vs_path = GetVSPath()

  for path in paths:
    src = path[0] if isinstance(path, tuple) else path
    # Note that vs_path is ignored if src is an absolute path.
    combined = ExpandWildcards(vs_path, src)<commit_msg>Fix redist packaging for VS 2017 Update 3 Preview 2<commit_after># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
From a system-installed copy of the toolchain, packages all the required bits
into a .zip file.

It assumes default install locations for tools, in particular:
- C:\Program Files (x86)\Microsoft Visual Studio 12.0\...
- C:\Program Files (x86)\Windows Kits\10\...

1. Start from a fresh Win7 VM image.
2. Install VS Pro. Deselect everything except MFC.
3. Install Windows 10 SDK. Select only the Windows SDK and Debugging Tools for
Windows.
4. Run this script, which will build a <sha1>.zip.

Express is not yet supported by this script, but patches welcome (it's not too
useful as the resulting zip can't be redistributed, and most will presumably
have a Pro license anyway).
"""

import collections
import glob
import json
import optparse
import os
import platform
import shutil
import subprocess
import sys
import tempfile
import zipfile

import get_toolchain_if_necessary


VS_VERSION = None
WIN_VERSION = None
VC_TOOLS = None


def GetVSPath():
  if VS_VERSION == '2015':
    return r'C:\Program Files (x86)\Microsoft Visual Studio 14.0'
  elif VS_VERSION == '2017':
    # Use vswhere to find the VS 2017 installation. This handles preview
    # versions automatically. This assumes that only one version is installed.
    command = (r'C:\Program Files (x86)\Microsoft Visual Studio\Installer'
               r'\vswhere.exe')
    marker = 'installationPath: '
    for line in subprocess.check_output(command).splitlines():
      if line.startswith(marker):
        return line[len(marker):]
    raise Exception('VS 2017 path not found in vswhere output')
  else:
    raise ValueError(VS_VERSION)


def ExpandWildcards(root, sub_dir):
  # normpath is needed to change '/' to '\\' characters.
  path = os.path.normpath(os.path.join(root, sub_dir))
  matches = glob.glob(path)
  if len(matches) != 1:
    raise Exception('%s had %d matches - should be one' % (path, len(matches)))
  return matches[0]


def BuildFileList(override_dir):
  result = []

  # Subset of VS corresponding roughly to VC.
  paths = [
      'DIA SDK/bin',
      'DIA SDK/idl',
      'DIA SDK/include',
      'DIA SDK/lib',
      VC_TOOLS + '/atlmfc',
      VC_TOOLS + '/crt',
      'VC/redist',
  ]

  if override_dir:
    paths += [
        (os.path.join(override_dir, 'bin'), VC_TOOLS + '/bin'),
        (os.path.join(override_dir, 'include'), VC_TOOLS + '/include'),
        (os.path.join(override_dir, 'lib'), VC_TOOLS + '/lib'),
    ]
  else:
    paths += [
        VC_TOOLS + '/bin',
        VC_TOOLS + '/include',
        VC_TOOLS + '/lib',
    ]

  if VS_VERSION == '2015':
    paths += [
        ('VC/redist/x86/Microsoft.VC140.CRT', 'sys32'),
        ('VC/redist/x86/Microsoft.VC140.CRT', 'win_sdk/bin/x86'),
        ('VC/redist/x86/Microsoft.VC140.MFC', 'sys32'),
        ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugCRT', 'sys32'),
        ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugMFC', 'sys32'),
        ('VC/redist/x64/Microsoft.VC140.CRT', 'sys64'),
        ('VC/redist/x64/Microsoft.VC140.CRT', 'VC/bin/amd64_x86'),
        ('VC/redist/x64/Microsoft.VC140.CRT', 'VC/bin/amd64'),
        ('VC/redist/x64/Microsoft.VC140.CRT', 'win_sdk/bin/x64'),
        ('VC/redist/x64/Microsoft.VC140.MFC', 'sys64'),
        ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugCRT', 'sys64'),
        ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugMFC', 'sys64'),
    ]
  elif VS_VERSION == '2017':
    paths += [
        ('VC/redist/MSVC/14.*.*/x86/Microsoft.VC*.CRT', 'sys32'),
        ('VC/redist/MSVC/14.*.*/x86/Microsoft.VC*.CRT', 'win_sdk/bin/x86'),
        ('VC/redist/MSVC/14.*.*/debug_nonredist/x86/Microsoft.VC*.DebugCRT', 'sys32'),
        ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', 'sys64'),
        ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', 'VC/bin/amd64_x86'),
        ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', 'VC/bin/amd64'),
        ('VC/redist/MSVC/14.*.*/x64/Microsoft.VC*.CRT', 'win_sdk/bin/x64'),
        ('VC/redist/MSVC/14.*.*/debug_nonredist/x64/Microsoft.VC*.DebugCRT', 'sys64'),
    ]
  else:
    raise ValueError('VS_VERSION %s' % VS_VERSION)

  vs_path = GetVSPath()

  for path in paths:
    src = path[0] if isinstance(path, tuple) else path
    # Note that vs_path is ignored if src is an absolute path.
    combined = ExpandWildcards(vs_path, src)
{'fraction_non_alphanumeric': 0.1514114627887083, 'fraction_numerical': 0.04790419161676647, 'mean_word_length': 3.1021929824561405, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '962437', 'n_tokens_mistral': 3597, 'n_tokens_neox': 3221, 'n_words': 788}