<commit_before>#!/usr/bin/env python

"""Ninja toolchain abstraction for Microsoft compiler suite"""

import os
import subprocess

import toolchain

class MSVCToolchain(toolchain.Toolchain):

  def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables, subninja):
    #Local variable defaults
    self.sdkpath = ''
    self.toolchain = ''
    self.includepaths = []
    self.libpaths = libpaths
    self.ccompiler = 'cl'
    self.cxxcompiler = 'cl'
    self.archiver = 'lib'
    self.linker = 'link'
    self.dller = 'dll'

    #Command definitions
    self.cccmd = '$toolchain$cc /showIncludes /I. $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags /c $in /Fo$out /Fd$pdbpath /FS /nologo'
    self.cxxcmd = '$toolchain$cxx /showIncludes /I. $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags $cmoreflags /c $in /Fo$out /Fd$pdbpath /FS /nologo'
    self.ccdepfile = None
    self.ccdeps = 'msvc'
    self.arcmd = '$toolchain$ar $arflags $ararchflags $arconfigflags /NOLOGO /OUT:$out $in'
    self.linkcmd = '$toolchain$link $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags /DEBUG /NOLOGO /SUBSYSTEM:CONSOLE /DYNAMICBASE /NXCOMPAT /MANIFEST /MANIFESTUAC:\"level=\'asInvoker\' uiAccess=\'false\'\" /TLBID:1 /PDB:$pdbpath /OUT:$out $in $libs $archlibs $oslibs'
    self.dllcmd = self.linkcmd + ' /DLL'

    self.cflags = ['/D', '"' + project.upper() + '_COMPILE=1"', '/D', '"_UNICODE"',  '/D', '"UNICODE"', '/Zi', '/Oi', '/Oy-', '/GS-', '/Gy-', '/Qpar-', '/fp:fast', '/fp:except-', '/Zc:forScope', '/Zc:wchar_t', '/GR-', '/openmp-']
    self.cwarnflags = ['/W4', '/WX']
    self.cmoreflags = []
    self.arflags = ['/ignore:4221'] #Ignore empty object file warning]
    self.linkflags = ['/DEBUG']
    self.oslibs = ['kernel32', 'user32', 'shell32', 'advapi32']

    self.initialize_subninja(subninja)
    self.initialize_archs(archs)
    self.initialize_configs(configs)
    self.initialize_project(project)
    self.initialize_toolchain()
    self.initialize_depends(dependlibs)

    self.parse_default_variables(variables)
    self.read_build_prefs()

    self.includepaths = self.prefix_includepaths((includepaths or []) + ['.'])

    if self.is_monolithic():
      self.cflags += ['/D', '"BUILD_MONOLITHIC=1"']

    if not 'nowarning' in variables or not variables['nowarning']:
      self.cflags += self.cwarnflags
    self.cxxflags = list(self.cflags)

    #Overrides
    self.objext = '.obj'

    #Builders
    self.builders['c'] = self.builder_cc
    self.builders['cc'] = self.builder_cxx
    self.builders['cpp'] = self.builder_cxx
    self.builders['lib'] = self.builder_lib
    self.builders['multilib'] = self.builder_multicopy
    self.builders['sharedlib'] = self.builder_sharedlib
    self.builders['multisharedlib'] = self.builder_multicopy
    self.builders['bin'] = self.builder_bin
    self.builders['multibin'] = self.builder_multicopy

    #Setup toolchain
    self.build_toolchain()

  def name(self):
    return 'msvc'

  def parse_prefs(self, prefs):
    super(MSVCToolchain, self).parse_prefs(prefs)
    if 'msvc' in prefs:
      msvcprefs = prefs['msvc']
      if 'sdkpath' in msvcprefs:
        self.sdkpath = msvcprefs['sdkpath']
      if 'toolchain' in msvcprefs:
        self.toolchain = msvcprefs['toolchain']
      if 'toolchain_version' in msvcprefs:
        self.toolchain_version = msvcprefs['toolchain_version']

  def write_variables(self, writer):
    super(MSVCToolchain, self).write_variables(writer)
    writer.variable('cc', self.ccompiler)
    writer.variable('cxx', self.cxxcompiler)
    writer.variable('ar', self.archiver)
    writer.variable('link', self.linker)
    writer.variable('dll', self.dller)
    writer.variable('toolchain', self.toolchain)
    writer.variable('includepaths', self.make_includepaths(self.includepaths))
    writer.variable('moreincludepaths', '')
    writer.variable('pdbpath', 'ninja.pdb')
    writer.variable('cflags', self.cflags)
    writer.variable('carchflags', '')
    writer.variable('cconfigflags', '')
    writer.variable('cxxflags', self.cxxflags)
    writer.variable('cmoreflags', self.cmoreflags)
    writer.variable('arflags', self.arflags)
    writer.variable('ararchflags', '')
    writer.variable('arconfigflags', '')
    writer.variable('linkflags', self.linkflags)
    writer.variable('linkarchflags', '')
    writer.variable('linkconfigflags', '')
    writer.variable('libs', '')
    writer.variable('libpaths', self.make_libpaths(self.libpaths))
    writer.variable('configlibpaths', '')
    writer.variable('archlibs', '')
    writer.variable('oslibs', self.make_libs(self.oslibs))
    writer.newline()

  def write_rules(self, writer):
    super(MSVCToolchain, self).write_rules(writer)
    writer.rule('cc', command = self.cccmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CC $in')
    writer.rule('cxx', command = self.cxxcmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CXX $in')
    writer.rule('ar', command = self.arcmd, description = 'LIB $out')
    writer.rule('link', command = self.linkcmd, description = 'LINK $out')
    writer.rule('dll', command = self.dllcmd, description = 'DLL $out')
    writer.newline()

  def build_toolchain(self):
    if self.toolchain == '':
      versions = ['16.0', '15.0', '14.0', '13.0', '12.0', '11.0', '10.0']
      keys = [
        'HKLM\\SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VC7',
        'HKCU\\SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VC7',
        'HKLM\\SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VS7',
        'HKCU\\SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VS7',
        'HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\VisualStudio\\SxS\\VC7',
        'HKCU\\SOFTWARE\\Wow6432Node\\Microsoft\\VisualStudio\\SxS\\VC7',
        'HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\VisualStudio\\SxS\\VS7',
        'HKCU\\SOFTWARE\\Wow6432Node\\Microsoft\\VisualStudio\\SxS\\VS7'
      ]
      toolchain = ''
      for version in versions:
        for key in keys:
          try:
            query = subprocess.check_output(['reg', 'query', key, '/v', version ], stderr = subprocess.STDOUT).strip().splitlines()
            if len(query) == 2:
              toolchain = str(query[1]).split('REG_SZ')[-1].strip(" '\"\n\r\t")
          except:
            continue
          if not toolchain == '':
            #Thanks MS for making it _really_ hard to find the compiler
            if version == '15.0' or version == '16.0':
              tools_basepath = os.path.join(toolchain, 'VC', 'Tools', 'MSVC')
              tools_list = [item for item in os.listdir(tools_basepath) if os.path.isdir(os.path.join(tools_basepath, item))]
              from distutils.version import StrictVersion
              tools_list.sort(key=StrictVersion)
              toolchain = os.path.join(tools_basepath, tools_list[-1])
            self.includepaths += [os.path.join(toolchain, 'include')]
            self.toolchain = toolchain
            self.toolchain_version = version
            break
        if not toolchain == '':
          break
    if self.sdkpath == '':
      versions = ['v10.0', 'v8.1']
      keys = [
        'HKLM\\SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows',
        'HKCU\\SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows',
        'HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Microsoft SDKs\\Windows',
        'HKCU\\SOFTWARE\\Wow6432Node\\Microsoft\\Microsoft SDKs\\Windows'<commit_msg>update includes with predef toolchain path<commit_after>#!/usr/bin/env python

"""Ninja toolchain abstraction for Microsoft compiler suite"""

import os
import subprocess

import toolchain

class MSVCToolchain(toolchain.Toolchain):

  def initialize(self, project, archs, configs, includepaths, dependlibs, libpaths, variables, subninja):
    #Local variable defaults
    self.sdkpath = ''
    self.toolchain = ''
    self.includepaths = []
    self.libpaths = libpaths
    self.ccompiler = 'cl'
    self.cxxcompiler = 'cl'
    self.archiver = 'lib'
    self.linker = 'link'
    self.dller = 'dll'

    #Command definitions
    self.cccmd = '$toolchain$cc /showIncludes /I. $includepaths $moreincludepaths $cflags $carchflags $cconfigflags $cmoreflags /c $in /Fo$out /Fd$pdbpath /FS /nologo'
    self.cxxcmd = '$toolchain$cxx /showIncludes /I. $includepaths $moreincludepaths $cxxflags $carchflags $cconfigflags $cmoreflags /c $in /Fo$out /Fd$pdbpath /FS /nologo'
    self.ccdepfile = None
    self.ccdeps = 'msvc'
    self.arcmd = '$toolchain$ar $arflags $ararchflags $arconfigflags /NOLOGO /OUT:$out $in'
    self.linkcmd = '$toolchain$link $libpaths $configlibpaths $linkflags $linkarchflags $linkconfigflags /DEBUG /NOLOGO /SUBSYSTEM:CONSOLE /DYNAMICBASE /NXCOMPAT /MANIFEST /MANIFESTUAC:\"level=\'asInvoker\' uiAccess=\'false\'\" /TLBID:1 /PDB:$pdbpath /OUT:$out $in $libs $archlibs $oslibs'
    self.dllcmd = self.linkcmd + ' /DLL'

    self.cflags = ['/D', '"' + project.upper() + '_COMPILE=1"', '/D', '"_UNICODE"',  '/D', '"UNICODE"', '/Zi', '/Oi', '/Oy-', '/GS-', '/Gy-', '/Qpar-', '/fp:fast', '/fp:except-', '/Zc:forScope', '/Zc:wchar_t', '/GR-', '/openmp-']
    self.cwarnflags = ['/W4', '/WX']
    self.cmoreflags = []
    self.arflags = ['/ignore:4221'] #Ignore empty object file warning]
    self.linkflags = ['/DEBUG']
    self.oslibs = ['kernel32', 'user32', 'shell32', 'advapi32']

    self.initialize_subninja(subninja)
    self.initialize_archs(archs)
    self.initialize_configs(configs)
    self.initialize_project(project)
    self.initialize_toolchain()
    self.initialize_depends(dependlibs)

    self.parse_default_variables(variables)
    self.read_build_prefs()

    self.includepaths = self.prefix_includepaths((includepaths or []) + ['.'])

    if self.is_monolithic():
      self.cflags += ['/D', '"BUILD_MONOLITHIC=1"']

    if not 'nowarning' in variables or not variables['nowarning']:
      self.cflags += self.cwarnflags
    self.cxxflags = list(self.cflags)

    #Overrides
    self.objext = '.obj'

    #Builders
    self.builders['c'] = self.builder_cc
    self.builders['cc'] = self.builder_cxx
    self.builders['cpp'] = self.builder_cxx
    self.builders['lib'] = self.builder_lib
    self.builders['multilib'] = self.builder_multicopy
    self.builders['sharedlib'] = self.builder_sharedlib
    self.builders['multisharedlib'] = self.builder_multicopy
    self.builders['bin'] = self.builder_bin
    self.builders['multibin'] = self.builder_multicopy

    #Setup toolchain
    self.build_toolchain()

  def name(self):
    return 'msvc'

  def parse_prefs(self, prefs):
    super(MSVCToolchain, self).parse_prefs(prefs)
    if 'msvc' in prefs:
      msvcprefs = prefs['msvc']
      if 'sdkpath' in msvcprefs:
        self.sdkpath = msvcprefs['sdkpath']
      if 'toolchain' in msvcprefs:
        self.toolchain = msvcprefs['toolchain']
      if 'toolchain_version' in msvcprefs:
        self.toolchain_version = msvcprefs['toolchain_version']

  def write_variables(self, writer):
    super(MSVCToolchain, self).write_variables(writer)
    writer.variable('cc', self.ccompiler)
    writer.variable('cxx', self.cxxcompiler)
    writer.variable('ar', self.archiver)
    writer.variable('link', self.linker)
    writer.variable('dll', self.dller)
    writer.variable('toolchain', self.toolchain)
    writer.variable('includepaths', self.make_includepaths(self.includepaths))
    writer.variable('moreincludepaths', '')
    writer.variable('pdbpath', 'ninja.pdb')
    writer.variable('cflags', self.cflags)
    writer.variable('carchflags', '')
    writer.variable('cconfigflags', '')
    writer.variable('cxxflags', self.cxxflags)
    writer.variable('cmoreflags', self.cmoreflags)
    writer.variable('arflags', self.arflags)
    writer.variable('ararchflags', '')
    writer.variable('arconfigflags', '')
    writer.variable('linkflags', self.linkflags)
    writer.variable('linkarchflags', '')
    writer.variable('linkconfigflags', '')
    writer.variable('libs', '')
    writer.variable('libpaths', self.make_libpaths(self.libpaths))
    writer.variable('configlibpaths', '')
    writer.variable('archlibs', '')
    writer.variable('oslibs', self.make_libs(self.oslibs))
    writer.newline()

  def write_rules(self, writer):
    super(MSVCToolchain, self).write_rules(writer)
    writer.rule('cc', command = self.cccmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CC $in')
    writer.rule('cxx', command = self.cxxcmd, depfile = self.ccdepfile, deps = self.ccdeps, description = 'CXX $in')
    writer.rule('ar', command = self.arcmd, description = 'LIB $out')
    writer.rule('link', command = self.linkcmd, description = 'LINK $out')
    writer.rule('dll', command = self.dllcmd, description = 'DLL $out')
    writer.newline()

  def build_toolchain(self):
    if self.toolchain == '':
      versions = ['16.0', '15.0', '14.0', '13.0', '12.0', '11.0', '10.0']
      keys = [
        'HKLM\\SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VC7',
        'HKCU\\SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VC7',
        'HKLM\\SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VS7',
        'HKCU\\SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VS7',
        'HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\VisualStudio\\SxS\\VC7',
        'HKCU\\SOFTWARE\\Wow6432Node\\Microsoft\\VisualStudio\\SxS\\VC7',
        'HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\VisualStudio\\SxS\\VS7',
        'HKCU\\SOFTWARE\\Wow6432Node\\Microsoft\\VisualStudio\\SxS\\VS7'
      ]
      toolchain = ''
      for version in versions:
        for key in keys:
          try:
            query = subprocess.check_output(['reg', 'query', key, '/v', version ], stderr = subprocess.STDOUT).strip().splitlines()
            if len(query) == 2:
              toolchain = str(query[1]).split('REG_SZ')[-1].strip(" '\"\n\r\t")
          except:
            continue
          if not toolchain == '':
            #Thanks MS for making it _really_ hard to find the compiler
            if version == '15.0' or version == '16.0':
              tools_basepath = os.path.join(toolchain, 'VC', 'Tools', 'MSVC')
              tools_list = [item for item in os.listdir(tools_basepath) if os.path.isdir(os.path.join(tools_basepath, item))]
              from distutils.version import StrictVersion
              tools_list.sort(key=StrictVersion)
              toolchain = os.path.join(tools_basepath, tools_list[-1])
            self.toolchain = toolchain
            self.toolchain_version = version
            break
        if not toolchain == '':
          break
    self.includepaths += [os.path.join(self.toolchain, 'include')]
    if self.sdkpath == '':
      versions = ['v10.0', 'v8.1']
      keys = [
        'HKLM\\SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows',
        'HKCU\\SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows',
        'HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Microsoft SDKs\\Windows',
        'HKCU\\SOFTWARE\\Wow6432Node\\Microsoft\\Microsoft SDKs\\Windows'
{'fraction_non_alphanumeric': 0.15897814134370392, 'fraction_numerical': 0.011264583612712887, 'mean_word_length': 3.2773157441927157, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2714071', 'n_tokens_mistral': 5200, 'n_tokens_neox': 5017, 'n_words': 894}