<commit_before>#!/usr/bin/python


from optparse import OptionParser
import os
import subprocess
import sys


"""NEXE building script

This module will take a set of source files, include paths, library paths, and
additional arguments, and use them to build.
"""


def ErrOut(text):
  """ErrOut prints an error message and the command-line that caused it.

  Prints to standard err, both the command-line normally, and separated by
  >>...<< to make it easier to copy and paste the command, or to
  find command formating issues.
  """
  sys.stderr.write('\n\n')
  sys.stderr.write( '>>>' + '>> <<'.join(sys.argv) + '<<\n\n')
  sys.stderr.write(' '.join(sys.argv) + '<<\n\n')
  sys.stderr.write(text + '\n')
  sys.exit(-1)


def MakeDir(outdir):
  if outdir and not os.path.exists(outdir):
    # There may be a race creating this directory, so ignore failure
    try:
      os.makedirs(outdir)
    except OSError:
      pass


def RemoveQuotes(opt):
  if opt and opt[0] == '"':
    return opt[1:-1]
  return opt


def ArgToList(opt):
  outlist = []
  optlist = RemoveQuotes(opt).split(' ')
  for optitem in optlist:
    optitem = RemoveQuotes(optitem)
    if optitem:
      outlist.append(optitem)
  return outlist


class Builder(object):
  """Builder object maintains options and generates build command-lines.

  The Builder object takes a set of script command-line options, and generates
  a set of paths, and command-line options for the NaCl toolchain.
  """
  def __init__(self, options):
    arch = options.arch
    build_type = options.build.split('_')
    toolname = build_type[0]
    self.outtype = build_type[1]

    if sys.platform.startswith('linux'):
      self.osname = 'linux'
    elif sys.platform.startswith('win'):
      self.osname = 'win'
    elif sys.platform.startswith('darwin'):
      self.osname = 'mac'
    else:
      ErrOut('Toolchain OS %s not supported.' % sys.platform)

    if arch  in ['x86-32', 'x86-64']:
      self.arch = arch
      self.mainarch = 'x86'
      self.subarch = arch.split('-')[1]
      tool_subdir = 'x86_64-nacl'
    else:
      ErrOut('Toolchain architecture %s not supported.' % arch)

    if toolname == 'newlib':
      toolchain = '%s_%s_newlib' % (self.osname, self.mainarch)
      self.toolname = 'newlib'
    elif toolname == 'glibc':
      toolchain = '%s_%s' % (self.osname, self.mainarch)
      self.toolname = 'glibc'
    else:
      ErrOut('Toolchain of type %s not supported.' % toolname)


    self.root_path = options.root
    self.nacl_path = os.path.join(self.root_path, 'native_client')

    project_path, project_name = os.path.split(options.name)
    self.outdir = options.objdir

    # Set the toolchain directories
    self.toolchain = self.GenNaClPath(os.path.join('toolchain', toolchain))
    self.toolbin = os.path.join(self.toolchain, tool_subdir, 'bin')
    self.toollib = os.path.join(self.toolchain, tool_subdir, 'lib'+self.subarch)
    self.toolinc = os.path.join(self.toolchain, tool_subdir, 'include')

    self.inc_paths = ArgToList(options.incdirs)
    self.lib_paths = ArgToList(options.libdirs)

    self.name = options.name
    self.BuildCompileOptions(options.compile_flags, options.defines)
    self.BuildLinkOptions(options.link_flags)
    self.BuildArchiveOptions()
    self.verbose = options.verbose
    self.suffix = options.suffix
    self.strip = options.strip

    if self.verbose:
      print 'Compile options: %s' % self.compile_options
      print 'Linker options: %s' % self.link_options

  def GenNaClPath(self, path):
    """Helper which prepends path with the native client source directory."""
    return os.path.join(self.root_path, 'native_client', path)

  def GetBinName(self, name):
    """Helper which prepends executable with the toolchain bin directory."""
    return os.path.join(self.toolbin, name)

  def BuildAssembleOptions(self, options):
    options = ArgToList(options)
    self.assemble_options = options + ['-I' + name for name in self.inc_paths]

  def BuildCompileOptions(self, options, defines):
    """Generates compile options, called once by __init__."""
    options = ArgToList(options)
    options += ['-D' + define for define in defines]
    self.compile_options = options + ['-I' + name for name in self.inc_paths]

  def BuildLinkOptions(self, options):
    """Generates link options, called once by __init__."""
    options = ArgToList(options)
    print 'LINK OPTIONS: >>%s<<' % '<< >>'.join(options)
    if self.toolname in ['glibc', 'newlib'] and self.mainarch == 'x86':
      options += ['-B' + self.toollib]
    self.link_options = options + ['-L' + name for name in self.lib_paths]

  def BuildArchiveOptions(self):
    """Generates link options, called once by __init__."""
    self.archive_options = []

  def Run(self, cmd_line, out):
    """Helper which runs a command line."""
    if self.verbose:
      print ' '.join(cmd_line)
    try:
      ecode = subprocess.call(cmd_line)
    except Exception, err:
      ErrOut('\n%s\nFAILED: %s\n\n' % (' '.join(cmd_line), str(err)))
    if ecode == 0:
      print '    nacl-%s %s' % (os.path.basename(cmd_line[0]), out)<commit_msg>Convert paths to POSIX style on Windows and remove spew.<commit_after>#!/usr/bin/python


from optparse import OptionParser
import os
import subprocess
import sys


"""NEXE building script

This module will take a set of source files, include paths, library paths, and
additional arguments, and use them to build.
"""


def ErrOut(text):
  """ErrOut prints an error message and the command-line that caused it.

  Prints to standard err, both the command-line normally, and separated by
  >>...<< to make it easier to copy and paste the command, or to
  find command formating issues.
  """
  sys.stderr.write('\n\n')
  sys.stderr.write( '>>>' + '>> <<'.join(sys.argv) + '<<\n\n')
  sys.stderr.write(' '.join(sys.argv) + '<<\n\n')
  sys.stderr.write(text + '\n')
  sys.exit(-1)


def MakeDir(outdir):
  if outdir and not os.path.exists(outdir):
    # There may be a race creating this directory, so ignore failure
    try:
      os.makedirs(outdir)
    except OSError:
      pass


def RemoveQuotes(opt):
  if opt and opt[0] == '"':
    return opt[1:-1]
  return opt


def ArgToList(opt):
  outlist = []
  optlist = RemoveQuotes(opt).split(' ')
  for optitem in optlist:
    optitem = RemoveQuotes(optitem)
    if optitem:
      outlist.append(optitem)
  return outlist


class Builder(object):
  """Builder object maintains options and generates build command-lines.

  The Builder object takes a set of script command-line options, and generates
  a set of paths, and command-line options for the NaCl toolchain.
  """
  def __init__(self, options):
    arch = options.arch
    build_type = options.build.split('_')
    toolname = build_type[0]
    self.outtype = build_type[1]

    if sys.platform.startswith('linux'):
      self.osname = 'linux'
    elif sys.platform.startswith('win'):
      self.osname = 'win'
    elif sys.platform.startswith('darwin'):
      self.osname = 'mac'
    else:
      ErrOut('Toolchain OS %s not supported.' % sys.platform)

    if arch  in ['x86-32', 'x86-64']:
      self.arch = arch
      self.mainarch = 'x86'
      self.subarch = arch.split('-')[1]
      tool_subdir = 'x86_64-nacl'
    else:
      ErrOut('Toolchain architecture %s not supported.' % arch)

    if toolname == 'newlib':
      toolchain = '%s_%s_newlib' % (self.osname, self.mainarch)
      self.toolname = 'newlib'
    elif toolname == 'glibc':
      toolchain = '%s_%s' % (self.osname, self.mainarch)
      self.toolname = 'glibc'
    else:
      ErrOut('Toolchain of type %s not supported.' % toolname)


    self.root_path = options.root
    self.nacl_path = os.path.join(self.root_path, 'native_client')

    project_path, project_name = os.path.split(options.name)
    self.outdir = options.objdir

    # Set the toolchain directories
    self.toolchain = self.GenNaClPath(os.path.join('toolchain', toolchain))
    self.toolbin = os.path.join(self.toolchain, tool_subdir, 'bin')
    self.toollib = os.path.join(self.toolchain, tool_subdir, 'lib'+self.subarch)
    self.toolinc = os.path.join(self.toolchain, tool_subdir, 'include')

    self.inc_paths = ArgToList(options.incdirs)
    self.lib_paths = ArgToList(options.libdirs)

    self.name = options.name
    self.BuildCompileOptions(options.compile_flags, options.defines)
    self.BuildLinkOptions(options.link_flags)
    self.BuildArchiveOptions()
    self.verbose = options.verbose
    self.suffix = options.suffix
    self.strip = options.strip

    if self.verbose:
      print 'Compile options: %s' % self.compile_options
      print 'Linker options: %s' % self.link_options

  def GenNaClPath(self, path):
    """Helper which prepends path with the native client source directory."""
    return os.path.join(self.root_path, 'native_client', path)

  def GetBinName(self, name):
    """Helper which prepends executable with the toolchain bin directory."""
    return os.path.join(self.toolbin, name)

  def BuildAssembleOptions(self, options):
    options = ArgToList(options)
    self.assemble_options = options + ['-I' + name for name in self.inc_paths]

  def BuildCompileOptions(self, options, defines):
    """Generates compile options, called once by __init__."""
    options = ArgToList(options)
    options += ['-D' + define for define in defines]
    self.compile_options = options + ['-I' + name for name in self.inc_paths]

  def BuildLinkOptions(self, options):
    """Generates link options, called once by __init__."""
    options = ArgToList(options)
    if self.toolname in ['glibc', 'newlib'] and self.mainarch == 'x86':
      options += ['-B' + self.toollib]
    self.link_options = options + ['-L' + name for name in self.lib_paths]

  def BuildArchiveOptions(self):
    """Generates link options, called once by __init__."""
    self.archive_options = []

  def Run(self, cmd_line, out):
    """Helper which runs a command line."""

    # For POSIX style path on windows for POSIX based toolchain
    cmd_line = [cmd.replace('\\', '/') for cmd in cmd_line]

    if self.verbose:
      print ' '.join(cmd_line)
    try:
      ecode = subprocess.call(cmd_line)
    except Exception, err:
      ErrOut('\n%s\nFAILED: %s\n\n' % (' '.join(cmd_line), str(err)))
    if ecode == 0:
      print '    nacl-%s %s' % (os.path.basename(cmd_line[0]), out)
{'fraction_non_alphanumeric': 0.13052712878934158, 'fraction_numerical': 0.0048271867155821584, 'mean_word_length': 3.3415758591785414, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7177732', 'n_tokens_mistral': 3456, 'n_tokens_neox': 3348, 'n_words': 921}