<commit_before>import sys
import os
import subprocess
import shutil
import glob
import urllib2
import tarfile
import re
import numpy
from setuptools import setup, find_packages, Extension

"""
This file builds and installs the NuPIC binaries.
"""

NUPIC_CORE_BUCKET = (
  "https://s3-us-west-2.amazonaws.com/artifacts.numenta.org/numenta/nupic.core"
)
REPO_DIR = os.path.dirname(os.path.realpath(__file__))
DARWIN_PLATFORM = "darwin"
LINUX_PLATFORM = "linux"
UNIX_PLATFORMS = [LINUX_PLATFORM, DARWIN_PLATFORM]
WINDOWS_PLATFORMS = ["windows"]



def downloadFile(url, destFile, silent=False):
  """
  Download a file to the specified location
  """

  if not silent:
    print "Downloading from\n\t%s\nto\t%s.\n" % (url, destFile)

  destDir = os.path.dirname(destFile)
  if not os.path.exists(destDir):
    os.makedirs(destDir)

  try:
    response = urllib2.urlopen(url)
  except urllib2.URLError:
    return False

  with open(destFile, "wb") as fileObj:
    totalSize = response.info().getheader("Content-Length").strip()
    totalSize = int(totalSize)
    bytesSoFar = 0

    # Download chunks writing them to target file
    chunkSize = 8192
    oldPercent = 0
    while True:
      chunk = response.read(chunkSize)
      bytesSoFar += len(chunk)

      if not chunk:
        break

      fileObj.write(chunk)

      # Show progress
      if not silent:
        percent = (float(bytesSoFar) / totalSize) * 100
        percent = int(percent)
        if percent != oldPercent and percent % 5 == 0:
          print ("Downloaded %i of %i bytes (%i%%)."
                 % (bytesSoFar, totalSize, int(percent)))
          oldPercent = percent

  return True



def unpackFile(package, dirToUnpack, destDir, silent=False):
  """
  Unpack package file to the specified directory
  """

  if not silent:
    print "Unpacking %s into %s..." % (package, destDir)

  with tarfile.open(package, "r:gz") as tarFileObj:
    tarFileObj.extractall(destDir)

  # Copy subdirectories to a level up
  subDirs = os.listdir(destDir + "/" + dirToUnpack)
  for subDir in subDirs:
    shutil.rmtree(destDir + "/" + subDir, True)
    shutil.move(destDir + "/" + dirToUnpack + "/" + subDir,
                destDir + "/" + subDir)

  shutil.rmtree(destDir + "/" + dirToUnpack, True)



def printOptions(optionsDesc):
  """
  Print command line options.
  """

  print "Options:\n"

  for option in optionsDesc:
    optionUsage = "--" + option[0]
    if option[1] != "":
      optionUsage += "=[" + option[1] + "]"

    optionDesc = option[2]
    print "    " + optionUsage.ljust(30) + " = " + optionDesc



def getCommandLineOptions():

  # optionDesc = [name, value, description]
  optionsDesc = []
  optionsDesc.append(
    ["nupic-core-dir",
     "dir",
     "(optional) Absolute path to nupic.core binary release directory"]
  )
  optionsDesc.append(
    ["skip-compare-versions",
     "",
     "(optional) Skip nupic.core version comparison"]
  )
  optionsDesc.append(
    ["optimizations-native",
    "value",
    "(optional) enable aggressive compiler optimizations"]
  )
  optionsDesc.append(
    ["optimizations-lto",
    "value",
    "(optional) enable link-time optimizations (LTO); currently only for gcc and linker ld.gold"]
  )
  optionsDesc.append(
    ["debug",
    "value",
    "(optional) compile in mode suitable for debugging; overrides any optimizations"]
  )



  # Read command line options looking for extra options
  # For example, an user could type:
  #   python setup.py install --nupic-core-dir="path/to/release"
  # which will set the nupic.core release dir
  optionsValues = dict()
  for arg in sys.argv[:]:
    optionFound = False
    for option in optionsDesc:
      name = option[0]
      if "--" + name in arg:
        value = None
        hasValue = (option[1] != "")
        if hasValue:
          value = arg.partition("=")[2]

        optionsValues[name] = value
        sys.argv.remove(arg)
        optionFound = True
        break

    if not optionFound:
      if ("--help-nupic" in arg):
        printOptions(optionsDesc)
        sys.exit()

  return optionsValues



def getPlatformInfo():
  """
  Identify platform
  """

  if "linux" in sys.platform:
    platform = "linux"
  elif "darwin" in sys.platform:
    platform = "darwin"
  elif "windows" in sys.platform:
    platform = "windows"
  else:
    raise Exception("Platform '%s' is unsupported!" % sys.platform)

  if sys.maxsize > 2**32:
    bitness = "64"
  else:
    bitness = "32"

  return platform, bitness



def getVersion():
  """
  Get version from local file.
  """
  with open("VERSION", "r") as versionFile:
    return versionFile.read().strip()



def generateSwigWrap(swigExecutable, swigFlags, interfaceFile):
  """
  Generate C++ code from the specified SWIG interface file.
  """
  wrap = interfaceFile.replace(".i", "_wrap.cxx")

  cmd = swigExecutable + " -c++ -python "
  for flag in swigFlags:
    cmd += flag + " "
  cmd += interfaceFile
  print cmd
  proc = subprocess.Popen(cmd, shell=True)
  proc.wait()

  return wrap



def findRequirements():
  """
  Read the requirements.txt file and parse into requirements for setup's
  install_requirements option.
  """
  requirementsPath = os.path.join(REPO_DIR, "external/common/requirements.txt")
  return [
    line.strip()
    for line in open(requirementsPath).readlines()
    if not line.startswith("#")
  ]



def getLibPrefix(platform):
  """
  Returns the default system prefix of a compiled library.
  """
  if platform in UNIX_PLATFORMS:
    return "lib"
  elif platform in WINDOWS_PLATFORMS:
    return ""



def getStaticLibExtension(platform):
  """
  Returns the default system extension of a compiled static library.
  """
  if platform in UNIX_PLATFORMS:
    return ".a"
  elif platform in WINDOWS_PLATFORMS:
    return ".lib"



def getSharedLibExtension(platform):
  """
  Returns the default system extension of a compiled shared library.
  """
  if platform in UNIX_PLATFORMS:
    return ".so"
  elif platform in WINDOWS_PLATFORMS:
    return ".dll"




def extractNupicCoreTarget():
  # First, get the nupic.core SHA and remote location from local config.
  nupicConfig = {}
  if os.path.exists(REPO_DIR + "/.nupic_config"):
    execfile(
      os.path.join(REPO_DIR, ".nupic_config"), {}, nupicConfig
    )
  elif os.path.exists(os.environ["HOME"] + "/.nupic_config"):
    execfile(
      os.path.join(os.environ["HOME"], ".nupic_config"), {}, nupicConfig
    )
  else:
    execfile(
      os.path.join(REPO_DIR, ".nupic_modules"), {}, nupicConfig
    )
  return nupicConfig["NUPIC_CORE_COMMITISH"]



def getDefaultNupicCoreDirectories():
  # Default nupic.core location is relative to the NuPIC checkout.
  return (
    REPO_DIR + "/extensions/core/build/release",
    REPO_DIR + "/extensions/core"
  )



def getExtensionModules(nupicCoreReleaseDir, platform, bitness, cmdOptions=None):
  #
  # Gives the version of Python necessary to get installation directories
  # for use with pythonVersion, etc.
  #
  if sys.version_info < (2, 7):
    raise Exception("Fatal Error: Python 2.7 or later is required.")

  pythonVersion = str(sys.version_info[0]) + '.' + str(sys.version_info[1])

  #
  # Find out where system installation of python is.
  #
  pythonPrefix = sys.prefix
  pythonPrefix = pythonPrefix.replace("\\", "/")
  pythonIncludeDir = pythonPrefix + "/include/python" + pythonVersion

  commonDefines = [
    ("NUPIC2", None),
    ("NTA_OS_" + platform.upper(), None),
    ("NTA_ARCH_" + bitness, None),
    ("NTA_PYTHON_SUPPORT", pythonVersion),
    ("NTA_INTERNAL", None),
    ("NTA_ASSERTIONS_ON", None),
    ("NTA_ASM", None),
    ("HAVE_CONFIG_H", None),
    ("BOOST_NO_WREGEX", None)]

  commonIncludeDirs = [
    REPO_DIR + "/external/" + platform + bitness + "/include",
    REPO_DIR + "/external/common/include",
    REPO_DIR + "/extensions",
    REPO_DIR,
    nupicCoreReleaseDir + "/include",
    pythonIncludeDir]

  commonCompileFlags = [
    # Adhere to c++11 spec
    "-std=c++11",
    # Generate 32 or 64 bit code
    "-m" + bitness,
    # `position independent code`, required for shared libraries
    "-fPIC",
    "-fvisibility=hidden",
    "-Wall",
    "-Wextra",
    "-Wreturn-type",
    "-Wunused",
    "-Wno-unused-parameter",
    # optimization flags (generic builds used for binary distribution)
    "-mtune=generic",
    "-O2",
  ]
  if platform == "darwin":
    commonCompileFlags.append("-stdlib=libc++")

  commonLinkFlags = [
    "-m" + bitness,
    "-fPIC",
    "-L" + nupicCoreReleaseDir + "/lib",
    # for Cap'n'Proto serialization
    "-lkj",
    "-lcapnp",
    "-lcapnpc",
    # optimization (safe defaults)
    "-O2",
  ]

  # Optimizations
  if getCommandLineOption("debug", cmdOptions):
    commonCompileFlags.append("-Og")
    commonCompileFlags.append("-g")
    commonLinkFlags.append("-O0")
  else:
    if getCommandLineOption("optimizations-native", cmdOptions):
      commonCompileFlags.append("-march=native")
      commonCompileFlags.append("-O3")
      commonLinkFlags.append("-O3")
    if getCommandLineOption("optimizations-lto", cmdOptions):
      commonCompileFlags.append("-fuse-linker-plugin")
      commonCompileFlags.append("-flto-report")
      commonCompileFlags.append("-fuse-ld=gold")
      commonCompileFlags.append("-flto")
      commonLinkFlags.append("-flto")



  commonLibraries = [
    "dl",
    "python" + pythonVersion,
    "kj",
    "capnp",
    "capnpc"]
  if platform == "linux":
    commonLibraries.extend(["pthread"])

  commonObjects = [
    nupicCoreReleaseDir + "/lib/" +
      getLibPrefix(platform) + "nupic_core" + getStaticLibExtension(platform)]

  # Find py_support cpp files in nupic.core
  pythonSupportSources = [
    nupicCoreReleaseDir + "include/nupic/py_support/NumpyVector.cpp",
    nupicCoreReleaseDir + "include/nupic/py_support/PyArray.cpp",
    nupicCoreReleaseDir + "include/nupic/py_support/PyHelpers.cpp",
    nupicCoreReleaseDir + "include/nupic/py_support/PythonStream.cpp"]

  extensions = []

  #
  # SWIG
  #
  swigDir = REPO_DIR + "/external/common/share/swig/3.0.2"<commit_msg>adjusted path to py_support files<commit_after>import sys
import os
import subprocess
import shutil
import glob
import urllib2
import tarfile
import re
import numpy
from setuptools import setup, find_packages, Extension

"""
This file builds and installs the NuPIC binaries.
"""

NUPIC_CORE_BUCKET = (
  "https://s3-us-west-2.amazonaws.com/artifacts.numenta.org/numenta/nupic.core"
)
REPO_DIR = os.path.dirname(os.path.realpath(__file__))
DARWIN_PLATFORM = "darwin"
LINUX_PLATFORM = "linux"
UNIX_PLATFORMS = [LINUX_PLATFORM, DARWIN_PLATFORM]
WINDOWS_PLATFORMS = ["windows"]



def downloadFile(url, destFile, silent=False):
  """
  Download a file to the specified location
  """

  if not silent:
    print "Downloading from\n\t%s\nto\t%s.\n" % (url, destFile)

  destDir = os.path.dirname(destFile)
  if not os.path.exists(destDir):
    os.makedirs(destDir)

  try:
    response = urllib2.urlopen(url)
  except urllib2.URLError:
    return False

  with open(destFile, "wb") as fileObj:
    totalSize = response.info().getheader("Content-Length").strip()
    totalSize = int(totalSize)
    bytesSoFar = 0

    # Download chunks writing them to target file
    chunkSize = 8192
    oldPercent = 0
    while True:
      chunk = response.read(chunkSize)
      bytesSoFar += len(chunk)

      if not chunk:
        break

      fileObj.write(chunk)

      # Show progress
      if not silent:
        percent = (float(bytesSoFar) / totalSize) * 100
        percent = int(percent)
        if percent != oldPercent and percent % 5 == 0:
          print ("Downloaded %i of %i bytes (%i%%)."
                 % (bytesSoFar, totalSize, int(percent)))
          oldPercent = percent

  return True



def unpackFile(package, dirToUnpack, destDir, silent=False):
  """
  Unpack package file to the specified directory
  """

  if not silent:
    print "Unpacking %s into %s..." % (package, destDir)

  with tarfile.open(package, "r:gz") as tarFileObj:
    tarFileObj.extractall(destDir)

  # Copy subdirectories to a level up
  subDirs = os.listdir(destDir + "/" + dirToUnpack)
  for subDir in subDirs:
    shutil.rmtree(destDir + "/" + subDir, True)
    shutil.move(destDir + "/" + dirToUnpack + "/" + subDir,
                destDir + "/" + subDir)

  shutil.rmtree(destDir + "/" + dirToUnpack, True)



def printOptions(optionsDesc):
  """
  Print command line options.
  """

  print "Options:\n"

  for option in optionsDesc:
    optionUsage = "--" + option[0]
    if option[1] != "":
      optionUsage += "=[" + option[1] + "]"

    optionDesc = option[2]
    print "    " + optionUsage.ljust(30) + " = " + optionDesc



def getCommandLineOptions():

  # optionDesc = [name, value, description]
  optionsDesc = []
  optionsDesc.append(
    ["nupic-core-dir",
     "dir",
     "(optional) Absolute path to nupic.core binary release directory"]
  )
  optionsDesc.append(
    ["skip-compare-versions",
     "",
     "(optional) Skip nupic.core version comparison"]
  )
  optionsDesc.append(
    ["optimizations-native",
    "value",
    "(optional) enable aggressive compiler optimizations"]
  )
  optionsDesc.append(
    ["optimizations-lto",
    "value",
    "(optional) enable link-time optimizations (LTO); currently only for gcc and linker ld.gold"]
  )
  optionsDesc.append(
    ["debug",
    "value",
    "(optional) compile in mode suitable for debugging; overrides any optimizations"]
  )



  # Read command line options looking for extra options
  # For example, an user could type:
  #   python setup.py install --nupic-core-dir="path/to/release"
  # which will set the nupic.core release dir
  optionsValues = dict()
  for arg in sys.argv[:]:
    optionFound = False
    for option in optionsDesc:
      name = option[0]
      if "--" + name in arg:
        value = None
        hasValue = (option[1] != "")
        if hasValue:
          value = arg.partition("=")[2]

        optionsValues[name] = value
        sys.argv.remove(arg)
        optionFound = True
        break

    if not optionFound:
      if ("--help-nupic" in arg):
        printOptions(optionsDesc)
        sys.exit()

  return optionsValues



def getPlatformInfo():
  """
  Identify platform
  """

  if "linux" in sys.platform:
    platform = "linux"
  elif "darwin" in sys.platform:
    platform = "darwin"
  elif "windows" in sys.platform:
    platform = "windows"
  else:
    raise Exception("Platform '%s' is unsupported!" % sys.platform)

  if sys.maxsize > 2**32:
    bitness = "64"
  else:
    bitness = "32"

  return platform, bitness



def getVersion():
  """
  Get version from local file.
  """
  with open("VERSION", "r") as versionFile:
    return versionFile.read().strip()



def generateSwigWrap(swigExecutable, swigFlags, interfaceFile):
  """
  Generate C++ code from the specified SWIG interface file.
  """
  wrap = interfaceFile.replace(".i", "_wrap.cxx")

  cmd = swigExecutable + " -c++ -python "
  for flag in swigFlags:
    cmd += flag + " "
  cmd += interfaceFile
  print cmd
  proc = subprocess.Popen(cmd, shell=True)
  proc.wait()

  return wrap



def findRequirements():
  """
  Read the requirements.txt file and parse into requirements for setup's
  install_requirements option.
  """
  requirementsPath = os.path.join(REPO_DIR, "external/common/requirements.txt")
  return [
    line.strip()
    for line in open(requirementsPath).readlines()
    if not line.startswith("#")
  ]



def getLibPrefix(platform):
  """
  Returns the default system prefix of a compiled library.
  """
  if platform in UNIX_PLATFORMS:
    return "lib"
  elif platform in WINDOWS_PLATFORMS:
    return ""



def getStaticLibExtension(platform):
  """
  Returns the default system extension of a compiled static library.
  """
  if platform in UNIX_PLATFORMS:
    return ".a"
  elif platform in WINDOWS_PLATFORMS:
    return ".lib"



def getSharedLibExtension(platform):
  """
  Returns the default system extension of a compiled shared library.
  """
  if platform in UNIX_PLATFORMS:
    return ".so"
  elif platform in WINDOWS_PLATFORMS:
    return ".dll"




def extractNupicCoreTarget():
  # First, get the nupic.core SHA and remote location from local config.
  nupicConfig = {}
  if os.path.exists(REPO_DIR + "/.nupic_config"):
    execfile(
      os.path.join(REPO_DIR, ".nupic_config"), {}, nupicConfig
    )
  elif os.path.exists(os.environ["HOME"] + "/.nupic_config"):
    execfile(
      os.path.join(os.environ["HOME"], ".nupic_config"), {}, nupicConfig
    )
  else:
    execfile(
      os.path.join(REPO_DIR, ".nupic_modules"), {}, nupicConfig
    )
  return nupicConfig["NUPIC_CORE_COMMITISH"]



def getDefaultNupicCoreDirectories():
  # Default nupic.core location is relative to the NuPIC checkout.
  return (
    REPO_DIR + "/extensions/core/build/release",
    REPO_DIR + "/extensions/core"
  )



def getExtensionModules(nupicCoreReleaseDir, platform, bitness, cmdOptions=None):
  #
  # Gives the version of Python necessary to get installation directories
  # for use with pythonVersion, etc.
  #
  if sys.version_info < (2, 7):
    raise Exception("Fatal Error: Python 2.7 or later is required.")

  pythonVersion = str(sys.version_info[0]) + '.' + str(sys.version_info[1])

  #
  # Find out where system installation of python is.
  #
  pythonPrefix = sys.prefix
  pythonPrefix = pythonPrefix.replace("\\", "/")
  pythonIncludeDir = pythonPrefix + "/include/python" + pythonVersion

  commonDefines = [
    ("NUPIC2", None),
    ("NTA_OS_" + platform.upper(), None),
    ("NTA_ARCH_" + bitness, None),
    ("NTA_PYTHON_SUPPORT", pythonVersion),
    ("NTA_INTERNAL", None),
    ("NTA_ASSERTIONS_ON", None),
    ("NTA_ASM", None),
    ("HAVE_CONFIG_H", None),
    ("BOOST_NO_WREGEX", None)]

  commonIncludeDirs = [
    REPO_DIR + "/external/" + platform + bitness + "/include",
    REPO_DIR + "/external/common/include",
    REPO_DIR + "/extensions",
    REPO_DIR,
    nupicCoreReleaseDir + "/include",
    pythonIncludeDir]

  commonCompileFlags = [
    # Adhere to c++11 spec
    "-std=c++11",
    # Generate 32 or 64 bit code
    "-m" + bitness,
    # `position independent code`, required for shared libraries
    "-fPIC",
    "-fvisibility=hidden",
    "-Wall",
    "-Wextra",
    "-Wreturn-type",
    "-Wunused",
    "-Wno-unused-parameter",
    # optimization flags (generic builds used for binary distribution)
    "-mtune=generic",
    "-O2",
  ]
  if platform == "darwin":
    commonCompileFlags.append("-stdlib=libc++")

  commonLinkFlags = [
    "-m" + bitness,
    "-fPIC",
    "-L" + nupicCoreReleaseDir + "/lib",
    # for Cap'n'Proto serialization
    "-lkj",
    "-lcapnp",
    "-lcapnpc",
    # optimization (safe defaults)
    "-O2",
  ]

  # Optimizations
  if getCommandLineOption("debug", cmdOptions):
    commonCompileFlags.append("-Og")
    commonCompileFlags.append("-g")
    commonLinkFlags.append("-O0")
  else:
    if getCommandLineOption("optimizations-native", cmdOptions):
      commonCompileFlags.append("-march=native")
      commonCompileFlags.append("-O3")
      commonLinkFlags.append("-O3")
    if getCommandLineOption("optimizations-lto", cmdOptions):
      commonCompileFlags.append("-fuse-linker-plugin")
      commonCompileFlags.append("-flto-report")
      commonCompileFlags.append("-fuse-ld=gold")
      commonCompileFlags.append("-flto")
      commonLinkFlags.append("-flto")



  commonLibraries = [
    "dl",
    "python" + pythonVersion,
    "kj",
    "capnp",
    "capnpc"]
  if platform == "linux":
    commonLibraries.extend(["pthread"])

  commonObjects = [
    nupicCoreReleaseDir + "/lib/" +
      getLibPrefix(platform) + "nupic_core" + getStaticLibExtension(platform)]

  # Find py_support cpp files in nupic.core
  pythonSupportSources = [
    nupicCoreReleaseDir + "/include/nupic/py_support/NumpyVector.cpp",
    nupicCoreReleaseDir + "/include/nupic/py_support/PyArray.cpp",
    nupicCoreReleaseDir + "/include/nupic/py_support/PyHelpers.cpp",
    nupicCoreReleaseDir + "/include/nupic/py_support/PythonStream.cpp"]

  extensions = []

  #
  # SWIG
  #
  swigDir = REPO_DIR + "/external/common/share/swig/3.0.2"
{'fraction_non_alphanumeric': 0.12972704714640199, 'fraction_numerical': 0.005459057071960298, 'mean_word_length': 3.1781049139539705, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7258212', 'n_tokens_mistral': 6792, 'n_tokens_neox': 6568, 'n_words': 1579}