<commit_before>#!/usr/bin/env python


#App="$1" # argument is the application to fixup
#LibrariesPrefix="Contents/Libraries"
#echo ""
#echo "Fixing up $App"
#echo "All required frameworks/libraries will be placed under $App/$LibrariesPrefix"
#echo ""
#echo "----------------------------"
#echo "Locating all executables and dylibs already in the package ... "
#
## the sed-call removes the : Mach-O.. suffix that "file" generates
#executables=`find $App | xargs file | grep -i "Mach-O.*executable" | sed "s/:.*//" | sort`
#
#echo "----------------------------"
#echo "Found following executables:"
#for i in $executables; do
#  echo $i
#done
#
## for each executable, find any external library.
#
#
#libraries=`find $App | xargs file | grep -i "Mach-O.*shared library" | sed "s/:.*//" | sort`
#
## command to find all external libraries referrenced in package:
## find paraview.app | xargs file | grep "Mach-O" | sed "s/:.*//" | xargs otool -l | grep " name" | sort | uniq | sed "s/name\ //" | grep -v "@executable"
#
## find non-system libs
## find paraview.app | xargs file | grep "Mach-O" | sed "s/:.*//" | xargs otool -l | grep " name" | sort | uniq | sed "s/name\ //" | grep -v "@executable" | grep -v "/System/" | grep -v "/usr/lib/"

import commands
import sys
import os.path
import re
import shutil

class Library(object):
  def __init__(self):
    # This is the actual path to a physical file
    self.RealPath = None

    # This is the id for shared library.
    self.Id = None

    # These are names for symbolic links to this file.
    self.SymLinks = []

    # This is a symlink to the library that may have been used in other libraries' otool -L output
    self.LinkPath = None

    self.__depencies = None
    pass

  def __hash__(self):
    return self.RealPath.__hash__()

  def __eq__(self, other):
    return self.RealPath == other.RealPath

  def __repr__(self):
    return "Library(%s : %s)" % (self.Id, self.RealPath)

  def dependencies(self, exepath):
    if self.__depencies:
      return self.__depencies
    collection = set()
    for dep in _getdependencies(self.RealPath):
      collection.add(Library.createFromReference(dep, exepath))
    self.__depencies = collection
    return self.__depencies

  def copyToApp(self, app, fakeCopy=False):
    if _isframework(self.RealPath):
      m = re.match(r'(.*)/(\w+\.framework)/(.*)', self.RealPath)
      # FIXME: this could be optimized to only copy the particular version.
      if not fakeCopy:
        print "Copying %s/%s ==> %s" % (m.group(1), m.group(2), ".../Contents/Frameworks/")
        dirdest = os.path.join(os.path.join(app, "Contents/Frameworks/"), m.group(2))
        filedest = os.path.join(dirdest, m.group(3))
        shutil.copytree(os.path.join(m.group(1), m.group(2)), dirdest, symlinks=True)
      self.Id = "@executable_path/../Frameworks/%s" % (os.path.join(m.group(2), m.group(3)))
      #print self.Id, dirdest, filedest
      if not fakeCopy:
        commands.getoutput('install_name_tool -id "%s" %s' % (self.Id, filedest))
    else:
      if not fakeCopy:
        print "Copying %s ==> %s" % (self.RealPath, ".../Contents/Libraries/%s" % os.path.basename(self.RealPath))
        shutil.copy(self.RealPath, os.path.join(app, "Contents/Libraries"))
      self.Id = "@executable_path/../Libraries/%s" % os.path.basename(self.RealPath)
      if not fakeCopy:
        # Sometimes the library can't be modified due to it original permissions... fix them
        import stat
        os.chmod(os.path.join(app, "Contents/Libraries/%s" % os.path.basename(self.RealPath)),\
                 stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
        commands.getoutput('install_name_tool -id "%s" %s' % (self.Id,
                            os.path.join(app, "Contents/Libraries/%s" % os.path.basename(self.RealPath))))

      # Create symlinks for this copied file in the install location
      # as were present in the source dir.
      destdir = os.path.join(app, "Contents/Libraries")
      # sourcefile is the file we copied already into the app bundle. We need to create symlink
      # to it itself in the app bundle.
      sourcefile = os.path.basename(self.RealPath)
      for symlink in self.SymLinks:
        print "Creating Symlink %s ==> .../Contents/Libraries/%s" % (symlink, os.path.basename(self.RealPath))
        if not fakeCopy:
          commands.getoutput("ln -s %s %s" % (sourcefile, os.path.join(destdir, symlink)))

  @classmethod
  def createFromReference(cls, ref, exepath):
    path = ref.replace("@executable_path", exepath)
    if not os.path.exists(path):
      path = _find(ref)
    return cls.createFromPath(path)

  @classmethod
  def createFromPath(cls, path):
    if not os.path.exists(path):
      raise RuntimeError, "%s is not a filename" % path
    lib = Library()
    lib.RealPath = os.path.realpath(path)
    lib.Id = _getid(path)
    if not os.path.abspath(path) == lib.Id:
        lib.LinkPath = os.path.abspath(path)
    elif lib.RealPath != lib.Id:
        lib.LinkPath = lib.RealPath
    # locate all symlinks to this file in the containing directory. These are used when copying.
    # We ensure that we copy all symlinks too.
    dirname = os.path.dirname(lib.RealPath)
    symlinks = commands.getoutput("find -L %s -samefile %s" % (dirname, lib.RealPath))
    symlinks = symlinks.split()
    try:
      symlinks.remove(lib.RealPath)
    except ValueError:
      pass
    linknames = []
    for link in symlinks:
      linkname = os.path.basename(link)
      linknames.append(linkname)
    lib.SymLinks = linknames
    return lib


def _getid(lib):
  """Returns the id for the library"""
  val = commands.getoutput("otool -D %s" % lib)
  m = re.match(r"[^:]+:\s*([^\s]+)", val)
  if m:
    return m.group(1)
  raise RuntimeError, "Could not determine id for %s" % lib

def _getdependencies(path):
  val = commands.getoutput('otool -l %s| grep " name" | sort | uniq | sed "s/name\ //" | sed "s/(offset.*)//"' % path)
  return val.split()

def isexcluded(id):
  # we don't consider the libgfortran or libquadmath a system library since
  # it will rarely be on the installed machine
  if re.match(r".*libgfortran.*", id) or re.match(r".*libquadmath.*", id):
    return False
  if re.match(r"^/System/Library", id):
    return True
  if re.match(r"^/usr/lib", id):
    return True
  if re.match(r"^/usr/local", id):
    return True
  if re.match(r"^libz.1.dylib", id):
    return True
  return False

def _isframework(path):
  if re.match(".*\.framework.*", path):
    return True

def _find(ref):
  name = os.path.basename(ref)
  for loc in SearchLocations:
    output = commands.getoutput('find "%s" -name "%s"' % (loc, name)).strip()
    if output and '\n' in output:
      files = output.split('\n')
      fileinfo = [ (f, commands.getoutput('file %s | grep -i "Mach-O"' % (f,))) for f in files]
      libfiles = [ f[0] for f in fileinfo if len(f[1]) > 0]
      if len(libfiles) > 0:
        return libfiles[0] # And hope it is the right one
    if output:
      return output<commit_msg>osx: include libgcc_s since it is needed by libgfortran<commit_after>#!/usr/bin/env python


#App="$1" # argument is the application to fixup
#LibrariesPrefix="Contents/Libraries"
#echo ""
#echo "Fixing up $App"
#echo "All required frameworks/libraries will be placed under $App/$LibrariesPrefix"
#echo ""
#echo "----------------------------"
#echo "Locating all executables and dylibs already in the package ... "
#
## the sed-call removes the : Mach-O.. suffix that "file" generates
#executables=`find $App | xargs file | grep -i "Mach-O.*executable" | sed "s/:.*//" | sort`
#
#echo "----------------------------"
#echo "Found following executables:"
#for i in $executables; do
#  echo $i
#done
#
## for each executable, find any external library.
#
#
#libraries=`find $App | xargs file | grep -i "Mach-O.*shared library" | sed "s/:.*//" | sort`
#
## command to find all external libraries referrenced in package:
## find paraview.app | xargs file | grep "Mach-O" | sed "s/:.*//" | xargs otool -l | grep " name" | sort | uniq | sed "s/name\ //" | grep -v "@executable"
#
## find non-system libs
## find paraview.app | xargs file | grep "Mach-O" | sed "s/:.*//" | xargs otool -l | grep " name" | sort | uniq | sed "s/name\ //" | grep -v "@executable" | grep -v "/System/" | grep -v "/usr/lib/"

import commands
import sys
import os.path
import re
import shutil

class Library(object):
  def __init__(self):
    # This is the actual path to a physical file
    self.RealPath = None

    # This is the id for shared library.
    self.Id = None

    # These are names for symbolic links to this file.
    self.SymLinks = []

    # This is a symlink to the library that may have been used in other libraries' otool -L output
    self.LinkPath = None

    self.__depencies = None
    pass

  def __hash__(self):
    return self.RealPath.__hash__()

  def __eq__(self, other):
    return self.RealPath == other.RealPath

  def __repr__(self):
    return "Library(%s : %s)" % (self.Id, self.RealPath)

  def dependencies(self, exepath):
    if self.__depencies:
      return self.__depencies
    collection = set()
    for dep in _getdependencies(self.RealPath):
      collection.add(Library.createFromReference(dep, exepath))
    self.__depencies = collection
    return self.__depencies

  def copyToApp(self, app, fakeCopy=False):
    if _isframework(self.RealPath):
      m = re.match(r'(.*)/(\w+\.framework)/(.*)', self.RealPath)
      # FIXME: this could be optimized to only copy the particular version.
      if not fakeCopy:
        print "Copying %s/%s ==> %s" % (m.group(1), m.group(2), ".../Contents/Frameworks/")
        dirdest = os.path.join(os.path.join(app, "Contents/Frameworks/"), m.group(2))
        filedest = os.path.join(dirdest, m.group(3))
        shutil.copytree(os.path.join(m.group(1), m.group(2)), dirdest, symlinks=True)
      self.Id = "@executable_path/../Frameworks/%s" % (os.path.join(m.group(2), m.group(3)))
      #print self.Id, dirdest, filedest
      if not fakeCopy:
        commands.getoutput('install_name_tool -id "%s" %s' % (self.Id, filedest))
    else:
      if not fakeCopy:
        print "Copying %s ==> %s" % (self.RealPath, ".../Contents/Libraries/%s" % os.path.basename(self.RealPath))
        shutil.copy(self.RealPath, os.path.join(app, "Contents/Libraries"))
      self.Id = "@executable_path/../Libraries/%s" % os.path.basename(self.RealPath)
      if not fakeCopy:
        # Sometimes the library can't be modified due to it original permissions... fix them
        import stat
        os.chmod(os.path.join(app, "Contents/Libraries/%s" % os.path.basename(self.RealPath)),\
                 stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
        commands.getoutput('install_name_tool -id "%s" %s' % (self.Id,
                            os.path.join(app, "Contents/Libraries/%s" % os.path.basename(self.RealPath))))

      # Create symlinks for this copied file in the install location
      # as were present in the source dir.
      destdir = os.path.join(app, "Contents/Libraries")
      # sourcefile is the file we copied already into the app bundle. We need to create symlink
      # to it itself in the app bundle.
      sourcefile = os.path.basename(self.RealPath)
      for symlink in self.SymLinks:
        print "Creating Symlink %s ==> .../Contents/Libraries/%s" % (symlink, os.path.basename(self.RealPath))
        if not fakeCopy:
          commands.getoutput("ln -s %s %s" % (sourcefile, os.path.join(destdir, symlink)))

  @classmethod
  def createFromReference(cls, ref, exepath):
    path = ref.replace("@executable_path", exepath)
    if not os.path.exists(path):
      path = _find(ref)
    return cls.createFromPath(path)

  @classmethod
  def createFromPath(cls, path):
    if not os.path.exists(path):
      raise RuntimeError, "%s is not a filename" % path
    lib = Library()
    lib.RealPath = os.path.realpath(path)
    lib.Id = _getid(path)
    if not os.path.abspath(path) == lib.Id:
        lib.LinkPath = os.path.abspath(path)
    elif lib.RealPath != lib.Id:
        lib.LinkPath = lib.RealPath
    # locate all symlinks to this file in the containing directory. These are used when copying.
    # We ensure that we copy all symlinks too.
    dirname = os.path.dirname(lib.RealPath)
    symlinks = commands.getoutput("find -L %s -samefile %s" % (dirname, lib.RealPath))
    symlinks = symlinks.split()
    try:
      symlinks.remove(lib.RealPath)
    except ValueError:
      pass
    linknames = []
    for link in symlinks:
      linkname = os.path.basename(link)
      linknames.append(linkname)
    lib.SymLinks = linknames
    return lib


def _getid(lib):
  """Returns the id for the library"""
  val = commands.getoutput("otool -D %s" % lib)
  m = re.match(r"[^:]+:\s*([^\s]+)", val)
  if m:
    return m.group(1)
  raise RuntimeError, "Could not determine id for %s" % lib

def _getdependencies(path):
  val = commands.getoutput('otool -l %s| grep " name" | sort | uniq | sed "s/name\ //" | sed "s/(offset.*)//"' % path)
  return val.split()

def isexcluded(id):
  # we don't consider the libgfortran or libquadmath a system library since
  # it will rarely be on the installed machine
  # libgfortran depends on libgcc_s, so that one also should not be considered
  # a system library
  if re.match(r".*libgfortran.*", id) or re.match(r".*libquadmath.*", id) or re.match(r".*libgcc_s.*", id):
    return False
  if re.match(r"^/System/Library", id):
    return True
  if re.match(r"^/usr/lib", id):
    return True
  if re.match(r"^/usr/local", id):
    return True
  if re.match(r"^libz.1.dylib", id):
    return True
  return False

def _isframework(path):
  if re.match(".*\.framework.*", path):
    return True

def _find(ref):
  name = os.path.basename(ref)
  for loc in SearchLocations:
    output = commands.getoutput('find "%s" -name "%s"' % (loc, name)).strip()
    if output and '\n' in output:
      files = output.split('\n')
      fileinfo = [ (f, commands.getoutput('file %s | grep -i "Mach-O"' % (f,))) for f in files]
      libfiles = [ f[0] for f in fileinfo if len(f[1]) > 0]
      if len(libfiles) > 0:
        return libfiles[0] # And hope it is the right one
    if output:
      return output
{'fraction_non_alphanumeric': 0.1509433962264151, 'fraction_numerical': 0.002244511468050782, 'mean_word_length': 2.8935008192244673, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4601559', 'n_tokens_mistral': 4846, 'n_tokens_neox': 4778, 'n_words': 1393}