<commit_before>import buildtool.path
import buildtool.plist
import random
import posixpath
import sys
import cStringIO

TYPES = {
    'c': 'sourcecode.c.c',
    'cxx': 'sourcecode.cpp.cpp',
    'h': 'sourcecode.c.h',
    'hxx': 'sourcecode.cpp.h',
    'm': 'sourcecode.c.objc',
    'framework': 'wrapper.framework',
    'xib': 'file.xib',
    'plist': 'text.plist.xml',
}

RESOURCE_BUILD = set([
    'file.xib',
])

SOURCE_BUILD = set([
    'sourcecode.c.c',
    'sourcecode.cpp.cpp',
    'sourcecode.c.objc',
])

FRAMEWORK_BUILD = set([
    'wrapper.framework',
])

ISAS = [
    'PBXBuildFile',
    'PBXFileReference',
    'PBXFrameworksBuildPhase',
    'PBXGroup',
    'PBXNativeTarget',
    'PBXProject',
    'PBXResourcesBuildPhase',
    'PBXShellScriptBuildPhase',
    'PBXSourcesBuildPhase',
    'XCBuildConfiguration',
    'XCConfigurationList',
]

# UNSET = token('UNSET')

class optional(object):
    def __init__(self, what):
        self.what = what
    def __repr__(self):
        return 'optional(%s)' % repr(self.what)

class XcodeObject(object):
    COMPACT = False

    def _allobjs(self, dest):
        if self in dest:
            return
        dest.add(self)
        for attr in self.ATTRS:
            if isinstance(attr, optional):
                attr = attr.what
            v = getattr(self, attr)
            if isinstance(v, list):
                for x in v:
                    if isinstance(x, XcodeObject):
                        x._allobjs(dest)
            elif isinstance(v, dict):
                for x in v.itervalues():
                    if isinstance(x, XcodeObject):
                        x._allobjs(dest)
            elif isinstance(v, XcodeObject):
                v._allobjs(dest)

    def allobjs(self):
        objs = set([])
        self._allobjs(objs)
        return objs

    def emit(self, w, ids):
        def objemit(x):
            if isinstance(x, dict):
                w.start_dict()
                for k, v in x.iteritems():
                    w.write_key(k)
                    objemit(v)
                w.end_dict()
            elif isinstance(x, list):
                w.start_list()
                for v in x:
                    objemit(v)
                w.end_list()
            elif isinstance(x, XcodeObject):
                w.write_object(ids[x])
            else:
                w.write_object(x)
        w.write_key(ids[self])
        w.start_dict(self.COMPACT)
        w.write_pair('isa', self.ISA)
        for attr in self.ATTRS:
            if isinstance(attr, optional):
                attr = attr.what
                isoptional = True
            else:
                isoptional = False
            val = getattr(self, attr)
            if val is None:
                if not isoptional:
                    raise Exception('Missing %r attribiute from %r class' %
                        (attr, self.__class__.__name__))
            else:
                w.write_key(attr)
                objemit(val)
        w.end_dict()

def relpath(p, x):
    # This function doesn't work for all inputs
    p = p.split('/') if p else []
    x = x.split('/') if x else []
    while p and x and p[0] == x[0]:
        p.pop(0)
        x.pop(0)
    return '/'.join(['..' for i in x] + p)

class PathObject(XcodeObject):
    def __init__(self, path, root=None, name=None):
        self.parent = None
        path = path or ''
        self.abspath = path
        self.name = name
        if root is None:
            if path.startswith('/'):
                root = '<absolute>'
            else:
                root = 'SOURCE_ROOT'
        self.root = root
        self.make_relative()
    def make_relative(self):
        """Make the path of this object relative, if possible."""
        op = self.abspath
        if self.parent:
            pp = self.parent.abspath
            pr = self.parent.root
        elif self.root == 'SOURCE_ROOT':
            pp = ''
            pr = 'SOURCE_ROOT'
        else:
            pp = ''
            pr = ''
        if self.root == pr:
            self.path = relpath(op, pp)
            self.sourceTree = '<group>'
        else:
            self.path = op
            self.sourceTree = self.root

    def __cmp__(self, other):
        if isinstance(other, PathObject):
            x = cmp(self.CSORT, other.CSORT)
            if x: return x
            x = cmp(self.root, other.root)
            if x: return x
            x = cmp(self.abspath, other.abspath)
            if x: return x
            n = getattr(self, 'name', None)
            m = getattr(other, 'name', None)
            if n is None:
                if m is None:
                    return 0
                return -1
            else:
                if m is None:
                    return 1
                return cmp(n, m)
        return NotImplemented

class Group(PathObject):
    CSORT = 0
    ISA = 'PBXGroup'
    ATTRS = ['children', optional('name'), optional('path'), 'sourceTree']
    def __init__(self, path, root=None, name=None):
        PathObject.__init__(self, path, root=root, name=name)
        self.children = []
    def add(self, obj):
        if not isinstance(obj, PathObject):
            raise TypeError('Can only add PathObject to Group')
        if obj.parent is not None:
            raise ValueError('Can only add object to one group')
        self.children.append(obj)
        obj.parent = self
        obj.make_relative()

class FileRef(PathObject):
    CSORT = 1
    COMPACT = True
    ISA = 'PBXFileReference'
    ATTRS = [optional('fileEncoding'), optional('lastKnownFileType'),
             optional('explicitFileType'), optional('includeInIndex'),
             optional('name'), 'path', 'sourceTree']
    def __init__(self, path, root=None, name=None, etype=None):
        PathObject.__init__(self, path, root=root, name=name)
        ext = posixpath.splitext(path)[1]
        if etype is None:
            try:
                ftype = buildtool.path.EXTS[ext]
            except KeyError:
                ftype = ext[1:] if ext else ''
            ftype = TYPES[ftype]
            self.lastKnownFileType = ftype
            self.explicitFileType = None
        else:
            self.lastKnownFileType = None
            self.explicitFileType = etype
            ftype = etype
        self.includeInIndex = None
        if ftype.startswith('sourcecode.') or ftype.startswith('.text'):
            self.fileEncoding = 4
        else:
            self.fileEncoding = None
    @property
    def ftype(self):
        return self.explicitFileType or self.lastKnownFileType

class BuildFile(XcodeObject):
    COMPACT = True
    ISA = 'PBXBuildFile'
    ATTRS = ['fileRef']
    def __init__(self, f):
        self.fileRef = f

class BuildPhase(XcodeObject):
    ATTRS = ['buildActionMask', 'files', 'runOnlyForDeploymentPostprocessing']
    buildActionMask = 0x7fffffff
    runOnlyForDeploymentPostprocessing = 0
    def __init__(self):
        self.files = []
    def add(self, x):
        self.files.append(x)

class FrameworksBuildPhase(BuildPhase):
    ISA = 'PBXFrameworksBuildPhase'

class ResourcesBuildPhase(BuildPhase):
    ISA = 'PBXResourcesBuildPhase'

class SourcesBuildPhase(BuildPhase):
    ISA = 'PBXSourcesBuildPhase'

class ShellScriptBuildPhase(BuildPhase):
    ISA = 'PBXShellScriptBuildPhase'
    ATTRS = BuildPhase.ATTRS + ['inputPaths', 'outputPaths', 'shellPath', 'shellScript']
    shellPath = '/bin/sh'
    def __init__(self, script):
        BuildPhase.__init__(self)
        self.shellScript = script
        self.inputPaths = []
        self.outputPaths = []

class NativeTarget(XcodeObject):
    ISA = 'PBXNativeTarget'
    ATTRS = ['buildConfigurationList', 'buildPhases',
             'buildRules', 'dependencies', 'name',
             'productName', 'productReference', 'productType']
    def __init__(self, ttype, name, productName=None):
        productName = productName or name
        self.buildConfigurationList = None # Object
        self.buildPhases = [] # Object list
        self.buildRules = []
        self.dependencies = []
        self.name = name
        self.productName = productName
        product = FileRef(productName + '.app', root='BUILT_PRODUCTS_DIR',
                          etype='wrapper.application')
        self.productReference = product
        self.productType = ttype<commit_msg>Remove environment vars from Xcode build log<commit_after>import buildtool.path
import buildtool.plist
import random
import posixpath
import sys
import cStringIO

TYPES = {
    'c': 'sourcecode.c.c',
    'cxx': 'sourcecode.cpp.cpp',
    'h': 'sourcecode.c.h',
    'hxx': 'sourcecode.cpp.h',
    'm': 'sourcecode.c.objc',
    'framework': 'wrapper.framework',
    'xib': 'file.xib',
    'plist': 'text.plist.xml',
}

RESOURCE_BUILD = set([
    'file.xib',
])

SOURCE_BUILD = set([
    'sourcecode.c.c',
    'sourcecode.cpp.cpp',
    'sourcecode.c.objc',
])

FRAMEWORK_BUILD = set([
    'wrapper.framework',
])

ISAS = [
    'PBXBuildFile',
    'PBXFileReference',
    'PBXFrameworksBuildPhase',
    'PBXGroup',
    'PBXNativeTarget',
    'PBXProject',
    'PBXResourcesBuildPhase',
    'PBXShellScriptBuildPhase',
    'PBXSourcesBuildPhase',
    'XCBuildConfiguration',
    'XCConfigurationList',
]

# UNSET = token('UNSET')

class optional(object):
    def __init__(self, what):
        self.what = what
    def __repr__(self):
        return 'optional(%s)' % repr(self.what)

class XcodeObject(object):
    COMPACT = False

    def _allobjs(self, dest):
        if self in dest:
            return
        dest.add(self)
        for attr in self.ATTRS:
            if isinstance(attr, optional):
                attr = attr.what
            v = getattr(self, attr)
            if isinstance(v, list):
                for x in v:
                    if isinstance(x, XcodeObject):
                        x._allobjs(dest)
            elif isinstance(v, dict):
                for x in v.itervalues():
                    if isinstance(x, XcodeObject):
                        x._allobjs(dest)
            elif isinstance(v, XcodeObject):
                v._allobjs(dest)

    def allobjs(self):
        objs = set([])
        self._allobjs(objs)
        return objs

    def emit(self, w, ids):
        def objemit(x):
            if isinstance(x, dict):
                w.start_dict()
                for k, v in x.iteritems():
                    w.write_key(k)
                    objemit(v)
                w.end_dict()
            elif isinstance(x, list):
                w.start_list()
                for v in x:
                    objemit(v)
                w.end_list()
            elif isinstance(x, XcodeObject):
                w.write_object(ids[x])
            else:
                w.write_object(x)
        w.write_key(ids[self])
        w.start_dict(self.COMPACT)
        w.write_pair('isa', self.ISA)
        for attr in self.ATTRS:
            if isinstance(attr, optional):
                attr = attr.what
                isoptional = True
            else:
                isoptional = False
            val = getattr(self, attr)
            if val is None:
                if not isoptional:
                    raise Exception('Missing %r attribiute from %r class' %
                        (attr, self.__class__.__name__))
            else:
                w.write_key(attr)
                objemit(val)
        w.end_dict()

def relpath(p, x):
    # This function doesn't work for all inputs
    p = p.split('/') if p else []
    x = x.split('/') if x else []
    while p and x and p[0] == x[0]:
        p.pop(0)
        x.pop(0)
    return '/'.join(['..' for i in x] + p)

class PathObject(XcodeObject):
    def __init__(self, path, root=None, name=None):
        self.parent = None
        path = path or ''
        self.abspath = path
        self.name = name
        if root is None:
            if path.startswith('/'):
                root = '<absolute>'
            else:
                root = 'SOURCE_ROOT'
        self.root = root
        self.make_relative()
    def make_relative(self):
        """Make the path of this object relative, if possible."""
        op = self.abspath
        if self.parent:
            pp = self.parent.abspath
            pr = self.parent.root
        elif self.root == 'SOURCE_ROOT':
            pp = ''
            pr = 'SOURCE_ROOT'
        else:
            pp = ''
            pr = ''
        if self.root == pr:
            self.path = relpath(op, pp)
            self.sourceTree = '<group>'
        else:
            self.path = op
            self.sourceTree = self.root

    def __cmp__(self, other):
        if isinstance(other, PathObject):
            x = cmp(self.CSORT, other.CSORT)
            if x: return x
            x = cmp(self.root, other.root)
            if x: return x
            x = cmp(self.abspath, other.abspath)
            if x: return x
            n = getattr(self, 'name', None)
            m = getattr(other, 'name', None)
            if n is None:
                if m is None:
                    return 0
                return -1
            else:
                if m is None:
                    return 1
                return cmp(n, m)
        return NotImplemented

class Group(PathObject):
    CSORT = 0
    ISA = 'PBXGroup'
    ATTRS = ['children', optional('name'), optional('path'), 'sourceTree']
    def __init__(self, path, root=None, name=None):
        PathObject.__init__(self, path, root=root, name=name)
        self.children = []
    def add(self, obj):
        if not isinstance(obj, PathObject):
            raise TypeError('Can only add PathObject to Group')
        if obj.parent is not None:
            raise ValueError('Can only add object to one group')
        self.children.append(obj)
        obj.parent = self
        obj.make_relative()

class FileRef(PathObject):
    CSORT = 1
    COMPACT = True
    ISA = 'PBXFileReference'
    ATTRS = [optional('fileEncoding'), optional('lastKnownFileType'),
             optional('explicitFileType'), optional('includeInIndex'),
             optional('name'), 'path', 'sourceTree']
    def __init__(self, path, root=None, name=None, etype=None):
        PathObject.__init__(self, path, root=root, name=name)
        ext = posixpath.splitext(path)[1]
        if etype is None:
            try:
                ftype = buildtool.path.EXTS[ext]
            except KeyError:
                ftype = ext[1:] if ext else ''
            ftype = TYPES[ftype]
            self.lastKnownFileType = ftype
            self.explicitFileType = None
        else:
            self.lastKnownFileType = None
            self.explicitFileType = etype
            ftype = etype
        self.includeInIndex = None
        if ftype.startswith('sourcecode.') or ftype.startswith('.text'):
            self.fileEncoding = 4
        else:
            self.fileEncoding = None
    @property
    def ftype(self):
        return self.explicitFileType or self.lastKnownFileType

class BuildFile(XcodeObject):
    COMPACT = True
    ISA = 'PBXBuildFile'
    ATTRS = ['fileRef']
    def __init__(self, f):
        self.fileRef = f

class BuildPhase(XcodeObject):
    ATTRS = ['buildActionMask', 'files', 'runOnlyForDeploymentPostprocessing']
    buildActionMask = 0x7fffffff
    runOnlyForDeploymentPostprocessing = 0
    def __init__(self):
        self.files = []
    def add(self, x):
        self.files.append(x)

class FrameworksBuildPhase(BuildPhase):
    ISA = 'PBXFrameworksBuildPhase'

class ResourcesBuildPhase(BuildPhase):
    ISA = 'PBXResourcesBuildPhase'

class SourcesBuildPhase(BuildPhase):
    ISA = 'PBXSourcesBuildPhase'

class ShellScriptBuildPhase(BuildPhase):
    ISA = 'PBXShellScriptBuildPhase'
    ATTRS = BuildPhase.ATTRS + ['inputPaths', 'outputPaths', 'shellPath', 'shellScript',
                                'showEnvVarsInLog']
    shellPath = '/bin/sh'
    showEnvVarsInLog = 0
    def __init__(self, script):
        BuildPhase.__init__(self)
        self.shellScript = script
        self.inputPaths = []
        self.outputPaths = []

class NativeTarget(XcodeObject):
    ISA = 'PBXNativeTarget'
    ATTRS = ['buildConfigurationList', 'buildPhases',
             'buildRules', 'dependencies', 'name',
             'productName', 'productReference', 'productType']
    def __init__(self, ttype, name, productName=None):
        productName = productName or name
        self.buildConfigurationList = None # Object
        self.buildPhases = [] # Object list
        self.buildRules = []
        self.dependencies = []
        self.name = name
        self.productName = productName
        product = FileRef(productName + '.app', root='BUILT_PRODUCTS_DIR',
                          etype='wrapper.application')
        self.productReference = product
        self.productType = ttype
{'fraction_non_alphanumeric': 0.1111243166151652, 'fraction_numerical': 0.0018421678155455194, 'mean_word_length': 1.803431617524571, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2614231', 'n_tokens_mistral': 5065, 'n_tokens_neox': 4908, 'n_words': 1198}