<commit_before>import os
import sys
import subprocess as sp
import re
from .utils import llvmCompilerPathEnv

from .popenwrapper import Popen

from .utils import elfSectionName
from .utils import darwinSegmentName
from .utils import darwinSectionName

from .filetype import FileType

from .logconfig import *

import logging
import pprint
import tempfile
import shutil
import argparse

"""
(Fix: 2016/02/16: __LLVM is now used by MacOS's ld so we changed the segment name to __WLLVM).

"""



# Python 2 does not have exceptions automatically
# imported whereas python 3 does. Handle this
try:
    dir(UnicodeDecodeError)
except NameError:
  import exceptions

bitCodeArchiveExtension='bca'
moduleExtension='bc'

def getSectionSizeAndOffset(sectionName, filename):
    """Returns the size and offset of the section, both in bytes.

    Use objdump on the provided binary; parse out the fields
    to find the given section.  Parses the output,and
    extracts thesize and offset of that section (in bytes).
    """
    objdumpCmd = ['objdump', '-h', '-w', filename]
    objdumpProc = Popen(objdumpCmd, stdout=sp.PIPE)

    objdumpOutput = objdumpProc.communicate()[0]
    if objdumpProc.returncode != 0:
        logging.error('Could not dump %s' % filename)
        sys.exit(-1)

    for line in [l.decode('utf-8') for l in objdumpOutput.splitlines()] :
        fields = line.split()
        if len(fields) <= 7:
            continue
        if fields[1] != sectionName:
            continue
        try:
            idx = int(fields[0])
            size = int(fields[2], 16)
            offset = int(fields[5], 16)
            return (size, offset)
        except ValueError:
            continue

    # The needed section could not be found
    logging.warning('Could not find "{0}" ELF section in "{1}", so skipping this entry.'.format(sectionName,filename))
    return None

def getSectionContent(size, offset, filename):
    """Reads the entire content of an ELF section into a string."""
    with open(filename, mode='rb') as f:
        f.seek(offset)
        d = ''
        try:
            c = f.read(size)
            d = c.decode('utf-8')
        except UnicodeDecodeError:
            logging.error('Failed to read section containing:')
            print(c)
            raise
        # The linker pads sections with null bytes; our real data
        # cannot have null bytes because it is just text.  Discard
        # nulls.
        return d.replace('\0', '')

def appleQualityControl(lines):
    hexdata = re.compile('^[0-9a-f\s]+$', re.IGNORECASE)
    flines = []
    for line in lines:
        if hexdata.match(line):
            flines.append(line)
    return flines
    
def extract_section_darwin(inputFile):
    """Extracts the section as a string, the darwin version.

    Uses otool to extract the section, then processes it
    to a usable state.

    Quality control at Apple seems lax. The -X flags effect on the
    output seems to be deteriorating. So there is a hack here to
    repair that.
    """

    otoolCmd  = ['otool', '-X', '-s', darwinSegmentName, darwinSectionName, inputFile]
    otoolProc = Popen(otoolCmd, stdout=sp.PIPE)

    otoolOutput = otoolProc.communicate()[0]
    if otoolProc.returncode != 0:
        logging.error('otool failed on %s' % inputFile)
        sys.exit(-1)

    lines = otoolOutput.splitlines()

    lines = appleQualityControl(lines)

    octets = []
    for line in lines:
        (_, octetline) = line.split('\t')
        octets.extend(octetline.split())
    octets = ''.join(octets)
    contents = octets.decode('hex').splitlines()
    if not contents:
        logging.error('{0} contained no {1} segment'.format(inputFile, darwinSegmentName))
    return contents<commit_msg>Cleaner and more verbose fix to otool problems.<commit_after>import os
import sys
import subprocess as sp
import re
from .utils import llvmCompilerPathEnv

from .popenwrapper import Popen

from .utils import elfSectionName
from .utils import darwinSegmentName
from .utils import darwinSectionName

from .filetype import FileType

from .logconfig import *

import logging
import pprint
import tempfile
import shutil
import argparse

"""
(Fix: 2016/02/16: __LLVM is now used by MacOS's ld so we changed the segment name to __WLLVM).

"""



# Python 2 does not have exceptions automatically
# imported whereas python 3 does. Handle this
try:
    dir(UnicodeDecodeError)
except NameError:
  import exceptions

bitCodeArchiveExtension='bca'
moduleExtension='bc'

def getSectionSizeAndOffset(sectionName, filename):
    """Returns the size and offset of the section, both in bytes.

    Use objdump on the provided binary; parse out the fields
    to find the given section.  Parses the output,and
    extracts thesize and offset of that section (in bytes).
    """
    objdumpCmd = ['objdump', '-h', '-w', filename]
    objdumpProc = Popen(objdumpCmd, stdout=sp.PIPE)

    objdumpOutput = objdumpProc.communicate()[0]
    if objdumpProc.returncode != 0:
        logging.error('Could not dump %s' % filename)
        sys.exit(-1)

    for line in [l.decode('utf-8') for l in objdumpOutput.splitlines()] :
        fields = line.split()
        if len(fields) <= 7:
            continue
        if fields[1] != sectionName:
            continue
        try:
            idx = int(fields[0])
            size = int(fields[2], 16)
            offset = int(fields[5], 16)
            return (size, offset)
        except ValueError:
            continue

    # The needed section could not be found
    logging.warning('Could not find "{0}" ELF section in "{1}", so skipping this entry.'.format(sectionName,filename))
    return None

def getSectionContent(size, offset, filename):
    """Reads the entire content of an ELF section into a string."""
    with open(filename, mode='rb') as f:
        f.seek(offset)
        d = ''
        try:
            c = f.read(size)
            d = c.decode('utf-8')
        except UnicodeDecodeError:
            logging.error('Failed to read section containing:')
            print(c)
            raise
        # The linker pads sections with null bytes; our real data
        # cannot have null bytes because it is just text.  Discard
        # nulls.
        return d.replace('\0', '')

def extract_section_darwin(inputFile):
    """Extracts the section as a string, the darwin version.

    Uses otool to extract the section, then processes it
    to a usable state.

    Quality control at Apple seems lax. The -X flags effect on the
    output seems to be deteriorating. So there is a hack here to
    repair that.
    """
    retval = None
    
    otoolCmd  = ['otool', '-X', '-s', darwinSegmentName, darwinSectionName, inputFile]
    otoolProc = Popen(otoolCmd, stdout=sp.PIPE)

    otoolOutput = otoolProc.communicate()[0]
    if otoolProc.returncode != 0:
        logging.error('otool failed on %s' % inputFile)
        sys.exit(-1)

    lines = otoolOutput.splitlines()

    try:
        octets = []
        for line in lines:
            splat = line.split('\t')
            if len(splat) is 2:
                (_, octetline) = splat
                octets.extend(octetline.split())
        octets = ''.join(octets)
        retval = octets.decode('hex').splitlines()
        if not retval:
            logging.error('{0} contained no {1} segment'.format(inputFile, darwinSegmentName))
    except Exception as e:
        logging.error('{0}'.format(str(e)))
    return retval
{'fraction_non_alphanumeric': 0.09802869786777524, 'fraction_numerical': 0.009118948638862813, 'mean_word_length': 2.6540911317981384, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1262824', 'n_tokens_mistral': 2210, 'n_tokens_neox': 2146, 'n_words': 749}