<commit_before>#!/usr/bin/python
#-*- coding: utf-8 -*-

## Binary Analysis Tool
## Copyright 2011-2014 <NAME> for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

'''
This file contains code to extract identifiers (strings, function names,
variable names, etc.) from binaries and make them available for further
processing by various other scans.

Configuration parameters for databases are:

BAT_NAMECACHE_$LANGUAGE :: location of database containing cached
                           function names and variable names per package.
                           This is only used to look up Linux kernel functions.
'''

import string, re, os, os.path, sys, tempfile, shutil, copy
import sqlite3
import subprocess
import extractor

## mapping of names for databases per language
namecacheperlanguage = { 'C':       'BAT_NAMECACHE_C'
                       , 'Java':    'BAT_NAMECACHE_JAVA'
                       }

## some regular expressions for Java, precompiled
rejavaclass = re.compile("This class: \d+=([\w\.$]+), super")
rejavaattribute = re.compile("Attribute \"SourceFile\", length:\d+, #\d+=\"([\w\.]+)\"")
rejavafield = re.compile("Field name:\"([\w$]+)\"")
rejavamethod= re.compile("Method name:\"([\w$]+)\"")
rejavastring = re.compile("#\d+: String \d+=\"")

## Main part of the scan
##
## 1. extract string constants, function names, variable names, etc.
## 2. Then run strings through computeScore, that queries the database and does
## funky statistics as described in our paper.
##
## Original code (in Perl) was written by <NAME>.
## Reimplementation in Python done by <NAME>.
##
def searchGeneric(filepath, tags, blacklist=[], offsets={}, debug=False, envvars=None, unpacktempdir=None):
	filesize = os.stat(filepath).st_size
	## whole file is blacklisted, so no need to scan
	if extractor.inblacklist(0, blacklist) == filesize:
		return None

	scanenv = os.environ.copy()
	if envvars != None:
		for en in envvars.split(':'):
			try:
				(envname, envvalue) = en.split('=')
				scanenv[envname] = envvalue
			except Exception, e:
				pass

	## Only consider strings that are len(stringcutoff) or larger
	## it is *very* important to keep this value in sync with the
	## database creation scripts!
	if scanenv.has_key('BAT_STRING_CUTOFF'):
		try:
			stringcutoff = int(scanenv['BAT_STRING_CUTOFF'])
			if stringcutoff < 1:
				stringcutoff = 5
		except ValueError, e:
			stringcutoff = 5
	else:
		stringcutoff = 5

	## use extra information for a few file types
	## * ELF files
	## * bFLT files
	## * Java class files + Dalvik VM files
	## * Windows executables and libraries
	## * Mono/.NET files
	## * Flash/ActionScript

	if 'elf' in tags:
		language = 'C'
	elif "java" in tags:
		language = 'Java'
	elif "dalvik" in tags:
		language = 'Java'
	else:
		## Treat everything else as C
		## TODO check for Javascript, .NET and others
		## JavaScript
		language='C'

	linuxkernel = False

	if 'linuxkernel' in tags:
		linuxkernel = True

	if language == 'C':
		res = extractC(filepath, tags, scanenv, filesize, stringcutoff, linuxkernel, blacklist, debug, unpacktempdir)
		if res == None:
			return None
		(lines, cmeta) = res
		if linuxkernel:
			res = extractKernelData(lines, filepath, scanenv)
			if res != None:
				if res.has_key('kernelfunctions'):
					if res['kernelfunctions'] != []:
						cmeta['kernelfunctions'] = copy.deepcopy(res['kernelfunctions'])
		return (['identifier'], (lines, cmeta, language))
	elif language == 'Java':
		res = extractJava(filepath, tags, scanenv, filesize, stringcutoff, blacklist, debug, unpacktempdir)
		if res == None:
			return None
		(lines, javameta) = res
		return (['identifier'], (lines, javameta, language))

def extractC(filepath, tags, scanenv, filesize, stringcutoff, linuxkernel, blacklist=[], debug=False, unpacktempdir=None):
	## special var to indicate whether or not the file is a Linux kernel
	## image. If so extra checks can be done.
	kernelsymbols = []
	cmeta = {}

	## ELF files are always scanned as a whole. Sometimes there are sections that
	## contain compressed data, like .gnu_debugdata which should not trigger the
	## black list.<commit_msg>remove bogus results<commit_after>#!/usr/bin/python
#-*- coding: utf-8 -*-

## Binary Analysis Tool
## Copyright 2011-2014 <NAME> for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

'''
This file contains code to extract identifiers (strings, function names,
variable names, etc.) from binaries and make them available for further
processing by various other scans.

Configuration parameters for databases are:

BAT_NAMECACHE_$LANGUAGE :: location of database containing cached
                           function names and variable names per package.
                           This is only used to look up Linux kernel functions.
'''

import string, re, os, os.path, sys, tempfile, shutil, copy
import sqlite3
import subprocess
import extractor

## mapping of names for databases per language
namecacheperlanguage = { 'C':       'BAT_NAMECACHE_C'
                       , 'Java':    'BAT_NAMECACHE_JAVA'
                       }

## some regular expressions for Java, precompiled
rejavaclass = re.compile("This class: \d+=([\w\.$]+), super")
rejavaattribute = re.compile("Attribute \"SourceFile\", length:\d+, #\d+=\"([\w\.]+)\"")
rejavafield = re.compile("Field name:\"([\w$]+)\"")
rejavamethod= re.compile("Method name:\"([\w$]+)\"")
rejavastring = re.compile("#\d+: String \d+=\"")

## Main part of the scan
##
## 1. extract string constants, function names, variable names, etc.
## 2. Then run strings through computeScore, that queries the database and does
## funky statistics as described in our paper.
##
## Original code (in Perl) was written by <NAME>.
## Reimplementation in Python done by <NAME>.
##
def searchGeneric(filepath, tags, blacklist=[], offsets={}, debug=False, envvars=None, unpacktempdir=None):
	filesize = os.stat(filepath).st_size
	## whole file is blacklisted, so no need to scan
	if extractor.inblacklist(0, blacklist) == filesize:
		return None

	scanenv = os.environ.copy()
	if envvars != None:
		for en in envvars.split(':'):
			try:
				(envname, envvalue) = en.split('=')
				scanenv[envname] = envvalue
			except Exception, e:
				pass

	## Only consider strings that are len(stringcutoff) or larger
	## it is *very* important to keep this value in sync with the
	## database creation scripts!
	if scanenv.has_key('BAT_STRING_CUTOFF'):
		try:
			stringcutoff = int(scanenv['BAT_STRING_CUTOFF'])
			if stringcutoff < 1:
				stringcutoff = 5
		except ValueError, e:
			stringcutoff = 5
	else:
		stringcutoff = 5

	## use extra information for a few file types
	## * ELF files
	## * bFLT files
	## * Java class files + Dalvik VM files
	## * Windows executables and libraries
	## * Mono/.NET files
	## * Flash/ActionScript

	if 'elf' in tags:
		language = 'C'
	elif "java" in tags:
		language = 'Java'
	elif "dalvik" in tags:
		language = 'Java'
	else:
		## Treat everything else as C
		## TODO check for Javascript, .NET and others
		## JavaScript
		language='C'

	linuxkernel = False

	if 'linuxkernel' in tags:
		linuxkernel = True

	if language == 'C':
		res = extractC(filepath, tags, scanenv, filesize, stringcutoff, linuxkernel, blacklist, debug, unpacktempdir)
		if res == None:
			return None
		(lines, cmeta) = res
		if lines == []:
			empty = True
			for c in cmeta.keys():
				if len(cmeta[c]) != 0:
					empty = False
					break
			if empty:
				return None
		if linuxkernel:
			res = extractKernelData(lines, filepath, scanenv)
			if res != None:
				if res.has_key('kernelfunctions'):
					if res['kernelfunctions'] != []:
						cmeta['kernelfunctions'] = copy.deepcopy(res['kernelfunctions'])
		return (['identifier'], (lines, cmeta, language))
	elif language == 'Java':
		res = extractJava(filepath, tags, scanenv, filesize, stringcutoff, blacklist, debug, unpacktempdir)
		if res == None:
			return None
		(lines, javameta) = res
		return (['identifier'], (lines, javameta, language))

def extractC(filepath, tags, scanenv, filesize, stringcutoff, linuxkernel, blacklist=[], debug=False, unpacktempdir=None):
	## special var to indicate whether or not the file is a Linux kernel
	## image. If so extra checks can be done.
	kernelsymbols = []
	cmeta = {}

	## ELF files are always scanned as a whole. Sometimes there are sections that
	## contain compressed data, like .gnu_debugdata which should not trigger the
	## black list.
{'fraction_non_alphanumeric': 0.12148653416441256, 'fraction_numerical': 0.004586616488298247, 'mean_word_length': 3.766816143497758, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '474400', 'n_tokens_mistral': 2821, 'n_tokens_neox': 2580, 'n_words': 923}