<commit_before>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2012-2015 <NAME> for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

'''
This is a plugin for the Binary Analysis Tool. Its purpose is to determine the
package a file belongs to based on the name of a package. This information is
mined from distributions like Fedora and Debian.

This scan should be run as a leaf scan.
'''

import os, os.path, sqlite3, sys, subprocess, copy
import xml.dom.minidom

def filename2package(path, tags, blacklist=[], scanenv={}, scandebug=False):
	if not scanenv.has_key('BAT_PACKAGE_DB'):
		return
	## open the database containing the mapping of filenames to package
	conn = sqlite3.connect(scanenv.get('BAT_PACKAGE_DB'))
	c = conn.cursor()
	## select the packages that are available. It would be better to also have the directory
	## name available, so we should get rid of 'path' and use something else that is better
	## suited
	c.execute("select distinct package, packageversion, source from file where filename = '%s'" % (os.path.basename(path),))
	res = c.fetchall()
	## TODO: filter results, only return files that are not in tons of packages
	if res != []:
		return (['file2package'], res)
	return None

def xmlprettyprint(res, root, scanenv={}):
	topnode = root.createElement("filelist")
	#tags = ['packagename', 'version', 'distribution']
	tags = ['packagename', 'distribution']
	for i in res:
		tmpnode = root.createElement('packageguess')
		for j in range(0,len(tags)):
			tagnode = root.createElement(tags[j])
			tagnodetext = xml.dom.minidom.Text()
			tagnodetext.data = i[j]
			tagnode.appendChild(tagnodetext)
			tmpnode.appendChild(tagnode)
		topnode.appendChild(tmpnode)
	return topnode

def file2packagesetup(scanenv, debug=False):
	newenv = copy.deepcopy(scanenv)

	## Is the package database defined?
	if not scanenv.has_key('BAT_PACKAGE_DB'):
		return (False, None)
<commit_msg>prepare for multiple db backends<commit_after>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2012-2015 <NAME> for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

'''
This is a plugin for the Binary Analysis Tool. Its purpose is to determine the
package a file belongs to based on the name of a package. This information is
mined from distributions like Fedora and Debian.

This scan should be run as a leaf scan.
'''

import os, os.path, sqlite3, sys, subprocess, copy
import xml.dom.minidom

def filename2package(path, tags, blacklist=[], scanenv={}, scandebug=False):
	if not scanenv.has_key('BAT_PACKAGE_DB'):
		return
	## open the database containing the mapping of filenames to package
	conn = sqlite3.connect(scanenv.get('BAT_PACKAGE_DB'))
	c = conn.cursor()
	## select the packages that are available. It would be better to also have the directory
	## name available, so we should get rid of 'path' and use something else that is better
	## suited
	c.execute("select distinct package, packageversion, source from file where filename = '%s'" % (os.path.basename(path),))
	res = c.fetchall()
	## TODO: filter results, only return files that are not in tons of packages
	if res != []:
		return (['file2package'], res)
	return None

def xmlprettyprint(res, root, scanenv={}):
	topnode = root.createElement("filelist")
	#tags = ['packagename', 'version', 'distribution']
	tags = ['packagename', 'distribution']
	for i in res:
		tmpnode = root.createElement('packageguess')
		for j in range(0,len(tags)):
			tagnode = root.createElement(tags[j])
			tagnodetext = xml.dom.minidom.Text()
			tagnodetext.data = i[j]
			tagnode.appendChild(tagnodetext)
			tmpnode.appendChild(tagnode)
		topnode.appendChild(tmpnode)
	return topnode

def file2packagesetup(scanenv, debug=False):
	if not 'DBBACKEND' in scanenv:
		return (False, None)
	if scanenv['DBBACKEND'] == 'sqlite3':
		return file2packagesetup_sqlite3(scanenv, debug)
	return (False, None)

## checks specific for sqlite3 databases
def file2packagesetup_sqlite3(scanenv, debug=False):
	newenv = copy.deepcopy(scanenv)

	## Is the package database defined?
	if not scanenv.has_key('BAT_PACKAGE_DB'):
		return (False, None)

{'fraction_non_alphanumeric': 0.10896057347670252, 'fraction_numerical': 0.00908004778972521, 'mean_word_length': 4.997134670487106, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '321089', 'n_tokens_mistral': 1369, 'n_tokens_neox': 1304, 'n_words': 459}