<commit_before>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2015 <NAME> for Tjaldur Software Governance Solutions
## Copyright 2015 Black Duck Software, Inc. All Rights Reserved.
## Licensed under Apache 2.0, see LICENSE file for details

'''
Convert BAT databases from SQLite to PostgreSQL
'''

import os, sys, sqlite3, datetime, multiprocessing
import psycopg2
from optparse import OptionParser

def insertintopostgresql((sqlitedatabase, tablename, preparedstatement, execquery)):
	print "importing %s" % tablename, datetime.datetime.utcnow().isoformat()
	sqliteconn = sqlite3.connect(sqlitedatabase)
	sqlitecursor = sqliteconn.cursor()
	sqlitecursor.execute("PRAGMA synchronous=off")
	postgresqlconn = psycopg2.connect("dbname=bat user=bat password=<PASSWORD>")
	postgresqlcursor = postgresqlconn.cursor()
	postgresqlcursor.execute('set synchronous_commit=off')

	selectquery = "select distinct * from %s" % tablename
	sqlitecursor.execute(selectquery)
	data = sqlitecursor.fetchmany(10000)
	postgresqlcursor.execute(preparedstatement)
	while data != []:
		postgresqlcursor.executemany(execquery, data)
		data = sqlitecursor.fetchmany(10000)

	postgresqlconn.commit()
	postgresqlcursor.close()
	postgresqlconn.close()
	sqlitecursor.close()
	sqliteconn.close()
	print "importing %s finished" % tablename, datetime.datetime.utcnow().isoformat()

def main(argv):
	parser = OptionParser()
	parser.add_option("-d", "--database", action="store", dest="sqlitedb", help="path to SQLite database file", metavar="FILE")
	parser.add_option("-l", "--licensedatabase", action="store", dest="licensesqlitedb", help="path to SQLite license database file", metavar="FILE")
	parser.add_option("-f", "--filedatabase", action="store", dest="filesqlitedb", help="path to SQLite license database file", metavar="FILE")
	parser.add_option("-k", "--kernelcache", action="store", dest="kernelcachedb", help="path to SQLite kernel cache database file", metavar="FILE")
	(options, args) = parser.parse_args()
	if options.sqlitedb == None:
		parser.error("Specify SQLite database file")

	if options.kernelcachedb == None:
		parser.error("Specify SQLite kernelcache database file")

	if not os.path.exists(options.sqlitedb):
		print >>sys.stderr, "SQLite database file does not exist, exiting"
		sys.exit(1)

	if options.filesqlitedb != None:
		if not os.path.exists(options.filesqlitedb):
			print >>sys.stderr, "SQLite file database file specified, but does not exist, exiting"
			sys.exit(1)
	## first set up sqlite cursor
	sqliteconn = sqlite3.connect(options.sqlitedb)
	sqlitecursor = sqliteconn.cursor()

	## the set up PostgreSQL cursor
	## TODO: make configurable
	postgresqlconn = psycopg2.connect("dbname=bat user=bat password=<PASSWORD>")
	postgresqlcursor = postgresqlconn.cursor()

	indexes = ["processed_index",
                   "processed_checksum",
                   "processed_origin",
                   "processedfile_package_checksum_index",
                   "processedfile_package_version_index",
                   "processedfile_filename_index",
                   "stringidentifier_index",
                   "extracted_hash_index",
                   "extracted_language_index",
                   "function_index",
                   "functionname_index",
                   "functionname_language",
                   "name_checksum_index",
                   "name_name_index",<commit_msg>close unneeded cursor<commit_after>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2015 <NAME> for Tjaldur Software Governance Solutions
## Copyright 2015 Black Duck Software, Inc. All Rights Reserved.
## Licensed under Apache 2.0, see LICENSE file for details

'''
Convert BAT databases from SQLite to PostgreSQL
'''

import os, sys, sqlite3, datetime, multiprocessing
import psycopg2
from optparse import OptionParser

def insertintopostgresql((sqlitedatabase, tablename, preparedstatement, execquery)):
	print "importing %s" % tablename, datetime.datetime.utcnow().isoformat()
	sqliteconn = sqlite3.connect(sqlitedatabase)
	sqlitecursor = sqliteconn.cursor()
	sqlitecursor.execute("PRAGMA synchronous=off")
	postgresqlconn = psycopg2.connect("dbname=bat user=bat password=<PASSWORD>")
	postgresqlcursor = postgresqlconn.cursor()
	postgresqlcursor.execute('set synchronous_commit=off')

	selectquery = "select distinct * from %s" % tablename
	sqlitecursor.execute(selectquery)
	data = sqlitecursor.fetchmany(10000)
	postgresqlcursor.execute(preparedstatement)
	while data != []:
		postgresqlcursor.executemany(execquery, data)
		data = sqlitecursor.fetchmany(10000)

	postgresqlconn.commit()
	postgresqlcursor.close()
	postgresqlconn.close()
	sqlitecursor.close()
	sqliteconn.close()
	print "importing %s finished" % tablename, datetime.datetime.utcnow().isoformat()

def main(argv):
	parser = OptionParser()
	parser.add_option("-d", "--database", action="store", dest="sqlitedb", help="path to SQLite database file", metavar="FILE")
	parser.add_option("-l", "--licensedatabase", action="store", dest="licensesqlitedb", help="path to SQLite license database file", metavar="FILE")
	parser.add_option("-f", "--filedatabase", action="store", dest="filesqlitedb", help="path to SQLite license database file", metavar="FILE")
	parser.add_option("-k", "--kernelcache", action="store", dest="kernelcachedb", help="path to SQLite kernel cache database file", metavar="FILE")
	(options, args) = parser.parse_args()
	if options.sqlitedb == None:
		parser.error("Specify SQLite database file")

	if options.kernelcachedb == None:
		parser.error("Specify SQLite kernelcache database file")

	if not os.path.exists(options.sqlitedb):
		print >>sys.stderr, "SQLite database file does not exist, exiting"
		sys.exit(1)

	if options.filesqlitedb != None:
		if not os.path.exists(options.filesqlitedb):
			print >>sys.stderr, "SQLite file database file specified, but does not exist, exiting"
			sys.exit(1)

	## set up PostgreSQL cursor
	## TODO: make configurable
	postgresqlconn = psycopg2.connect("dbname=bat user=bat password=<PASSWORD>")
	postgresqlcursor = postgresqlconn.cursor()

	indexes = ["processed_index",
                   "processed_checksum",
                   "processed_origin",
                   "processedfile_package_checksum_index",
                   "processedfile_package_version_index",
                   "processedfile_filename_index",
                   "stringidentifier_index",
                   "extracted_hash_index",
                   "extracted_language_index",
                   "function_index",
                   "functionname_index",
                   "functionname_language",
                   "name_checksum_index",
                   "name_name_index",
{'fraction_non_alphanumeric': 0.11614730878186968, 'fraction_numerical': 0.008200387654689131, 'mean_word_length': 4.248826291079812, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '964793', 'n_tokens_mistral': 2081, 'n_tokens_neox': 1950, 'n_words': 463}