<commit_before>#! /usr/bin/env python

import sys
import os
import re
import time
import stat
import profile
import sqlite3
import MySQLdb
import fileinput
import tempfile
import optparse

class DummyFilter:
	def match(self, dummy):
		return True

class EmitBase:
	def create_table(self, cols, name, type, unique):
		create = 'create table if not exists %s (id int %s' % (name, unique)

		sorted = [None] * len(cols)
		for x in cols.iterkeys():
			sorted[cols[x]] = x
		for x in sorted:
			create += (', %s %s' % (x, type))
		create += ');'
		return create

class EmitMysqlInfile(EmitBase):
	tmpfile_mode = stat.S_IREAD | stat.S_IROTH | stat.S_IWUSR

	def ex(self, args, tab, fname):
		res = os.fork()
		if res == 0:
			stmt = """load data infile '%s' into table %s fields terminated by ';'""" % (fname, tab)
			conn = MySQLdb.connect(**args)
			c = conn.cursor()
			c.execute(stmt)
			conn.commit()
			sys.exit(0)
		return res

	def __init__(self, options, tables, ctxcols, evcols):
		args = dict()
		if options.password:
			args['passwd'] = options.password
		if not options.host:
			options.host = 'localhost'
		args['user'] = options.user
		args['host'] = options.host
		args['db']   = options.database

		self.conn     = MySQLdb.connect(**args)
		self.ctxcols  = ctxcols
		self.evcols   = evcols
		self.options  = options
		self.ctxtab   = tables['ctx']
		self.evtab    = tables['ev']

		params = (tempfile.gettempdir(), os.sep, os.getpid())
		self.evfifo  = '%s%sstatev_ev_%d' % params
		self.ctxfifo = '%s%sstatev_ctx_%d' % params

		os.mkfifo(self.evfifo)
		os.mkfifo(self.ctxfifo)

		os.chmod(self.evfifo,  self.tmpfile_mode)
		os.chmod(self.ctxfifo, self.tmpfile_mode)

		c = self.conn.cursor()
		c.execute('drop table if exists ' + self.evtab)
		c.execute('drop table if exists ' + self.ctxtab)
		c.execute(self.create_table(self.ctxcols, self.ctxtab, 'char(80)', 'unique'))
		c.execute(self.create_table(self.evcols, self.evtab, 'double default null', ''))
		self.conn.commit()

		if options.verbose:
			print 'go for gold'

		self.pidev  = self.ex(args, self.evtab, self.evfifo)
		self.pidctx = self.ex(args, self.ctxtab, self.ctxfifo)

		if options.verbose:
			print "forked two mysql leechers: %d, %d" % (self.pidev, self.pidctx)

		self.evfile   = open(self.evfifo, 'w+t')
		self.ctxfile  = open(self.ctxfifo, 'w+t')

		if options.verbose:
			print 'fifo:  %s, %o' % (self.evfile.name, os.stat(self.evfile.name).st_mode)
			print 'fifo:  %s, %o' % (self.ctxfile.name, os.stat(self.ctxfile.name).st_mode)

	def ev(self, curr_id, evitems):
		field = ['\N'] * len(self.evcols)
		for key, val in evitems.iteritems():
			index = self.evcols[key]
			field[index] = val
		print >> self.evfile, ('%d;' % curr_id) + ';'.join(field)

	def ctx(self, curr_id, ctxitems):
 		field = ['\N'] * len(self.ctxcols)
 		for key, val in ctxitems.iteritems():
 			index = self.ctxcols[key]
 			field[index] = val
 		print >> self.ctxfile, ('%d;' % curr_id) + ';'.join(field)

	def commit(self):
		self.evfile.close()
		self.ctxfile.close()

		os.waitpid(self.pidev, 0)
		os.waitpid(self.pidctx, 0)

		os.unlink(self.evfile.name)
		os.unlink(self.ctxfile.name)


class EmitSqlite3(EmitBase):
	def __init__(self, options, tables, ctxcols, evcols):
		if os.path.isfile(options.database):
			os.unlink(options.database)<commit_msg>only import datatbases that are actually used<commit_after>#! /usr/bin/env python

import sys
import os
import re
import time
import stat
import fileinput
import tempfile
import optparse

class DummyFilter:
	def match(self, dummy):
		return True

class EmitBase:
	def create_table(self, cols, name, type, unique):
		create = 'create table if not exists %s (id int %s' % (name, unique)

		sorted = [None] * len(cols)
		for x in cols.iterkeys():
			sorted[cols[x]] = x
		for x in sorted:
			create += (', %s %s' % (x, type))
		create += ');'
		return create

class EmitMysqlInfile(EmitBase):
	tmpfile_mode = stat.S_IREAD | stat.S_IROTH | stat.S_IWUSR

	def ex(self, args, tab, fname):
		res = os.fork()
		if res == 0:
			stmt = """load data infile '%s' into table %s fields terminated by ';'""" % (fname, tab)
			conn = MySQLdb.connect(**args)
			c = conn.cursor()
			c.execute(stmt)
			conn.commit()
			sys.exit(0)
		return res

	def __init__(self, options, tables, ctxcols, evcols):
		import MySQLdb

		args = dict()
		if options.password:
			args['passwd'] = options.password
		if not options.host:
			options.host = 'localhost'
		args['user'] = options.user
		args['host'] = options.host
		args['db']   = options.database

		self.conn     = MySQLdb.connect(**args)
		self.ctxcols  = ctxcols
		self.evcols   = evcols
		self.options  = options
		self.ctxtab   = tables['ctx']
		self.evtab    = tables['ev']

		params = (tempfile.gettempdir(), os.sep, os.getpid())
		self.evfifo  = '%s%sstatev_ev_%d' % params
		self.ctxfifo = '%s%sstatev_ctx_%d' % params

		os.mkfifo(self.evfifo)
		os.mkfifo(self.ctxfifo)

		os.chmod(self.evfifo,  self.tmpfile_mode)
		os.chmod(self.ctxfifo, self.tmpfile_mode)

		c = self.conn.cursor()
		c.execute('drop table if exists ' + self.evtab)
		c.execute('drop table if exists ' + self.ctxtab)
		c.execute(self.create_table(self.ctxcols, self.ctxtab, 'char(80)', 'unique'))
		c.execute(self.create_table(self.evcols, self.evtab, 'double default null', ''))
		self.conn.commit()

		if options.verbose:
			print 'go for gold'

		self.pidev  = self.ex(args, self.evtab, self.evfifo)
		self.pidctx = self.ex(args, self.ctxtab, self.ctxfifo)

		if options.verbose:
			print "forked two mysql leechers: %d, %d" % (self.pidev, self.pidctx)

		self.evfile   = open(self.evfifo, 'w+t')
		self.ctxfile  = open(self.ctxfifo, 'w+t')

		if options.verbose:
			print 'fifo:  %s, %o' % (self.evfile.name, os.stat(self.evfile.name).st_mode)
			print 'fifo:  %s, %o' % (self.ctxfile.name, os.stat(self.ctxfile.name).st_mode)

	def ev(self, curr_id, evitems):
		field = ['\N'] * len(self.evcols)
		for key, val in evitems.iteritems():
			index = self.evcols[key]
			field[index] = val
		print >> self.evfile, ('%d;' % curr_id) + ';'.join(field)

	def ctx(self, curr_id, ctxitems):
 		field = ['\N'] * len(self.ctxcols)
 		for key, val in ctxitems.iteritems():
 			index = self.ctxcols[key]
 			field[index] = val
 		print >> self.ctxfile, ('%d;' % curr_id) + ';'.join(field)

	def commit(self):
		self.evfile.close()
		self.ctxfile.close()

		os.waitpid(self.pidev, 0)
		os.waitpid(self.pidctx, 0)

		os.unlink(self.evfile.name)
		os.unlink(self.ctxfile.name)


class EmitSqlite3(EmitBase):
	def __init__(self, options, tables, ctxcols, evcols):
		import sqlite3

		if os.path.isfile(options.database):
			os.unlink(options.database)
{'fraction_non_alphanumeric': 0.15843960990247563, 'fraction_numerical': 0.0024006001500375095, 'mean_word_length': 3.7176220806794054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3475511', 'n_tokens_mistral': 2849, 'n_tokens_neox': 2611, 'n_words': 587}