<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import division
import csv
import time
import json
import jinja2
import requests
import functools

from PySide.QtCore import *

import plot
import motif
from libs import *
from db import *
from utils import *
from config import *
from statistics import *

class Worker(QObject):
	update_progress = Signal(int)
	update_message = Signal(str)
	finished = Signal()
	_db = None

	def __init__(self):
		super(Worker, self).__init__()

	@property
	def db(self):
		if self._db is None:
			self._db = Database()
		return self._db

	def build_fasta_index(self, fasta_id, fasta_path):
		'''
		build index for fasta file and write fasta sequence to database
		@para fasta_id int, the fasta file id in database
		@para fasta_path str, the file path of fasta
		@return Fasta object
		'''
		seqs = fasta.GzipFasta(fasta_path)
		sql = "SELECT 1 FROM seq WHERE name='%s' LIMIT 1" % seqs.keys[0]
		if not self.db.get_one(sql):
			rows = []
			for name in seqs.keys:
				row = (None, name, fasta_id, seqs.get_len(name), seqs.get_gc(name), seqs.get_ns(name))
				rows.append(row)
			self.db.insert("INSERT INTO seq VALUES (?,?,?,?,?,?)", rows)

		return seqs

	def emit_progress(self, percent):
		self.update_progress.emit(percent)
		time.sleep(0.001)

	def emit_message(self, msg):
		self.update_message.emit(msg)
		time.sleep(0.001)

	def emit_finish(self, msg):
		self.update_progress.emit(100)
		self.update_message.emit(msg)
		self.finished.emit()

	def process(self):
		pass

	def run(self):
		self.emit_progress(0)
		#try:
		self.process()
		#except Exception, e:
		#	self.emit_finish('Error: %s' % str(e))<commit_msg>close debug<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import division
import csv
import time
import json
import jinja2
import requests
import functools

from PySide.QtCore import *

import plot
import motif
from libs import *
from db import *
from utils import *
from config import *
from statistics import *

class Worker(QObject):
	update_progress = Signal(int)
	update_message = Signal(str)
	finished = Signal()
	_db = None

	def __init__(self):
		super(Worker, self).__init__()

	@property
	def db(self):
		if self._db is None:
			self._db = Database()
		return self._db

	def build_fasta_index(self, fasta_id, fasta_path):
		'''
		build index for fasta file and write fasta sequence to database
		@para fasta_id int, the fasta file id in database
		@para fasta_path str, the file path of fasta
		@return Fasta object
		'''
		seqs = fasta.GzipFasta(fasta_path)
		sql = "SELECT 1 FROM seq WHERE name='%s' LIMIT 1" % seqs.keys[0]
		if not self.db.get_one(sql):
			rows = []
			for name in seqs.keys:
				row = (None, name, fasta_id, seqs.get_len(name), seqs.get_gc(name), seqs.get_ns(name))
				rows.append(row)
			self.db.insert("INSERT INTO seq VALUES (?,?,?,?,?,?)", rows)

		return seqs

	def emit_progress(self, percent):
		self.update_progress.emit(percent)
		time.sleep(0.001)

	def emit_message(self, msg):
		self.update_message.emit(msg)
		time.sleep(0.001)

	def emit_finish(self, msg):
		self.update_progress.emit(100)
		self.update_message.emit(msg)
		self.finished.emit()

	def process(self):
		pass

	def run(self):
		self.emit_progress(0)
		try:
			self.process()
		except Exception, e:
			self.emit_finish('Error: %s' % str(e))
{'fraction_non_alphanumeric': 0.12172619047619047, 'fraction_numerical': 0.01011904761904762, 'mean_word_length': 4.084720121028744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2304357', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1199, 'n_words': 334}