<commit_before>import config
import log
from datetime import datetime
import db
import os
import image
import restore
import sys

####
# create table match_index (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name VARCHAR(255), tar_index_id INT(255), proccess_datetime DATETIME , match_value DOUBLE , ml_wag_id INT(255) );

def getUnmatched():
  try:
    ml_db = db.connect(config.get('db_host','ml_analyse'),config.get('db_user','ml_analyse'),config.get('db_password','<PASSWORD>'),config.get('db_name','ml_analyse'))
    log.logger.debug('Succesfully connected to database ' + config.get('db_host','ml_analyse') )
  except Exception as e:
    log.logger.critical('Could not connect to database '+ config.get('db_host','ml_analyse'))
    log.logger.debug(e)
    exit(1)

  q = 'SELECT qr,path,selected,evaluated,Id,scan_date FROM ml WHERE uniq > 1 AND neglect = 0 AND restore_matched = 0  AND selected = 1 AND upload_date < "2013-08-31" ORDER BY upload_date DESC limit 10'

  try:
    unmatched = db.query(ml_db,q)
    log.logger.debug('Succesfully queried tha ml_db database')
    return unmatched
  except Exception as e:
    log.logger.critical('Unable to query  ml_db database')
    log.logger.debug(e)



def getCandidates(db_object):
  try:
    tar_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))
    log.logger.debug('Succesfully connected to database on: ' + config.get('db_host'))
  except Exception as e:
    log.logger.critical('Could not connect database to :' + config.get('db_host'))
    log.logger.debug(e)
    exit(1)

  filename = db_object['path'].split('/')[-1][0:-(len(db_object['path'].split('.')[-1]) + 1)]

  #print filename

  q = 'SELECT id,tar,filename,tar_index,name FROM tar_index WHERE name = "' + filename + '"'
<commit_msg>created a almost fully functional match function<commit_after>import config
import log
from datetime import datetime
import db
import os
import image
import restore
import sys

####
# create table match_index (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name VARCHAR(255), tar_index_id INT(255), proccess_datetime DATETIME , match_value DOUBLE , ml_wag_id INT(255) );

def getUnmatched():
  try:
    ml_db = db.connect(config.get('db_host','ml_analyse'),config.get('db_user','ml_analyse'),config.get('db_password','<PASSWORD>'),config.get('db_name','ml_analyse'))
    log.logger.debug('Succesfully connected to database ' + config.get('db_host','ml_analyse') )
  except Exception as e:
    log.logger.critical('Could not connect to database '+ config.get('db_host','ml_analyse'))
    log.logger.debug(e)
    exit(1)

  q = 'SELECT qr,path,selected,evaluated,Id,scan_date FROM ml WHERE uniq > 1 AND neglect = 0 AND restore_matched = 0  AND selected = 1 AND upload_date < "2013-08-31" ORDER BY upload_date limit 10'

  try:
    unmatched = db.query(ml_db,q)
    log.logger.debug('Succesfully queried tha ml_db database')
    return unmatched
  except Exception as e:
    log.logger.critical('Unable to query  ml_db database')
    log.logger.debug(e)



def getCandidates(db_object):
  try:
    tar_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))
    log.logger.debug('Succesfully connected to database on: ' + config.get('db_host'))
  except Exception as e:
    log.logger.critical('Could not connect database to :' + config.get('db_host'))
    log.logger.debug(e)
    exit(1)

  filename = db_object['path'].split('/')[-1][0:-(len(db_object['path'].split('.')[-1]) + 1)]

  #print filename

  q = 'SELECT id,tar,filename,tar_index,name FROM tar_index WHERE name = "' + filename + '"'

{'fraction_non_alphanumeric': 0.14160743575724438, 'fraction_numerical': 0.015855658829961726, 'mean_word_length': 4.9207119741100325, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '48410', 'n_tokens_mistral': 1269, 'n_tokens_neox': 1202, 'n_words': 289}