<commit_before>import lr
import affinegap
import numpy
import json

def writeSettings(file_name, data_model, predicates) :
  source_predicates = [(predicate[0].__name__,
                        predicate[1])
                       for predicate in predicates]
  with open(file_name, 'w') as f :
    json.dump({'data model' : data_model,
               'predicates' : source_predicates}, f)
  
  
def readSettings(file_name) :
  from predicates import *
  with open(file_name, 'r') as f :
    learned_settings = json.load(f)

  data_model = learned_settings['data model']
  predicates = [(eval(predicate[0]), predicate[1])
                for predicate in learned_settings['predicates']]

  return data_model, predicates

# based on field type, calculate using the appropriate distance function and return distance
def calculateDistance(instance_1, instance_2, fields, distances) :

  for i, name in enumerate(fields) :
    if fields[name]['type'] == 'String' :
      distanceFunc = affinegap.normalizedAffineGapDistance

    distances[0]['names'][i] = name
    distances[0]['values'][i] = distanceFunc(instance_1[name],instance_2[name], -5, 5, 4, 1, .125)
      #distances[0]['values'][i] = distanceFunc(instance_1[name],instance_2[name], 1, 11, 10, 7, .125)
    

  return distances

# using logistic regression, train weights for all fields in the data model
def trainModel(training_data, iterations, data_model) :
    trainer = lr.LogisticRegression()
    trainer.train(training_data, iterations)

    data_model['bias'] = trainer.bias
    for name in data_model['fields'] :
        data_model['fields'][name]['weight'] = trainer.weight[name]

    return(data_model)

# assign a score of how likely a pair of records are duplicates
def recordDistances(candidates, data_d, data_model) :
  # The record array has two elements, the first element is an array
  # of floats that has length equal the number of fields. The second
  # argument is a array of length 2 which stores the id of the
  # considered elements in the pair.

  fields = data_model['fields']

  field_dtype = [('names', 'a20', (len(fields)),),
                 ('values', 'f4', (len(fields)),)
                 ]
  
  distances = numpy.zeros(1, dtype=field_dtype)

  record_dtype = [('pairs', [('pair1', 'i4'),
                             ('pair2', 'i4')]),
                  ('field_distances', field_dtype)
                  ]

  record_distances = numpy.zeros(len(candidates), dtype=record_dtype)

  primary_fields = dict([(k, v) for k,v in data_model['fields'].iteritems() 
                    if data_model['fields'][k]['type'] != 'Interaction'])
                    
  
  for i, pair in enumerate(candidates) :

    c_distances = calculateDistance(data_d[pair[0]],
                                    data_d[pair[1]],
                                    primary_fields,
                                    distances)
                                    
    record_distances[i] = ((pair[0], pair[1]),
                           (c_distances['names'],
                            c_distances['values'])
                           )
    
  return record_distances  

def scorePairs(record_distances, data_model) :
  fields = data_model['fields']

  field_weights = [fields[name]['weight'] for name in fields]
  bias = data_model['bias']



  #field_distances = [col for row in record_distances['field_distances']
  #                   for col in row]
  field_distances = record_distances['field_distances']['values']

#  field_distances = numpy.reshape(field_distances, (-1, len(fields)))<commit_msg>removed primary fields from findRecords<commit_after>import lr
import affinegap
import numpy
import json

def writeSettings(file_name, data_model, predicates) :
  source_predicates = [(predicate[0].__name__,
                        predicate[1])
                       for predicate in predicates]
  with open(file_name, 'w') as f :
    json.dump({'data model' : data_model,
               'predicates' : source_predicates}, f)
  
  
def readSettings(file_name) :
  from predicates import *
  with open(file_name, 'r') as f :
    learned_settings = json.load(f)

  data_model = learned_settings['data model']
  predicates = [(eval(predicate[0]), predicate[1])
                for predicate in learned_settings['predicates']]

  return data_model, predicates

# based on field type, calculate using the appropriate distance function and return distance
def calculateDistance(instance_1, instance_2, fields, distances) :

  for i, name in enumerate(fields) :
    if fields[name]['type'] == 'String' :
      distanceFunc = affinegap.normalizedAffineGapDistance

    distances[0]['names'][i] = name
    distances[0]['values'][i] = distanceFunc(instance_1[name],instance_2[name], -5, 5, 4, 1, .125)
      #distances[0]['values'][i] = distanceFunc(instance_1[name],instance_2[name], 1, 11, 10, 7, .125)
    

  return distances

# using logistic regression, train weights for all fields in the data model
def trainModel(training_data, iterations, data_model) :
    trainer = lr.LogisticRegression()
    trainer.train(training_data, iterations)

    data_model['bias'] = trainer.bias
    for name in data_model['fields'] :
        data_model['fields'][name]['weight'] = trainer.weight[name]

    return(data_model)

# assign a score of how likely a pair of records are duplicates
def recordDistances(candidates, data_d, data_model) :
  # The record array has two elements, the first element is an array
  # of floats that has length equal the number of fields. The second
  # argument is a array of length 2 which stores the id of the
  # considered elements in the pair.

  fields = data_model['fields']

  field_dtype = [('names', 'a20', (len(fields)),),
                 ('values', 'f4', (len(fields)),)
                 ]
  
  distances = numpy.zeros(1, dtype=field_dtype)

  record_dtype = [('pairs', [('pair1', 'i4'),
                             ('pair2', 'i4')]),
                  ('field_distances', field_dtype)
                  ]

  record_distances = numpy.zeros(len(candidates), dtype=record_dtype)

  for i, pair in enumerate(candidates) :

    c_distances = calculateDistance(data_d[pair[0]],
                                    data_d[pair[1]],
                                    fields,
                                    distances)
                                    
    record_distances[i] = ((pair[0], pair[1]),
                           (c_distances['names'],
                            c_distances['values'])
                           )
    
  return record_distances  

def scorePairs(record_distances, data_model) :
  fields = data_model['fields']

  field_weights = [fields[name]['weight'] for name in fields]
  bias = data_model['bias']



  #field_distances = [col for row in record_distances['field_distances']
  #                   for col in row]
  field_distances = record_distances['field_distances']['values']

#  field_distances = numpy.reshape(field_distances, (-1, len(fields)))
{'fraction_non_alphanumeric': 0.11417942481041637, 'fraction_numerical': 0.012305050794105023, 'mean_word_length': 2.7003705664372686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5953563', 'n_tokens_mistral': 2104, 'n_tokens_neox': 2010, 'n_words': 527}