<commit_before>import itertools
#import distance #libdistance library http://monkey.org/~jose/software/libdistance/
import affinegap
import lr

def canonicalImport(filename) :
    import csv

    data_d = {}
    duplicates_d = {}
    with open(filename) as f :
        reader = csv.reader(f)
        header = reader.next()
        print header
        for i, row in enumerate(reader) :
            instance = {}
            for j, col in enumerate(row) :
              if header[j] == 'unique_id' :
                duplicates_d.setdefault(col, []).append(i)
              else :
                instance[header[j]] = col.strip()
                
            data_d[i] = instance

    duplicates_s = set([])
    for unique_id in duplicates_d :
      if len(duplicates_d[unique_id]) > 1 :
        for pair in itertools.combinations(duplicates_d[unique_id], 2) :
          duplicates_s.add(frozenset(pair))

    return(data_d, header, duplicates_s)

def dataModel() :
  return  {'fields': 
            {'name' : {'type': 'String', 'weight' : 1}, 
             'address' : {'type' :'String', 'weight' : 1},
             'city' : {'type': 'String', 'weight' : 1},
             'cuisine' : {'type': 'String', 'weight' : 1}
            },
           'bias' : 0}

def identifyCandidates(data_d) :
  return [data_d.keys()]

def findDuplicates(candidates, data_d, data_model, threshold) :
  duplicateScores = []

  for candidates_set in candidates :
    for pair in itertools.combinations(candidates_set, 2):
      scorePair = {}
      score = data_model['bias'] 
      fields = data_model['fields']

      distances = calculateDistance(data_d[pair[0]], data_d[pair[1]], fields)
      for name in fields :
        score += distances[name] * fields[name]['weight']
      scorePair[pair] = score
      #print (pair, score)
      if score > threshold :
        print (data_d[pair[0]],data_d[pair[1]])
        print score
        duplicateScores.append(scorePair)
  
  return duplicateScores

def calculateDistance(instance_1, instance_2, fields) :
  distances_d = {}
  for name in fields :
    if fields[name]['type'] == 'String' :
      distanceFunc = affinegap.affineGapDistance
    x = distanceFunc(instance_1[name],instance_2[name])
    distances_d[name] = x

  return distances_d

def createTrainingPairs(data_d, duplicates_s, n) :
  import random
  train_pairs = {}
  while len(train_pairs) < n :
    random_pair = frozenset(random.sample(data_d.keys(), 2))
    if random_pair in duplicates_s : 
      train_pairs[random_pair] = 1
    else :
      train_pairs[random_pair] = 0
      
  return(train_pairs)

def createTrainingData(data_d, duplicates_s, n, data_model) :
  train_pairs = createTrainingPairs(data_d, duplicates_s, n)

  training_data = []
  for pair in train_pairs :
      instance_1 = data_d[tuple(pair)[0]]
      instance_2 = data_d[tuple(pair)[1]]
      distances = calculateDistance(instance_1,
                                    instance_2,
                                    data_model['fields'])
      training_data.append((train_pairs[pair], distances))<commit_msg>changed distance function to normalized affine gap<commit_after>import itertools
#import distance #libdistance library http://monkey.org/~jose/software/libdistance/
import affinegap
import lr

def canonicalImport(filename) :
    import csv

    data_d = {}
    duplicates_d = {}
    with open(filename) as f :
        reader = csv.reader(f)
        header = reader.next()
        print header
        for i, row in enumerate(reader) :
            instance = {}
            for j, col in enumerate(row) :
              if header[j] == 'unique_id' :
                duplicates_d.setdefault(col, []).append(i)
              else :
                instance[header[j]] = col.strip().strip('"').strip("'")
                
            data_d[i] = instance

    duplicates_s = set([])
    for unique_id in duplicates_d :
      if len(duplicates_d[unique_id]) > 1 :
        for pair in itertools.combinations(duplicates_d[unique_id], 2) :
          duplicates_s.add(frozenset(pair))

    return(data_d, header, duplicates_s)

def dataModel() :
  return  {'fields': 
            {'name' : {'type': 'String', 'weight' : 1}, 
             'address' : {'type' :'String', 'weight' : 1},
             'city' : {'type': 'String', 'weight' : 1},
             'cuisine' : {'type': 'String', 'weight' : 1}
            },
           'bias' : 0}

def identifyCandidates(data_d) :
  return [data_d.keys()]

def findDuplicates(candidates, data_d, data_model, threshold) :
  duplicateScores = []

  for candidates_set in candidates :
    for pair in itertools.combinations(candidates_set, 2):
      scorePair = {}
      score = data_model['bias'] 
      fields = data_model['fields']

      distances = calculateDistance(data_d[pair[0]], data_d[pair[1]], fields)
      for name in fields :
        score += distances[name] * fields[name]['weight']
      scorePair[pair] = score
      #print (pair, score)
      if score > threshold :
        print (data_d[pair[0]],data_d[pair[1]])
        print score
        duplicateScores.append(scorePair)
  
  return duplicateScores

def calculateDistance(instance_1, instance_2, fields) :
  distances_d = {}
  for name in fields :
    if fields[name]['type'] == 'String' :
      distanceFunc = affinegap.normalizedAffineGapDistance
    x = distanceFunc(instance_1[name],instance_2[name])
    distances_d[name] = x

  return distances_d

def createTrainingPairs(data_d, duplicates_s, n) :
  import random
  train_pairs = {}
  while len(train_pairs) < n :
    random_pair = frozenset(random.sample(data_d.keys(), 2))
    if random_pair in duplicates_s : 
      train_pairs[random_pair] = 1
    else :
      train_pairs[random_pair] = 0
      
  return(train_pairs)

def createTrainingData(data_d, duplicates_s, n, data_model) :
  train_pairs = createTrainingPairs(data_d, duplicates_s, n)

  training_data = []
  for pair in train_pairs :
      instance_1 = data_d[tuple(pair)[0]]
      instance_2 = data_d[tuple(pair)[1]]
      distances = calculateDistance(instance_1,
                                    instance_2,
                                    data_model['fields'])
      training_data.append((train_pairs[pair], distances))
{'fraction_non_alphanumeric': 0.1203659123736158, 'fraction_numerical': 0.008024394158241053, 'mean_word_length': 2.6169471851421937, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5669241', 'n_tokens_mistral': 2044, 'n_tokens_neox': 1946, 'n_words': 423}