<commit_before>#!/usr/bin/env ruby

require 'yaml'
require 'optparse'
require 'pragmatic_segmenter'

#remove punctuation from the edges of words
def strip_punctuation(word)
  word.gsub(/^[¿¡"'“‘«(\[]+|[?!.,"'”’»)\]:;]+$/,'')
end

#returns an array of the words contained in string sentence
def collect_words(sentence)
  words = sentence.split(/ |--|—/) #split on spaces, double-dashes, and em-dashes
  #remove non-internal punctuation
  words = words.map do |word|
    strip_punctuation(word)
  end
  words.delete('') #remove empty strings
  words
end

#diagnostic to make sure words are being separated properly
#text is an array of sentences
def print_words(text)
  text.each do |sentence|
    puts collect_words(sentence)
  end
end

#compute number of words shared between two arrays
def num_shared_words(words_a,words_b)
  matches = 0
  words_a.each do |word_a|
    words_b.each_with_index do |word_b,i|
      if (word_a.downcase == word_b.downcase)
        matches += 1
        words_b.delete_at(i)
        break
      end
    end
  end
  matches
end

#similarity between two arrays of words
def similarity_score(words_a,words_b)
  return 0 if words_a.size == 0 || words_b.size == 0
  size = [words_a.size,words_b.size].max
  matches = num_shared_words(words_a,words_b)
  matches.to_f/size
end

def construct_comparandum(array,start_index,num_sentences)
  words = []
  array[start_index,num_sentences].each do |sentence|
    words += collect_words(sentence)
  end
  words
end

def merge_sentences(array,start_index,num_sentences)
  merged = ""
  array[start_index,num_sentences].each do |sentence|
    merged += "#{sentence} "
  end
  merged
end

def check_score(best_score,best_num1,best_num2,try_num1,try_num2,text1,index1,text2,index2)
  score = similarity_score(construct_comparandum(text1,index1,try_num1),construct_comparandum(text2,index2,try_num2))
  puts merge_sentences(text1,index1,try_num1)
  puts merge_sentences(text2,index2,try_num2)
  puts score

  if score > best_score
    best_score = score
    best_num1 = try_num1
    best_num2 = try_num2
  end
  [best_score,best_num1,best_num2]
end

def correlate_texts(source,mt,translation)
  index_s = 0
  index_t = 0
  min_words = 0#10
  max_sentences_to_group = 5
  results = []  
  #what if things don't end so nicely?
  #after the loop I should append the remainder
  while index_s < source.size && index_t < translation.size
    puts "["

    #what I want to is vary this base number if appropriate!
    base_num_sentences_s = 1
    base_num_sentences_t = 1

    #in case we can get empty sentences... take care of that here.
    while construct_comparandum(mt,index_s,base_num_sentences_s).size < 1
      base_num_sentences_s += 1
    end
    while construct_comparandum(translation,index_t,base_num_sentences_t).size < 1
      base_num_sentences_t += 1
    end

    #get the score of the one-sentence-to-one-sentence correspondence
    best_score,best_num_sentences_s,best_num_sentences_t = check_score(0.0,base_num_sentences_s,base_num_sentences_t,base_num_sentences_s,base_num_sentences_t,mt,index_s,translation,index_t)

    #The idea here is that we will either group EXACTLY one source sentence with 1 <= N <= 5 translation sentences OR we will group EXACTLY one translation sentence with 1 <= N <= 5 source sentences.
    #The exception to this rule is for short sentences. If we have a sentence (or group of sentences) less than 10 words, we will allow it to attach to a larger sentence (before or after), even in cases where normally we would only allow one sentence.
    max_sentences_to_group.times do |i|
      #Test grouping base source sentence with next i+1 source sentences
      #If permissible, also try grouping base translation sentence with extra sentences
      extra_s = i+1
      extra_t = 0
      while (index_s+base_num_sentences_s+extra_s <= source.size && index_t+base_num_sentences_t+extra_t <= translation.size)
        best_score,best_num_sentences_s,best_num_sentences_t = check_score(best_score,best_num_sentences_s,best_num_sentences_t,base_num_sentences_s+extra_s,base_num_sentences_t+extra_t,mt,index_s,translation,index_t)
        break unless (construct_comparandum(translation,index_t,base_num_sentences_t+extra_t).size < min_words ||
                      construct_comparandum(translation,index_t+base_num_sentences_t,extra_t+1).size < min_words)
        extra_t += 1
      end

      #Test grouping base translation sentence with next i+1 sentences
      #If permissible, also try grouping base source sentence with extra sentences
      extra_s = 0
      extra_t = i+1
      while (index_s+base_num_sentences_s+extra_s <= source.size && index_t+base_num_sentences_t+extra_t <= translation.size)
        best_score,best_num_sentences_s,best_num_sentences_t = check_score(best_score,best_num_sentences_s,best_num_sentences_t,base_num_sentences_s+extra_s,base_num_sentences_t+extra_t,mt,index_s,translation,index_t)
        break unless (construct_comparandum(translation,index_s,base_num_sentences_s+extra_s).size < min_words ||
                      construct_comparandum(translation,index_s+base_num_sentences_s,extra_s+1).size < min_words)
        extra_s += 1
      end
    end

    results << [merge_sentences(source,index_s,best_num_sentences_s),merge_sentences(mt,index_s,best_num_sentences_s), merge_sentences(translation,index_t,best_num_sentences_t)]

    index_s += best_num_sentences_s
    index_t += best_num_sentences_t
    puts "]"
  end
  results
end<commit_msg>Good checkpoint, before searching deeper layer<commit_after>#!/usr/bin/env ruby

require 'yaml'
require 'optparse'
require 'pragmatic_segmenter'

#remove punctuation from the edges of words
def strip_punctuation(word)
  word.gsub(/^[¿¡"'“‘«(\[]+|[?!.,"'”’»)\]:;]+$/,'')
end

#returns an array of the words contained in string sentence
def collect_words(sentence)
  words = sentence.split(/ |--|—/) #split on spaces, double-dashes, and em-dashes
  #remove non-internal punctuation
  words = words.map do |word|
    strip_punctuation(word)
  end
  words.delete('') #remove empty strings
  words
end

#diagnostic to make sure words are being separated properly
#text is an array of sentences
def print_words(text)
  text.each do |sentence|
    puts collect_words(sentence)
  end
end

#compute number of words shared between two arrays
def num_shared_words(words_a,words_b)
  matches = 0
  words_a.each do |word_a|
    words_b.each_with_index do |word_b,i|
      if (word_a.downcase == word_b.downcase)
        matches += 1
        words_b.delete_at(i)
        break
      end
    end
  end
  matches
end

#similarity between two arrays of words
def similarity_score(words_a,words_b)
  return 0 if words_a.size == 0 || words_b.size == 0
  size = [words_a.size,words_b.size].max
  matches = num_shared_words(words_a,words_b)
  matches.to_f/size
end

def construct_comparandum(array,start_index,num_sentences)
  words = []
  array[start_index,num_sentences].each do |sentence|
    words += collect_words(sentence)
  end
  words
end

def merge_sentences(array,start_index,num_sentences)
  merged = ""
  array[start_index,num_sentences].each do |sentence|
    merged += "#{sentence} "
  end
  merged
end

def check_score(best_score,best_num1,best_num2,try_num1,try_num2,text1,index1,text2,index2)
  score = similarity_score(construct_comparandum(text1,index1,try_num1),construct_comparandum(text2,index2,try_num2))

  if score > best_score
    best_score = score
    best_num1 = try_num1
    best_num2 = try_num2
  end
  [best_score,best_num1,best_num2]
end

def correlate_texts(source,mt,translation)
  index_s = 0 #s for source
  index_t = 0 #t for target
  min_words = 10
  max_extra_sentences = 5
  results = []  
  #what if things don't end so nicely?
  #after the loop I should append the remainder
  while index_s < source.size && index_t < translation.size
    #The idea here is that we will either group EXACTLY one source sentence with 1 <= N <= 5 translation sentences OR we will group EXACTLY one translation sentence with 1 <= N <= 5 source sentences.
    #The exception to this rule is for short sentences. If we have a sentence (or group of sentences) less than 10 words, we will allow it to attach to a larger sentence (before or after), even in cases where normally we would only allow one sentence.

    best_score = 0.0 #running tally of the best score
    best_num_sentences_s = 1 #keep track of the number of setences that gave us the best score
    best_num_sentences_t = 1

    #allow the base number of sentences to vary if the first sentence(s) are short
    num_sentences_to_min_words_s = 1
    num_sentences_to_min_words_t = 1

    while construct_comparandum(mt,index_s,num_sentences_to_min_words_s).size < min_words
      num_sentences_to_min_words_s += 1
    end
    while construct_comparandum(translation,index_t,num_sentences_to_min_words_t).size < min_words
      num_sentences_to_min_words_t += 1
    end

    1.upto(num_sentences_to_min_words_s) do |base_num_sentences_s|
      1.upto(num_sentences_to_min_words_t) do |base_num_sentences_t|
        #get the score comparing just the base sentences
        best_score,best_num_sentences_s,best_num_sentences_t = check_score(best_score,best_num_sentences_s,best_num_sentences_t,base_num_sentences_s,base_num_sentences_t,mt,index_s,translation,index_t)

        1.upto(max_extra_sentences) do |i|
          #Test grouping base source sentence with next i source sentences
          #If permissible, also try grouping base translation sentence with extra sentences
          extra_s = i
          extra_t = 0
          while (index_s+base_num_sentences_s+extra_s <= source.size && index_t+base_num_sentences_t+extra_t <= translation.size)
            best_score,best_num_sentences_s,best_num_sentences_t = check_score(best_score,best_num_sentences_s,best_num_sentences_t,base_num_sentences_s+extra_s,base_num_sentences_t+extra_t,mt,index_s,translation,index_t)
            break unless (construct_comparandum(translation,index_t+base_num_sentences_t,extra_t+1).size < min_words)
            extra_t += 1
          end

          #Test grouping base translation sentence with next i sentences
          #If permissible, also try grouping base source sentence with extra sentences
          extra_s = 0
          extra_t = i
          while (index_s+base_num_sentences_s+extra_s <= source.size && index_t+base_num_sentences_t+extra_t <= translation.size)
            best_score,best_num_sentences_s,best_num_sentences_t = check_score(best_score,best_num_sentences_s,best_num_sentences_t,base_num_sentences_s+extra_s,base_num_sentences_t+extra_t,mt,index_s,translation,index_t)
            break unless (construct_comparandum(mt,index_s+base_num_sentences_s,extra_s+1).size < min_words)
            extra_s += 1
          end
        end
      end
    end

    results << [merge_sentences(source,index_s,best_num_sentences_s),merge_sentences(mt,index_s,best_num_sentences_s), merge_sentences(translation,index_t,best_num_sentences_t)]

    index_s += best_num_sentences_s
    index_t += best_num_sentences_t
  end
  results
end
{'fraction_non_alphanumeric': 0.07946243348065302, 'fraction_numerical': 0.01028231261838189, 'mean_word_length': 4.443298969072165, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6291267', 'n_tokens_mistral': 3832, 'n_tokens_neox': 3705, 'n_words': 885}