<commit_before>File.open('results.txt', "w") do |output|   
  sub_size = nil

 
  end
end<commit_msg>really brute forced, but not working on a better solution right now.<commit_after>load 'fasta.rb'
fasta = FASTA.new()

$memo_matches = Hash.new(false)

def match_info(sub, target)  
  $memo_matches[sub] if $memo_matches[sub].nil? || $memo_matches[sub]    
  full = target.size
  target = target.clone
  remainder = true;
  sub.chars.each do |char|    
      if target.length == 0 || target.index(char).nil?
        remainder = false
        break
      end
      target = target[target.index(char)+1..-1]
  end
  $memo_matches[sub] = remainder ? [sub,full - target.size] : nil
end

fasrecs = fasta.load_file "long_com_sub.fas"

File.open('results.txt', "w") do |output|     
  source, target = fasrecs.values
  candidates = [[source[0], 1]]  
  source[1..-1].chars.each do |character|     
    candidates.clone.each do |candidate|                  
      potential = candidate.first + character            
      data = match_info(potential, target)            
      next unless data

      next if candidates.select {|x| x.first.size >= data.first.size && x.last <= data.last}.length > 0
      
      tie = candidates.index{|x| x.first.size == data.first.size}
      if tie 
        
        next if candidates[tie].last <= data.last
        
        candidates.delete_at(tie)
      end      
      candidates.delete(candidate) if (candidate.last+1) == data.last
            
      candidates.reject!{|x| x.first.size <= data.first.size && x.last >= data.last }
      
      candidates.push data
      
    end    
  end
  output.puts "#{candidates.max_by{|x| x.first.size}.first}"
end
{'fraction_non_alphanumeric': 0.13033175355450238, 'fraction_numerical': 0.00533175355450237, 'mean_word_length': 2.3445544554455444, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2252863', 'n_tokens_mistral': 566, 'n_tokens_neox': 558, 'n_words': 123}