<commit_before>-- The Great Computer Language Shootout
-- http://shootout.alioth.debian.org/
--
-- contributed by <NAME>


local function kFrequency(s,freq,k,frame)
   local n = string.len(s)-k+1      
   for i = frame,n,k do 
      local c = string.sub(s,i,i+k-1)
      record = freq[c]
      if record then
         record.count = record.count + 1
      else
         freq[c] = {key=c,count=1}
      end
   end
end

local function SortByFrequencyAndCode(a,b) 
   if a.count == b.count then
      return a.key < b.key
   else
      return a.count > b.count 
   end
end         

local function frequency(s,k)
   local freq = {}
   for i = 1,k do kFrequency(s,freq,k,i) end

   local sorted, sum = {}, 0
   for _,record in pairs(freq) do 
      table.insert(sorted,record) 
      sum = sum + record.count
   end

   table.sort(sorted, SortByFrequencyAndCode)
   
   for _,record in ipairs(sorted) do 
      local percent = (record.count/sum)*100
      io.write( string.format("%s %0.9f",record.key,percent), "\n") 
   end
   io.write("\n") 
end


local function count(sequence,fragment)
   local freq, record, count = {}
   local k = string.len(fragment)
   for i = 1,k do kFrequency(sequence,freq,k,i) end
   record = freq[fragment]
   if record then count = record.count else count = 0 end<commit_msg>*** empty log message ***<commit_after>-- The Great Computer Language Shootout
-- http://shootout.alioth.debian.org/
--
-- contributed by <NAME>


local function kFrequency(s,freq,k,frame)
   local n = string.len(s)-k+1      
   for i = frame,n,k do 
      local c = string.sub(s,i,i+k-1)
      record = freq[c]
      if record then
         record.count = record.count + 1
      else
         freq[c] = {key=c,count=1}
      end
   end
end

local function SortByFrequencyAndCode(a,b) 
   if a.count == b.count then
      return a.key < b.key
   else
      return a.count > b.count 
   end
end         

local function frequency(s,k)
   local freq = {}
   for i = 1,k do kFrequency(s,freq,k,i) end

   local sorted, sum = {}, 0
   for _,record in pairs(freq) do 
      table.insert(sorted,record) 
      sum = sum + record.count
   end

   table.sort(sorted, SortByFrequencyAndCode)
   
   for _,record in ipairs(sorted) do 
      local percent = (record.count/sum)*100
      io.write( string.format("%s %0.2f",record.key,percent), "\n") 
   end
   io.write("\n") 
end


local function count(sequence,fragment)
   local freq, record, count = {}
   local k = string.len(fragment)
   for i = 1,k do kFrequency(sequence,freq,k,i) end
   record = freq[fragment]
   if record then count = record.count else count = 0 end
{'fraction_non_alphanumeric': 0.12452253628724216, 'fraction_numerical': 0.009931245225362872, 'mean_word_length': 2.534412955465587, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7299158', 'n_tokens_mistral': 949, 'n_tokens_neox': 896, 'n_words': 247}