<commit_before>local unicode = require './unicode'

local BPE = torch.class('BPE')

function BPE:__init(codesfile_path)
  local codes = {}
  local f = assert(io.open(codesfile_path, "r"))
  local t=f:read("*line")
  local i=1

  while not(t == nil) do
    l=string.split(t," ")
    if (#l==2) then
      codes[t]=i
      i=i+1
    end
    t=f:read("*line")
  end
  self.codes = codes
end

local function getPairs(word)
  local pairs = {}
  for i=1, #word-1, 1 do
    table.insert(pairs, word[i]..' '..word[i+1])
  end
  return pairs
end

local function str2word(l)
  local word = {}
  for v, c in unicode.utf8_iter(l) do
    table.insert(word, c)
  end
  table.insert(word, '</w>')
  return word
end

function BPE:minPair(pairsTable)
  mintmp = 100000
  minpair = ''
  for i=1, #pairsTable, 1 do
    pair_cur = pairsTable[i]
    if (self.codes[pair_cur] ~= nil) then
      scoretmp = self.codes[pair_cur]
      if (scoretmp < mintmp) then
        mintmp = scoretmp
        minpair = pair_cur
      end
    end
  end
  return minpair
end

function BPE:encode(l)
  local nextv, nextc = unicode._utf8_to_cp(l, 1)
  if #l <= #nextc then
    return l
  end
  local word = str2word(l)
  local pairs = getPairs(word)
  while (true) do
    local bigram = self:minPair(pairs)
    if (bigram == '') then
      break
    end
    bigram = string.split(bigram," ")
    local new_word = {}
    local merge = false
    for ii, xx in ipairs(word) do
      if (merge) then
        if (bigram[2] == xx) then
          table.insert(new_word, bigram[1]..bigram[2])
        else
          table.insert(new_word, bigram[1])
	  end
        merge = false
      else
        if (bigram[1] == xx ) then
          merge = true
        else
          table.insert(new_word, xx)
        end
      end
    end
    word = new_word
    if #word == 1 then
      break
    else
      pairs = getPairs(word)
    end
  end
  if word[#word] == '</w>' then
    table.remove(word, #word)
  elseif (string.sub(word[#word],-string.len('</w>'))=='</w>') then
    word[#word] = string.sub(word[#word], 1, string.len('</w>'))
  end
  output = ''
  for i = 1, #word-1, 1 do
    output = output..word[i]..'@@ '
  end
  output = output..word[#word]
  return output
end
<commit_msg>modify BPE return from string to array<commit_after>local unicode = require './unicode'

local BPE = torch.class('BPE')

function BPE:__init(codesfile_path)
  local codes = {}
  local f = assert(io.open(codesfile_path, "r"))
  local t=f:read("*line")
  local i=1

  while not(t == nil) do
    l=string.split(t," ")
    if (#l==2) then
      codes[t]=i
      i=i+1
    end
    t=f:read("*line")
  end
  self.codes = codes
end

local function getPairs(word)
  local pairs = {}
  for i=1, #word-1, 1 do
    table.insert(pairs, word[i]..' '..word[i+1])
  end
  return pairs
end

local function str2word(l)
  local word = {}
  for v, c in unicode.utf8_iter(l) do
    table.insert(word, c)
  end
  table.insert(word, '</w>')
  return word
end

function BPE:minPair(pairsTable)
  mintmp = 100000
  minpair = ''
  for i=1, #pairsTable, 1 do
    pair_cur = pairsTable[i]
    if (self.codes[pair_cur] ~= nil) then
      scoretmp = self.codes[pair_cur]
      if (scoretmp < mintmp) then
        mintmp = scoretmp
        minpair = pair_cur
      end
    end
  end
  return minpair
end

function BPE:encode(l)
  local nextv, nextc = unicode._utf8_to_cp(l, 1)
  if #l <= #nextc then
    return l
  end
  local word = str2word(l)
  local pairs = getPairs(word)
  while (true) do
    local bigram = self:minPair(pairs)
    if (bigram == '') then
      break
    end
    bigram = string.split(bigram," ")
    local new_word = {}
    local merge = false
    for ii, xx in ipairs(word) do
      if (merge) then
        if (bigram[2] == xx) then
          table.insert(new_word, bigram[1]..bigram[2])
        else
          table.insert(new_word, bigram[1])
	  end
        merge = false
      else
        if (bigram[1] == xx ) then
          merge = true
        else
          table.insert(new_word, xx)
        end
      end
    end
    word = new_word
    if #word == 1 then
      break
    else
      pairs = getPairs(word)
    end
  end
  if word[#word] == '</w>' then
    table.remove(word, #word)
  elseif (string.sub(word[#word],-string.len('</w>'))=='</w>') then
    word[#word] = string.sub(word[#word], 1, string.len('</w>'))
  end
  return word
end

{'fraction_non_alphanumeric': 0.13913639479095272, 'fraction_numerical': 0.013022618231665525, 'mean_word_length': 2.3990683229813663, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7016439', 'n_tokens_mistral': 1678, 'n_tokens_neox': 1643, 'n_words': 426}