<commit_before>
local EncoderFactory = torch.class('EncoderFactory')


function EncoderFactory:build_lookup_table(params, load_embeddings, vocab_size, dim)
    local pre_trained_embeddings = load_embeddings ~= '' and torch.load(load_embeddings)
    local lookup_table
    -- never update word embeddings, these should be preloaded
    if params.noWordUpdate then
        require 'nn-modules/NoUpdateLookupTable'
        lookup_table = nn.NoUpdateLookupTable(vocab_size, dim)
    else
        lookup_table = nn.LookupTable(vocab_size, dim)
    end

    lookup_table.weight = pre_trained_embeddings or torch.rand(vocab_size, dim):add(-.1):mul(0.1) -- initialize in range [-.1, .1]
    return lookup_table
end

function EncoderFactory:lstm_encoder(params, load_embeddings, vocab_size, embedding_dim)
    local input_dim = params.tokenDim > 0 and params.tokenDim or embedding_dim
    local output_dim = embedding_dim
    local lookup_table = self:build_lookup_table(params, load_embeddings, vocab_size, input_dim)

    local encoder = nn.Sequential()
    -- word dropout
    if params.wordDropout > 0 then
        require 'nn-modules/WordDropout'
        encoder:add(nn.WordDropout(params.wordDropout, 1))
    end

    encoder:add(lookup_table)
    if params.dropout > 0.0 then encoder:add(nn.Dropout(params.dropout)) end
    encoder:add(nn.SplitTable(2)) -- tensor to table of tensors
<commit_msg>deal with loading a bigger embedding matrix than specified<commit_after>
local EncoderFactory = torch.class('EncoderFactory')


function EncoderFactory:build_lookup_table(params, load_embeddings, vocab_size, dim)
    local pre_trained_embeddings = load_embeddings ~= '' and torch.load(load_embeddings)
    vocab_size = pre_trained_embeddings and math.max(vocab_size, pre_trained_embeddings:size(1)) or vocab_size
    local lookup_table
    -- never update word embeddings, these should be preloaded
    if params.noWordUpdate then
        require 'nn-modules/NoUpdateLookupTable'
        lookup_table = nn.NoUpdateLookupTable(vocab_size, dim)
    else
        lookup_table = nn.LookupTable(vocab_size, dim)
    end

    lookup_table.weight = pre_trained_embeddings or torch.rand(vocab_size, dim):add(-.1):mul(0.1) -- initialize in range [-.1, .1]
    return lookup_table
end

function EncoderFactory:lstm_encoder(params, load_embeddings, vocab_size, embedding_dim)
    local input_dim = params.tokenDim > 0 and params.tokenDim or embedding_dim
    local output_dim = embedding_dim
    local lookup_table = self:build_lookup_table(params, load_embeddings, vocab_size, input_dim)

    local encoder = nn.Sequential()
    -- word dropout
    if params.wordDropout > 0 then
        require 'nn-modules/WordDropout'
        encoder:add(nn.WordDropout(params.wordDropout, 1))
    end

    encoder:add(lookup_table)
    if params.dropout > 0.0 then encoder:add(nn.Dropout(params.dropout)) end
    encoder:add(nn.SplitTable(2)) -- tensor to table of tensors

{'fraction_non_alphanumeric': 0.08627717391304347, 'fraction_numerical': 0.0078125, 'mean_word_length': 4.221631205673759, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4190156', 'n_tokens_mistral': 936, 'n_tokens_neox': 907, 'n_words': 213}