parameters: (train_raw, label2answer, idx2word, q_seq_limit,
                         ans_seq_limit, idx2vec)
return_statement: return questions_encoded, poss_encoded, negs_encoded
docstring: prepare train data to embedded word vector sequence given sequence limit
          return:
              questions_encoded: np ndarray, shape
                  (number samples, seq length, vector size)
              poss_encoded: same layout, sequence for positive answer
              negs_encoded: same layout, sequence for negative answer
docstring_summary: prepare train data to embedded word vector sequence given sequence limit
          return:
              questions_encoded: np ndarray, shape
                  (number samples, seq length, vector size)
              poss_encoded: same layout, sequence for positive answer
              negs_encoded: same layout, sequence for negative answer
docstring_tokens: prepare train data to embedded word vector sequence given sequence limit return : questions_encoded : np ndarray shape ( number samples seq length vector size ) poss_encoded : same layout sequence for positive answer negs_encoded : same layout sequence for negative answer
function: def limit_encode_train(train_raw, label2answer, idx2word, q_seq_limit,
                         ans_seq_limit, idx2vec):
      ''' prepare train data to embedded word vector sequence given sequence limit
          return:
              questions_encoded: np ndarray, shape
                  (number samples, seq length, vector size)
              poss_encoded: same layout, sequence for positive answer
              negs_encoded: same layout, sequence for negative answer
      '''
      questions = [question for question, answers, candis in train_raw]
      # choose 1 answer from answer pool
      poss = [
          label2answer[random.choice(answers)]
          for question, answers, candis in train_raw
      ]
      # choose 1 candidate from candidate pool
      negs = [
          label2answer[random.choice(candis)]
          for question, answers, candis in train_raw
      ]
  
      # filtered word not in idx2vec
      questions_filtered = [
          [idx for idx in q if idx in idx2vec] for q in questions
      ]
      poss_filtered = [[idx for idx in ans if idx in idx2vec] for ans in poss]
      negs_filtered = [[idx for idx in ans if idx in idx2vec] for ans in negs]
  
      # crop to seq limit
      questions_crop = [
          q[:q_seq_limit] + [0] * max(0, q_seq_limit - len(q))
          for q in questions_filtered
      ]
      poss_crop = [
          ans[:ans_seq_limit] + [0] * max(0, ans_seq_limit - len(ans))
          for ans in poss_filtered
      ]
      negs_crop = [
          ans[:ans_seq_limit] + [0] * max(0, ans_seq_limit - len(ans))
          for ans in negs_filtered
      ]
  
      # encoded, word idx to word vector
      questions_encoded = [[idx2vec[idx] for idx in q] for q in questions_crop]
      poss_encoded = [[idx2vec[idx] for idx in ans] for ans in poss_crop]
      negs_encoded = [[idx2vec[idx] for idx in ans] for ans in negs_crop]
  
      # make nd array
      questions_encoded = np.array(questions_encoded).astype(np.float32)
      poss_encoded = np.array(poss_encoded).astype(np.float32)
      negs_encoded = np.array(negs_encoded).astype(np.float32)
      return questions_encoded, poss_encoded, negs_encoded
function_tokens: def limit_encode_train ( train_raw , label2answer , idx2word , q_seq_limit , ans_seq_limit , idx2vec ) : questions = [ question for question , answers , candis in train_raw ] # choose 1 answer from answer pool poss = [ label2answer [ random . choice ( answers ) ] for question , answers , candis in train_raw ] # choose 1 candidate from candidate pool negs = [ label2answer [ random . choice ( candis ) ] for question , answers , candis in train_raw ] # filtered word not in idx2vec questions_filtered = [ [ idx for idx in q if idx in idx2vec ] for q in questions ] poss_filtered = [ [ idx for idx in ans if idx in idx2vec ] for ans in poss ] negs_filtered = [ [ idx for idx in ans if idx in idx2vec ] for ans in negs ] # crop to seq limit questions_crop = [ q [ : q_seq_limit ] + [ 0 ] * max ( 0 , q_seq_limit - len ( q ) ) for q in questions_filtered ] poss_crop = [ ans [ : ans_seq_limit ] + [ 0 ] * max ( 0 , ans_seq_limit - len ( ans ) ) for ans in poss_filtered ] negs_crop = [ ans [ : ans_seq_limit ] + [ 0 ] * max ( 0 , ans_seq_limit - len ( ans ) ) for ans in negs_filtered ] # encoded, word idx to word vector questions_encoded = [ [ idx2vec [ idx ] for idx in q ] for q in questions_crop ] poss_encoded = [ [ idx2vec [ idx ] for idx in ans ] for ans in poss_crop ] negs_encoded = [ [ idx2vec [ idx ] for idx in ans ] for ans in negs_crop ] # make nd array questions_encoded = np . array ( questions_encoded ) . astype ( np . float32 ) poss_encoded = np . array ( poss_encoded ) . astype ( np . float32 ) negs_encoded = np . array ( negs_encoded ) . astype ( np . float32 ) return questions_encoded , poss_encoded , negs_encoded