parameters: ()
docstring: collect and preprocess raw data from acl Imdb dataset
docstring_summary: collect and preprocess raw data from acl Imdb dataset
docstring_tokens: collect and preprocess raw data from acl Imdb dataset
function: def preprocess():
      ''' collect and preprocess raw data from acl Imdb dataset
      '''
      nltk.download('stopwords')
  
      print("preparing raw imdb data")
      data_gz = check_exist_or_download(imdb_dataset_link)
      data_dir = unzip_data(download_dir, data_gz)
  
      # imdb dirs
      # vocab_f = data_dir + '/imdb.vocab'
      train_pos_dir = data_dir + '/train/pos/'
      train_neg_dir = data_dir + '/train/neg/'
      test_pos_dir = data_dir + '/test/pos/'
      test_neg_dir = data_dir + '/test/neg/'
  
      # nltk helpers
      tokenizer = ToktokTokenizer()
      stopword_list = nltk.corpus.stopwords.words('english')
  
      # load pretrained word2vec binary
      print("loading pretrained word2vec")
      google_news_pretrain_fp = check_exist_or_download(
          google_news_pretrain_embeddings_link)
      wv = KeyedVectors.load_word2vec_format(google_news_pretrain_fp, binary=True)
  
      # parse flat files to memory
      data = []
      for data_dir, label in [(train_pos_dir, 1), (train_neg_dir, 0),
                              (test_pos_dir, 1), (test_neg_dir, 0)]:
          for filename in os.listdir(data_dir):
              if filename.endswith(".txt"):
                  with open(os.path.join(data_dir, filename),
                            "r",
                            encoding="utf-8") as fhdl:
                      data.append((fhdl.read(), label))
  
      # text review cleaning
      print("cleaning text review")
      imdb_data = pd.DataFrame(data, columns=["review", "label"])
      imdb_data['review'] = imdb_data['review'].apply(strip_html)
      imdb_data['review'] = imdb_data['review'].apply(
          remove_between_square_brackets)
      imdb_data['review'] = imdb_data['review'].apply(remove_special_characters)
      imdb_data['review'] = imdb_data['review'].apply(simple_stemmer)
      imdb_data['review'] = imdb_data['review'].apply(remove_stopwords,
                                                      args=(tokenizer,
                                                            stopword_list))
      imdb_data['token'] = imdb_data['review'].apply(tokenize)
  
      # build  word2index and index2word
      w2i = dict()
      i2w = dict()
  
      # add vocab <pad> as index 0
      w2i["<pad>"] = 0
      i2w[0] = "<pad>"
  
      idx = 1  # start from idx 1
      for index, row in imdb_data['token'].iteritems():
          for w in row:
              if w in wv and w not in w2i:
                  w2i[w] = idx
                  i2w[idx] = w
                  assert idx < 28241
                  idx += 1
      assert len(w2i) == len(i2w)
      print("vocab size: ", len(w2i))
  
      # encode tokens to int
      imdb_data['encoded'] = imdb_data['token'].apply(encode_token,
                                                      args=(wv, w2i))
  
      # select word vector weights for embedding layer from vocab
      embed_weights = []
      for w in w2i.keys():
          val = None
          if w in wv:
              val = wv[w]
          else:
              val = np.zeros([
                  300,
              ])
          embed_weights.append(val)
      embed_weights = np.array(embed_weights)
      print("embedding layer lookup weight shape: ", embed_weights.shape)
  
      # split into train and test
      train_data = imdb_data[['encoded', 'label']].values
      train, val = train_test_split(train_data, test_size=0.33, random_state=42)
  
      # save preprocessed for training
      imdb_processed = {
          "train": train,
          "val": val,
          "embed_weights": embed_weights,
          "w2i": w2i,
          "i2w": i2w
      }
      print("saving preprocessed file to ", preprocessed_imdb_data_fp)
      with open(preprocessed_imdb_data_fp, 'wb') as handle:
          pickle.dump(imdb_processed, handle, protocol=pickle.HIGHEST_PROTOCOL)
function_tokens: def preprocess ( ) : nltk . download ( 'stopwords' ) print ( "preparing raw imdb data" ) data_gz = check_exist_or_download ( imdb_dataset_link ) data_dir = unzip_data ( download_dir , data_gz ) # imdb dirs # vocab_f = data_dir + '/imdb.vocab' train_pos_dir = data_dir + '/train/pos/' train_neg_dir = data_dir + '/train/neg/' test_pos_dir = data_dir + '/test/pos/' test_neg_dir = data_dir + '/test/neg/' # nltk helpers tokenizer = ToktokTokenizer ( ) stopword_list = nltk . corpus . stopwords . words ( 'english' ) # load pretrained word2vec binary print ( "loading pretrained word2vec" ) google_news_pretrain_fp = check_exist_or_download ( google_news_pretrain_embeddings_link ) wv = KeyedVectors . load_word2vec_format ( google_news_pretrain_fp , binary = True ) # parse flat files to memory data = [ ] for data_dir , label in [ ( train_pos_dir , 1 ) , ( train_neg_dir , 0 ) , ( test_pos_dir , 1 ) , ( test_neg_dir , 0 ) ] : for filename in os . listdir ( data_dir ) : if filename . endswith ( ".txt" ) : with open ( os . path . join ( data_dir , filename ) , "r" , encoding = "utf-8" ) as fhdl : data . append ( ( fhdl . read ( ) , label ) ) # text review cleaning print ( "cleaning text review" ) imdb_data = pd . DataFrame ( data , columns = [ "review" , "label" ] ) imdb_data [ 'review' ] = imdb_data [ 'review' ] . apply ( strip_html ) imdb_data [ 'review' ] = imdb_data [ 'review' ] . apply ( remove_between_square_brackets ) imdb_data [ 'review' ] = imdb_data [ 'review' ] . apply ( remove_special_characters ) imdb_data [ 'review' ] = imdb_data [ 'review' ] . apply ( simple_stemmer ) imdb_data [ 'review' ] = imdb_data [ 'review' ] . apply ( remove_stopwords , args = ( tokenizer , stopword_list ) ) imdb_data [ 'token' ] = imdb_data [ 'review' ] . apply ( tokenize ) # build  word2index and index2word w2i = dict ( ) i2w = dict ( ) # add vocab <pad> as index 0 w2i [ "<pad>" ] = 0 i2w [ 0 ] = "<pad>" idx = 1 # start from idx 1 for index , row in imdb_data [ 'token' ] . iteritems ( ) : for w in row : if w in wv and w not in w2i : w2i [ w ] = idx i2w [ idx ] = w assert idx < 28241 idx += 1 assert len ( w2i ) == len ( i2w ) print ( "vocab size: " , len ( w2i ) ) # encode tokens to int imdb_data [ 'encoded' ] = imdb_data [ 'token' ] . apply ( encode_token , args = ( wv , w2i ) ) # select word vector weights for embedding layer from vocab embed_weights = [ ] for w in w2i . keys ( ) : val = None if w in wv : val = wv [ w ] else : val = np . zeros ( [ 300 , ] ) embed_weights . append ( val ) embed_weights = np . array ( embed_weights ) print ( "embedding layer lookup weight shape: " , embed_weights . shape ) # split into train and test train_data = imdb_data [ [ 'encoded' , 'label' ] ] . values train , val = train_test_split ( train_data , test_size = 0.33 , random_state = 42 ) # save preprocessed for training imdb_processed = { "train" : train , "val" : val , "embed_weights" : embed_weights , "w2i" : w2i , "i2w" : i2w } print ( "saving preprocessed file to " , preprocessed_imdb_data_fp ) with open ( preprocessed_imdb_data_fp , 'wb' ) as handle : pickle . dump ( imdb_processed , handle , protocol = pickle . HIGHEST_PROTOCOL )