<jupyter_start><jupyter_text>Stopped here<jupyter_code>model = BertModel.from_pretrained('bert-base-uncased')


tokens = BertTokenizer.from_pretrained('bert-base-uncased')
#filtered = []
#for d in range(len(data)):
 # sample = data.loc[d][6]
#  t = tokens(sample)["input_ids"][1:-1]
  #chunks = []
#  for i in range(0, len(t), 500):
   # chunks.append(tokens.decode(t[i:i+500]))
 # j = 0
 # for i in range(len(chunks)):
  #  if data.loc[d][5] in chunks[i]:
   #   j = i
   #   break
  #filtered.append(chunks[j])

#data["filtered"] = filtered
#data.to_csv("data_filtered.csv")

data = pd.read_csv("data_filtered.csv")<jupyter_output><empty_output><jupyter_text>Crashes here trying to find hidden states<jupyter_code>states = []
for i in range(len(data)):  
  if i /50 == 0:
    print(i)

  t = tokens(data.loc[i][-1], return_tensors="pt")

  states.append(model(**t).last_hidden_state)
sample = data.loc[0][6]
count = 0
print(len(t))
token_col = []
len_col = []
for i in range(len(data)):
  t = tokens(data.loc[i][6], return_tensors="pt")
  token_col.append(t)
  len_col.append(t["input_ids"].shape)
  if t["input_ids"].shape[1] > 512:
    count +=1
print(count)
print(sample)
t = tokens(sample, return_tensors="pt", max_length=512)
print(t["input_ids"].shape[1])
outputs = model(**t)
last_hidden_layers = outputs.last_hidden_state
print(last_hidden_layers.shape)

changed_text = []
t = tokens(data.loc[0][6])
print(tokens.decode(t["input_ids"]))
print(data.loc[0][6])
for i in range(len(data)):
  t = data.loc[i][6].split(' ')
print(len_col)
data = data[data["token_len"] < 512]
print(len(data))
from spacy.lang.en import English
spacy_sent = spacy.load('en_core_web_sm')
sentences = spacy_sent(data.loc[0][6]).sents
base = torch.zeros((1,0,768))
print(data.loc[0][6])
for i in sentences:
   print(i.text)
   #t = tokens(i.text, return_tensors="pt")
   #outputs = model(**t)
   #last_hidden_layers = outputs.last_hidden_state
   #base = torch.cat((base, last_hidden_layers), dim=1)

print(base.shape)
t = tokens(data.loc[0][6], return_tensors="pt")
print(t["input_ids"].shape)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18889416231608921, 'fraction_numerical': 0.023255813953488372, 'mean_word_length': 5.692063492063492, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '471654', 'license': 'MIT', 'n_tokens_mistral': 871, 'n_tokens_neox': 839, 'n_words': 141, 'path': 'glove-bilstm.ipynb', 'repo_name': 'arodriguezca/NLP-dataset', 'size': '2107'}