Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Update auditqa/process_chunks.py
Browse files
auditqa/process_chunks.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1 |
import glob
|
|
|
2 |
import os
|
3 |
from langchain.text_splitter import RecursiveCharacterTextSplitter, SentenceTransformersTokenTextSplitter
|
4 |
from transformers import AutoTokenizer
|
|
|
1 |
import glob
|
2 |
+
import json
|
3 |
import os
|
4 |
from langchain.text_splitter import RecursiveCharacterTextSplitter, SentenceTransformersTokenTextSplitter
|
5 |
from transformers import AutoTokenizer
|