ppsingh commited on
Commit
0306ea2
1 Parent(s): d1fcb29

Update appStore/doc_processing.py

Browse files
Files changed (1) hide show
  1. appStore/doc_processing.py +77 -76
appStore/doc_processing.py CHANGED
@@ -1,77 +1,78 @@
1
- # set path
2
- import glob, os, sys;
3
- sys.path.append('../utils')
4
- from typing import List, Tuple
5
- from typing_extensions import Literal
6
- from haystack.schema import Document
7
- from utils.config import get_classifier_params
8
- from utils.preprocessing import processingpipeline,paraLengthCheck
9
- import streamlit as st
10
- import logging
11
- import pandas as pd
12
- params = get_classifier_params("preprocessing")
13
-
14
- @st.cache_data
15
- def runPreprocessingPipeline(file_name:str, file_path:str,
16
- split_by: Literal["sentence", "word"] = 'sentence',
17
- split_length:int = 2, split_respect_sentence_boundary:bool = False,
18
- split_overlap:int = 0,remove_punc:bool = False)->List[Document]:
19
- """
20
- creates the pipeline and runs the preprocessing pipeline,
21
- the params for pipeline are fetched from paramconfig
22
- Params
23
- ------------
24
- file_name: filename, in case of streamlit application use
25
- st.session_state['filename']
26
- file_path: filepath, in case of streamlit application use st.session_state['filepath']
27
- split_by: document splitting strategy either as word or sentence
28
- split_length: when synthetically creating the paragrpahs from document,
29
- it defines the length of paragraph.
30
- split_respect_sentence_boundary: Used when using 'word' strategy for
31
- splititng of text.
32
- split_overlap: Number of words or sentences that overlap when creating
33
- the paragraphs. This is done as one sentence or 'some words' make sense
34
- when read in together with others. Therefore the overlap is used.
35
- remove_punc: to remove all Punctuation including ',' and '.' or not
36
- Return
37
- --------------
38
- List[Document]: When preprocessing pipeline is run, the output dictionary
39
- has four objects. For the Haysatck implementation of SDG classification we,
40
- need to use the List of Haystack Document, which can be fetched by
41
- key = 'documents' on output.
42
- """
43
-
44
- processing_pipeline = processingpipeline()
45
-
46
- output_pre = processing_pipeline.run(file_paths = file_path,
47
- params= {"FileConverter": {"file_path": file_path, \
48
- "file_name": file_name},
49
- "UdfPreProcessor": {"remove_punc": remove_punc, \
50
- "split_by": split_by, \
51
- "split_length":split_length,\
52
- "split_overlap": split_overlap, \
53
- "split_respect_sentence_boundary":split_respect_sentence_boundary}})
54
-
55
- return output_pre
56
-
57
-
58
- def app():
59
- with st.container():
60
- if 'filepath' in st.session_state:
61
- file_name = st.session_state['filename']
62
- file_path = st.session_state['filepath']
63
-
64
-
65
- all_documents = runPreprocessingPipeline(file_name= file_name,
66
- file_path= file_path, split_by= params['split_by'],
67
- split_length= params['split_length'],
68
- split_respect_sentence_boundary= params['split_respect_sentence_boundary'],
69
- split_overlap= params['split_overlap'], remove_punc= params['remove_punc'])
70
- paralist = paraLengthCheck(all_documents['documents'], 100)
71
- df = pd.DataFrame(paralist,columns = ['text','page'])
72
- # saving the dataframe to session state
73
- st.session_state['key0'] = df
74
-
75
- else:
76
- st.info("🤔 No document found, please try to upload it at the sidebar!")
 
77
  logging.warning("Terminated as no document provided")
 
1
+ # set path
2
+ import glob, os, sys;
3
+ sys.path.append('../utils')
4
+ from typing import List, Tuple
5
+ from typing_extensions import Literal
6
+ from haystack.schema import Document
7
+ from utils.config import get_classifier_params
8
+ from utils.preprocessing import processingpipeline,paraLengthCheck
9
+ import streamlit as st
10
+ import logging
11
+ import pandas as pd
12
+ params = get_classifier_params("preprocessing")
13
+
14
+ @st.cache_data
15
+ def runPreprocessingPipeline(file_name:str, file_path:str,
16
+ split_by: Literal["sentence", "word"] = 'sentence',
17
+ split_length:int = 2, split_respect_sentence_boundary:bool = False,
18
+ split_overlap:int = 0,remove_punc:bool = False)->List[Document]:
19
+ """
20
+ creates the pipeline and runs the preprocessing pipeline,
21
+ the params for pipeline are fetched from paramconfig
22
+ Params
23
+ ------------
24
+ file_name: filename, in case of streamlit application use
25
+ st.session_state['filename']
26
+ file_path: filepath, in case of streamlit application use st.session_state['filepath']
27
+ split_by: document splitting strategy either as word or sentence
28
+ split_length: when synthetically creating the paragrpahs from document,
29
+ it defines the length of paragraph.
30
+ split_respect_sentence_boundary: Used when using 'word' strategy for
31
+ splititng of text.
32
+ split_overlap: Number of words or sentences that overlap when creating
33
+ the paragraphs. This is done as one sentence or 'some words' make sense
34
+ when read in together with others. Therefore the overlap is used.
35
+ remove_punc: to remove all Punctuation including ',' and '.' or not
36
+ Return
37
+ --------------
38
+ List[Document]: When preprocessing pipeline is run, the output dictionary
39
+ has four objects. For the Haysatck implementation of SDG classification we,
40
+ need to use the List of Haystack Document, which can be fetched by
41
+ key = 'documents' on output.
42
+ """
43
+
44
+ processing_pipeline = processingpipeline()
45
+
46
+ output_pre = processing_pipeline.run(file_paths = file_path,
47
+ params= {"FileConverter": {"file_path": file_path, \
48
+ "file_name": file_name},
49
+ "UdfPreProcessor": {"remove_punc": remove_punc, \
50
+ "split_by": split_by, \
51
+ "split_length":split_length,\
52
+ "split_overlap": split_overlap, \
53
+ "split_respect_sentence_boundary":split_respect_sentence_boundary}})
54
+
55
+ return output_pre
56
+
57
+
58
+ def app():
59
+ with st.container():
60
+ if 'filepath' in st.session_state:
61
+ file_name = st.session_state['filename']
62
+ file_path = st.session_state['filepath']
63
+
64
+
65
+ all_documents = runPreprocessingPipeline(file_name= file_name,
66
+ file_path= file_path, split_by= params['split_by'],
67
+ split_length= params['split_length'],
68
+ split_respect_sentence_boundary= params['split_respect_sentence_boundary'],
69
+ split_overlap= params['split_overlap'], remove_punc= params['remove_punc'])
70
+ paralist = paraLengthCheck(all_documents['documents'], 100)
71
+ df = pd.DataFrame(paralist,columns = ['text','page'])
72
+ # saving the dataframe to session state
73
+ st.session_state['key0'] = df
74
+ st.session_state['pages'] = df.loc[len(df)-1,'page']
75
+
76
+ else:
77
+ st.info("🤔 No document found, please try to upload it at the sidebar!")
78
  logging.warning("Terminated as no document provided")