<jupyter_start><jupyter_text>Based on Lesson 1<jupyter_code># Setup and define function for parsing
from tika import parser
def convertPdf2TxtWithTika(in_pdf_file, out_text_file):
    # Load a file and extract information
    print ("INFO: - reading file = " + in_pdf_file)
    
    raw = parser.from_file(in_pdf_file)
    text = raw['content']
    
    ## Post-processing explained at: 
    # https://medium.com/@justinboylantoomey/fast-text-extraction-with-python-and-tika-41ac34b0fe61
    # Convert to string
    text = str(text)
    # Ensure text is utf-8 formatted
    safe_text = text.encode('utf-8', errors='ignore')
    # Escape any \ issues
    safe_text = str(safe_text).replace('\\n', ' ')
    safe_text = str(safe_text).strip()
    
    # Write out extracted content
    text_pdf = open(out_text_file, 'w')
    print ("INFO: - writing file = " + out_text_file)
    text_pdf.write(text)
    text_pdf.close()
    return safe_text
# Find pdf files in a directory and process its content
import glob

txtfiles = []
inpath = '../data/legal/'
outpath = '../data/legal/'
count = 0
all_contents = ""
for file in glob.glob(inpath + '*.pdf'):
    justfile = os.path.basename(file)
    justfile = justfile.replace(".pdf","")
    print('INFO: processing file = ' + os.path.basename(file))
    output_file = outpath + justfile + '.txt'
    print('INFO: - in = ' + file + ', out = ' + output_file)
    result_text = convertPdf2TxtWithTika(file, output_file)
    all_contents = all_contents + result_text 
    count = count + 1
print('INFO: processed total files = ' + str(count))
all_contents_file_name = outpath + "all_contents.txt"
all_text_file = open(all_contents_file_name, 'w')
all_text_file.write(all_contents)
all_text_file.close()
print('INFO: see contents in file = ' + all_contents_file_name)<jupyter_output>INFO: processing file = Example-Frontiers-Terms and Conditions.pdf
INFO: - in = ../data/legal/Example-Frontiers-Terms and Conditions.pdf, out = ../data/legal/Example-Frontiers-Terms and Conditions.txt
INFO: - reading file = ../data/legal/Example-Frontiers-Terms and Conditions.pdf
INFO: - writing file = ../data/legal/Example-Frontiers-Terms and Conditions.txt
INFO: processing file = Example-MyADT-Terms.pdf
INFO: - in = ../data/legal/Example-MyADT-Terms.pdf, out = ../data/legal/Example-MyADT-Terms.txt
INFO: - reading file = ../data/legal/Example-MyADT-Terms.pdf
INFO: - writing file = ../data/legal/Example-MyADT-Terms.txt
INFO: processing file = Example-TDBank-PersonalAcctAgree.pdf
INFO: - in = ../data/legal/Example-TDBank-PersonalAcctAgree.pdf, out = ../data/legal/Example-TDBank-PersonalAcctAgree.txt
INFO: - reading file = ../data/legal/Example-TDBank-PersonalAcctAgree.pdf
INFO: - writing file = ../data/legal/Example-TDBank-PersonalAcctAgree.txt
INFO: processed total files = 3
INFO: see [...]
{'fraction_non_alphanumeric': 0.13501063075832742, 'fraction_numerical': 0.005315379163713678, 'mean_word_length': 3.768581081081081, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '563041', 'license': 'MIT', 'n_tokens_mistral': 978, 'n_tokens_neox': 943, 'n_words': 209, 'path': 'l7-language/code/Create TextCorpus FromPdfs.ipynb', 'repo_name': 'biplav-s/course-nl', 'size': '2822'}