<jupyter_start><jupyter_text>Regex search in multiple pdf files<jupyter_code># create a list of files in the directory to be parsed, e.g. all pdf files

import glob
documents = glob.glob('*.pdf')
documents
# extract content of files
# results are highlighted with yellow background (ANSI escape code: \033[43m)

import textract
import re

regex = "[D][H][L]" #searchterm
area = 20 #space around results to be printed with the result

for i in documents:
    print('\033[4m'+'Document:'+i+'\033[0m'+'\n')
    
    text = textract.process(i).decode('utf-8') #use textract to convert pdf into text and format for further processing
    
    if re.findall(regex, text) == []: #check if any results are found, if not, next document
        print('No results.\n')
        
    else:
        print('%s hits \n' % len(re.findall(regex, text)))
        for m in re.finditer(regex, text):
            print('---------- Position %02d-%02d: %s ----------\n' % (m.start(), m.end(), m.group(0)))
            print(text[m.start()-area:m.start()]+'\033[43m'+m.group()+'\033[0m'+text[m.end():m.end()+area]+'\n')<jupyter_output>[4mDocument:test.pdf[0m

No results.

[4mDocument:test2.pdf[0m

2 hits 

---------- Position 00-03: DHL ----------

[43mDHL[0m Parcel For You
Inte

---------- Position 81-84: DHL ----------

,
please return to:
[43mDHL[0m Global Mail NL
Am E
{'fraction_non_alphanumeric': 0.19131714495952906, 'fraction_numerical': 0.03679175864606328, 'mean_word_length': 2.9650145772594754, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '337741', 'license': 'MIT', 'n_tokens_mistral': 499, 'n_tokens_neox': 461, 'n_words': 137, 'path': 'Regex_search_in_multiple_pdfs.ipynb', 'repo_name': 'calabria18/audit', 'size': '1359'}