<jupyter_start><jupyter_text>Match all occurrences of the given pattern in the given text.<jupyter_code>import re
path_to_file = "data/do.txt"<jupyter_output><empty_output><jupyter_text>approach 1: does not match across line breaks<jupyter_code># in the middle we have 2017.8.26, which looks like a date so 
# both the second and third elements should match one or 
# two-digit numbers (month and days)
process_pat = re.compile(r'\d{7}-\d{2}\.\d{4}\.\d{1,2}\.\d{1,2}\.\d{4}')
test1 = "0001040-18.2017.8.26.0100" # from the instructions
process_pat.findall(test1)
test2 = "0001040-18.2017.10.1.0100" # matches single-digit days and double-digit months too
process_pat.findall(test2)
test3 = "0001040-18.2017.8.1.0100 foo bar baz 0001040-18.2017.8.23.0100" # multiple occurrences
process_pat.findall(test3)
test4 = "0001040-18.2017.8.222.0100" # should NOT match
process_pat.findall(test4)
# numbers may be split across lines
test5 = """
0001040-18.2017.8
.26.0100
"""

# this will be solved in approach 2
process_pat.findall(test5)
def approach1(file,pattern):
   
    output = list()
    
    with open(file,"r") as f:
        for line in f:
            # there may be more than a single occurrence in the line so we must use findall
            matches = pattern.findall(line)
            
            for match in matches:
                output.append(match)
 
    # remove duplicates                
    return set(output)

# running
matches = approach1(path_to_file,process_pat)

print("{} distinct matches found: \n".format(len(matches)))
    
for match in matches:    
    print(match)<jupyter_output>8 distinct matches found: 

0033887-73.2017.8.26.0100
0047167-14.2017.8.26.0100
0001040-18.2017.8.26.0100
0028605-54.2017.8.26.0100
0032840-64.2017.8.26.0100
0030677-14.2017.8.26.0100
0026982-52.2017.8.26.0100
0047169-81.2017.8.26.0100<jupyter_text>approach 2:  match even across line breaks- read two lines at a time, so that we capture text which may have been split across lines- basic strategy: read two lines then rewind the file pointer 1 line (otherwise we would only be able to capture stuff every 2 lines)<jupyter_code>def approach2(file,pattern):
    
    line_break = r'\n'
   
    output = list()
    
    with open(file,"r") as f:           
        while True:
            
            line1 = f.readline()   
            last_position = f.tell()
            line2 = f.readline()
            
            # end of file
            if not line1: break
                           
            merged = re.sub(line_break,'',line1+line2)
                       
            # there may be more than a single occurrence in the line so we must use findall    
            matches = pattern.findall(merged)
            
            for match in matches:
                output.append(match)
            
            # rewind to position before last line was read
            f.seek(last_position)
    
    # remove duplicates
    return set(output)                
            
# running
matches = approach2(path_to_file,process_pat)

print("{} distinct matches found: \n".format(len(matches)))
    
for match in matches:    
    print(match)<jupyter_output>16 distinct matches found: 

0033887-73.2017.8.26.0100
0047167-14.2017.8.26.0100
0001040-18.2017.8.26.0100
1021197-29.2016.8.26.0100
0028605-54.2017.8.26.0100
1056022-04.2013.8.26.0100
1071666-79.2016.8.26.0100
0155113-55.2011.8.26.0100
0030677-14.2017.8.26.0100
0032840-64.2017.8.26.0100
1055991-47.2014.8.26.0100
0026982-52.2017.8.26.0100
1128654-91.2014.8.26.0100
0047169-81.2017.8.26.0100
1058515-46.2016.8.26.0100
1042408-29.2013.8.26.0100
{'fraction_non_alphanumeric': 0.11031241360243295, 'fraction_numerical': 0.1774951617362455, 'mean_word_length': 2.5024201355275895, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 29, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '591250', 'license': 'MIT', 'n_tokens_mistral': 1635, 'n_tokens_neox': 1222, 'n_words': 303, 'path': 'python3/notebooks/legal-documents/extract-process-codes.ipynb', 'repo_name': 'tobby2002/python-sandbox', 'size': '3617'}