<jupyter_start><jupyter_text>Open one of the files above<jupyter_code>def open_file(file):
    with open(file, 'r', encoding='utf-8') as f:
        lines = f.readlines()
        return lines<jupyter_output><empty_output><jupyter_text>Pattern to Identify Header<jupyter_code>PATTERN_SOURCE_DESTN = re.compile(r'\s+(?P<type>Source|Dest) : (?P<dir>C.*\b)')
def identify_header(lines, pattern):
    header_dict_regex = {'type': [], 'dir': []}
    for line in lines:
        source_destn_data_from_regex = re.finditer(pattern, line)
        for match in source_destn_data_from_regex:
            header_dict_regex['type'].append(match.group('type'))
            header_dict_regex['dir'].append(match.group('dir'))
    return header_dict_regex
headers = identify_header(open_file(file1), PATTERN_SOURCE_DESTN)
for key, val in headers.items():
    print(key, val)<jupyter_output>type ['Source', 'Dest']
dir ['C:\\RegularExpressionsWithDotNet\\robocopytest\\source\\தமிழ்\\हिन्दी\\English', 'C:\\RegularExpressionsWithDotNet\\robocopytest\\destn']<jupyter_text>Pattern to Capture Error Message<jupyter_code>PATTERN_ERROR_MSG = re.compile(r'(?P<ts>\b\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) ERROR '\
                               r'(?P<error>.*\b)')
def capture_error_msg(lines, pattern):
    error_dict_regex = {'ts': [], 'error': []}
    for line in lines:
        error_msg_data_from_regex = re.finditer(pattern, line)
        for match in error_msg_data_from_regex:
            error_dict_regex['ts'].append(match.group('ts'))
            error_dict_regex['error'].append(match.group('error'))
    return error_dict_regex
errors = capture_error_msg(open_file(file2), PATTERN_ERROR_MSG)
for key, val in errors.items():
    print(key, val)<jupyter_output>ts ['2016/05/28 19:24:30']
error ['2 (0x00000002) Accessing Source Directory C:\\RegularExpressionsWithDotNet\\robocopytest\\source2']<jupyter_text>Pattern to Capture Metrics Table<jupyter_code>PATTERN_METRICS_TBL = re.compile(r'\s+(?P<type>Dirs|Files|Bytes) :\s+'\
                                 r'(?P<total>\d+)\s+'\
                                 r'(?P<copied>\d+)\s+'\
                                 r'(?P<skipped>\d+)\s+'\
                                 r'(?P<mismatch>\d+)\s+'\
                                 r'(?P<failed>\d+)\s+'\
                                 r'(?P<extras>\d+)')
def capture_metrics_tbl(lines, pattern):
    metrics_dict_regex = {'type': [], 'total': [], 'copied': [], 'skipped': [], \
                          'mismatch': [], 'failed': [], 'extras': []}
    for line in lines:
        metrics_data_from_regex = re.finditer(pattern, line)
        for match in metrics_data_from_regex:
            metrics_dict_regex['type'].append(match.group('type'))
            metrics_dict_regex['total'].append(match.group('total'))
            metrics_dict_regex['copied'].append(match.group('copied'))
            metrics_dict_regex['skipped'].append(match.group('skipped'))
            metrics_dict_regex['mismatch'].append(match.group('mismatch'))
            metrics_dict_regex['failed'].append(match.group('failed'))
            metrics_dict_regex['extras'].append(match.group('extras'))
    return metrics_dict_regex
metrics = capture_metrics_tbl(open_file(file1), PATTERN_METRICS_TBL)
for key, val in metrics.items():
    print(key, val)<jupyter_output>type ['Dirs', 'Files', 'Bytes']
total ['7', '29', '133567']
copied ['6', '29', '133567']
skipped ['1', '0', '0']
mismatch ['0', '0', '0']
failed ['0', '0', '0']
extras ['0', '0', '0']<jupyter_text>Convert dictionaries to json format<jupyter_code>print(json.dumps(headers))
print(json.dumps(errors))
print(json.dumps(metrics))
with open('headers.json','w', encoding='utf-8') as wr:
        json.dump(headers, wr, ensure_ascii=False, indent=True)
with open('errors.json','w', encoding='utf-8') as wr:
        json.dump(errors, wr, ensure_ascii=False, indent=True)
with open('metrics.json','w', encoding='utf-8') as wr:
        json.dump(metrics, wr, ensure_ascii=False, indent=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17338609943111552, 'fraction_numerical': 0.016819193668068267, 'mean_word_length': 3.6006825938566553, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '637373', 'license': 'MIT', 'n_tokens_mistral': 1419, 'n_tokens_neox': 1376, 'n_words': 219, 'path': 'udemy/python-regular-expressions/my_progress/Project 1 - Robocopy Log File Parsing with Regex.ipynb', 'repo_name': 'djrgit/coursework', 'size': '4043'}