<commit_before>#!/usr/bin/env python3
import re
'''
Get diffs for commits on Github with
```
curl -H "Accept: application/vnd.github.diff" \
  https://api.github.com/repos/bezi/gitorial/commits/d5f43a7a3f4b3cacfc0fea98751e70f65c9da79d
```
'''
def parse(diff_text):
    files = []
    for file_diff in re.split('diff --git a/.* b/.*\n', diff_text)[1:]:
        f = {}
        lines = file_diff.split('\n')
        f['name'] = lines[1][5:]
        f['chunks'] = []
        for file_chunk_diff in file_diff.split('\n@@')[1:]:
            lines = file_chunk_diff.split('\n')
            old_line_numbers, new_line_numbers = ((int(s), int(e))
                for (s, e) in re.findall('[-+](\d*),(\d*) ', lines[0]))
            lines[0] = lines[0][re.search(' .* @@', lines[0]).end():]
            old_line_number = old_line_numbers[0]<commit_msg>Fixed 'most glaring' bugs with parser<commit_after>#!/usr/bin/env python3
import re
'''
Get diffs for commits on Github with
```
curl -H "Accept: application/vnd.github.diff" \
  https://api.github.com/repos/bezi/gitorial/commits/d5f43a7a3f4b3cacfc0fea98751e70f65c9da79d
```
'''
def parse(diff_text):
    files = []
    for file_diff in re.split('diff --git a/.* b/.*\n', diff_text)[1:]:
        f = {}
        lines = file_diff.split('\n')
        while 'index ' not in lines[0]:
            lines = lines[1:]
        if '/dev/null' not in lines[1]:
            f['name'] = lines[1][5:]
        else:
            f['name'] = lines[2][5:]
        f['chunks'] = []
        for file_chunk_diff in file_diff.split('\n@@')[1:]:
            lines = file_chunk_diff.split('\n')
            old_line_numbers, new_line_numbers = ((int(s), int(e))
                for (s, e) in re.findall('[-+](\d*),(\d*) ', lines[0]))
            lines[0] = lines[0][re.search(' .* @@', lines[0]).end():]
            old_line_number = old_line_numbers[0]
{'fraction_non_alphanumeric': 0.19722073757349012, 'fraction_numerical': 0.034740780331373594, 'mean_word_length': 2.5454545454545454, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3256619', 'n_tokens_mistral': 733, 'n_tokens_neox': 693, 'n_words': 127}