<commit_before>#!/usr/bin/env python3

from lxml import html
import argparse
import sys
import re
import logging
import requests

from models import LineChange

module = sys.modules['__main__'].__file__
log = logging.getLogger(module)

def load_blame_page(commit, file):
    blame_url = "https://github.com/hpi-swt2/wimi-portal/blame/" + commit + file
    page = requests.get(blame_url)

    html_tree = html.fromstring(page.content)
    return html_tree

def calc_modified_lines(added_lines, removed_lines):
    modified_lines = {}

    for key in set(removed_lines.keys()).intersection(set(added_lines.keys())):    
        modified_lines[key] = LineChange(key, LineChange.ChangeType.modified, added_lines[key].filename, added_lines[key].commit)
        del added_lines[key]
        del removed_lines[key]

    return added_lines, removed_lines, modified_lines

def load_file(filename):
    log.debug("Parsing {}".format(filename))

    with open(filename) as f:
        return f.readlines()

def parse_lines(lines):
    diff_start_re = re.compile('^@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@.*$')
    diff_long_commit_re = re.compile('^From ([a-f,0-9]{40}) [A-Z][a-z]{2} [A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4}$')
    diff_filename_re = re.compile('^diff --git (.*?) (.*?)$')
    diff_commits_re = re.compile('^index ([a-f,0-9]{7})\.\.([a-f,0-9]{7}).*$')

    blame_file = None
    current_file = None
    current_before_commit = None
    current_after_commit = None<commit_msg>Adapt to cahnged LineChange interface. Adapt file_name regexes.<commit_after>#!/usr/bin/env python3

from lxml import html
import argparse
import sys
import re
import logging
import requests

from models import LineChange

module = sys.modules['__main__'].__file__
log = logging.getLogger(module)

def load_blame_page(commit, file):
    blame_url = "https://github.com/hpi-swt2/wimi-portal/blame/" + commit + file
    page = requests.get(blame_url)

    html_tree = html.fromstring(page.content)
    return html_tree

def calc_modified_lines(added_lines, removed_lines):
    modified_lines = {}

    for key in set(removed_lines.keys()).intersection(set(added_lines.keys())):    
        modified_lines[key] = LineChange(key, LineChange.ChangeType.modified, added_lines[key].file_path, added_lines[key].commit_sha)
        del added_lines[key]
        del removed_lines[key]

    return added_lines, removed_lines, modified_lines

def load_file(filename):
    log.debug("Parsing {}".format(filename))

    with open(filename) as f:
        return f.readlines()

def parse_lines(lines):
    diff_start_re = re.compile('^@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@.*$')
    diff_long_commit_re = re.compile('^From ([a-f,0-9]{40}) [A-Z][a-z]{2} [A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4}$')
    diff_filename_re = re.compile('^diff --git a/(.*?) b/(.*?)$')
    diff_commits_re = re.compile('^index ([a-f,0-9]{7})\.\.([a-f,0-9]{7}).*$')

    blame_file = None
    current_file = None
    current_before_commit = None
    current_after_commit = None
{'fraction_non_alphanumeric': 0.178256611165524, 'fraction_numerical': 0.02415932092719556, 'mean_word_length': 4.089700996677741, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6210811', 'n_tokens_mistral': 1144, 'n_tokens_neox': 1128, 'n_words': 200}