<commit_before>from bs4 import BeautifulSoup


def parse_links_for_text(string_with_links):
    strings = [link.get_text()
               for link in BeautifulSoup(string_with_links, 'html.parser').find_all('a')]
    splitted = [' '.join(s.split()).strip() for s in strings]
    return [s for s in splitted if s]
<commit_msg>use regexes instead of BeautifulSoup in parse_links_for_text<commit_after>import re
linkre = re.compile(r'<a.*?>(.*?)<\/a>')


def parse_links_for_text(string_with_links):
    matches = linkre.finditer(string_with_links)
    strings = [link.group(1) for link in matches if link]
    splitted = [' '.join(s.split()).strip() for s in strings]
    return [s for s in splitted if s]

{'fraction_non_alphanumeric': 0.12926136363636365, 'fraction_numerical': 0.002840909090909091, 'mean_word_length': 4.222222222222222, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2238243', 'n_tokens_mistral': 240, 'n_tokens_neox': 236, 'n_words': 63}