<commit_before># vim: set ts=4 sw=4 et: coding=UTF-8

import re
import os

from .fileutils import FileUtils

LICENSES_CHANGES = 'licenses_changes.txt'
TEX_CONVERSIONS = 'tex_conversions.txt'
PKGCONFIG_CONVERSIONS = 'pkgconfig_conversions.txt'
PERL_CONVERSIONS = 'perl_conversions.txt'
CMAKE_CONVERSIONS = 'cmake_conversions.txt'
GROUPS_LIST = 'allowed_groups.txt'
BRACKETING_EXCLUDES = 'excludes-bracketing.txt'


def parse_rpm_showrc():
    macros = []

    re_rc_macrofunc = re.compile(r'^-[0-9]+[:=]\s(\w+)\(.*')
    output = os.popen('rpm --showrc')
    for line in output:
        line = line.rstrip('\n')
        found_macro = re_rc_macrofunc.sub(r'\1', line)
        if found_macro != line:
            macros += [found_macro]
    output.close()
    return macros


def load_keywords_whitelist():
    keywords = []

    files = FileUtils()
    files.open_datafile(BRACKETING_EXCLUDES)
    for line in files.f:
        keywords.append(line.rstrip('\n'))
    files.close()

    return keywords


def find_macros_with_arg(spec):
    macrofuncs = []

    re_spec_macrofunc = re.compile(r'^\s*%define\s(\w+)\(.*')
    files = FileUtils()
    files.open(spec, 'r')
    for line in files.f:
        line = line.rstrip('\n')
        found_macro = re_spec_macrofunc.sub(r'\1', line)
        if found_macro != line:
            macrofuncs += [found_macro]
    files.close()
    return macrofuncs


def read_conversion_changes(conversion_file):
    conversions = {}

    files = FileUtils()
    files.open_datafile(conversion_file)
    for line in files.f:
        # the values are split by  ': '
        pair = line.split(': ')
        conversions[pair[0]] = pair[1][:-1]
    files.close()
    return conversions


def read_tex_changes():
    return read_conversion_changes(CMAKE_CONVERSIONS)


def read_pkgconfig_changes():
    return read_conversion_changes(PKGCONFIG_CONVERSIONS)


def read_perl_changes():
    return read_conversion_changes(PERL_CONVERSIONS)


def read_cmake_changes():
    return read_conversion_changes(CMAKE_CONVERSIONS)


def read_licenses_changes():
    licenses = {}

    files = FileUtils()
    files.open_datafile(LICENSES_CHANGES)
    # Header starts with # first line so skip
    next(files.f)
    for line in files.f:
        # strip newline
        line = line.rstrip('\n')
        # file has format
        # correct license string<tab>known bad license string<commit_msg>Fix copy&pasto for variable definition.<commit_after># vim: set ts=4 sw=4 et: coding=UTF-8

import re
import os

from .fileutils import FileUtils

LICENSES_CHANGES = 'licenses_changes.txt'
TEX_CONVERSIONS = 'tex_conversions.txt'
PKGCONFIG_CONVERSIONS = 'pkgconfig_conversions.txt'
PERL_CONVERSIONS = 'perl_conversions.txt'
CMAKE_CONVERSIONS = 'cmake_conversions.txt'
GROUPS_LIST = 'allowed_groups.txt'
BRACKETING_EXCLUDES = 'excludes-bracketing.txt'


def parse_rpm_showrc():
    macros = []

    re_rc_macrofunc = re.compile(r'^-[0-9]+[:=]\s(\w+)\(.*')
    output = os.popen('rpm --showrc')
    for line in output:
        line = line.rstrip('\n')
        found_macro = re_rc_macrofunc.sub(r'\1', line)
        if found_macro != line:
            macros += [found_macro]
    output.close()
    return macros


def load_keywords_whitelist():
    keywords = []

    files = FileUtils()
    files.open_datafile(BRACKETING_EXCLUDES)
    for line in files.f:
        keywords.append(line.rstrip('\n'))
    files.close()

    return keywords


def find_macros_with_arg(spec):
    macrofuncs = []

    re_spec_macrofunc = re.compile(r'^\s*%define\s(\w+)\(.*')
    files = FileUtils()
    files.open(spec, 'r')
    for line in files.f:
        line = line.rstrip('\n')
        found_macro = re_spec_macrofunc.sub(r'\1', line)
        if found_macro != line:
            macrofuncs += [found_macro]
    files.close()
    return macrofuncs


def read_conversion_changes(conversion_file):
    conversions = {}

    files = FileUtils()
    files.open_datafile(conversion_file)
    for line in files.f:
        # the values are split by  ': '
        pair = line.split(': ')
        conversions[pair[0]] = pair[1][:-1]
    files.close()
    return conversions


def read_tex_changes():
    return read_conversion_changes(TEX_CONVERSIONS)


def read_pkgconfig_changes():
    return read_conversion_changes(PKGCONFIG_CONVERSIONS)


def read_perl_changes():
    return read_conversion_changes(PERL_CONVERSIONS)


def read_cmake_changes():
    return read_conversion_changes(CMAKE_CONVERSIONS)


def read_licenses_changes():
    licenses = {}

    files = FileUtils()
    files.open_datafile(LICENSES_CHANGES)
    # Header starts with # first line so skip
    next(files.f)
    for line in files.f:
        # strip newline
        line = line.rstrip('\n')
        # file has format
        # correct license string<tab>known bad license string
{'fraction_non_alphanumeric': 0.11718426501035197, 'fraction_numerical': 0.004140786749482402, 'mean_word_length': 3.260141093474427, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2821034', 'n_tokens_mistral': 1703, 'n_tokens_neox': 1627, 'n_words': 329}