<commit_before>#!/usr/bin/env python

from copy import deepcopy
import pprint

MULTIVALUED_ATTRS = set([
    'arch',
    'groups',
    'makedepends',
    'checkdepends',
    'optdepends',
    'depends',
    'provides',
    'conflicts',
    'replaces',
    'options',
    'license',
    'source',
    'noextract',
    'backup',
])

def IsMultiValued(attr):
    return attr in MULTIVALUED_ATTRS


class AurInfo(object):
    def __init__(self):
        self._pkgbase = {}
        self._packages = {}

    def GetPackageNames(self):
        return self._packages.keys()

    def GetMergedPackage(self, pkgname):
        package = deepcopy(self._pkgbase)
        package['pkgname'] = pkgname
        for k, v in self._packages.get(pkgname).items():
            package[k] = deepcopy(v)
        return package

    def AddPackage(self, pkgname):
        self._packages[pkgname] = {}
        return self._packages[pkgname]

    def SetPkgbase(self, pkgbasename):
        self._pkgbase = {'pkgname' : pkgbasename}
        return self._pkgbase

def ParseAurinfoFromIterable(iterable):
    aurinfo = AurInfo()

    current_package = None

    for line in iterable:
        line = line.rstrip()

        if not line:
            # end of package
            current_package = None
            continue

        if not line.startswith('\t'):
            # start of new package
            try:
                key, value = line.split(' = ', 1)
            except ValueError:
                print('ERROR: unexpected header format: section=%s, line=%s' % (
                    current_package['pkgname'], line))
                continue

            if key == 'pkgbase':
                current_package = aurinfo.SetPkgbase(value)
            else:
                current_package = aurinfo.AddPackage(value)
        else:
            if current_package is None:
                print('ERROR: package attribute found outside of a package section')
                continue

            # package attribute
            try:
                key, value = line.lstrip('\t').split(' = ', 1)
            except ValueError:
                print('ERROR: unexpected attribute format: section=%s, line=%s' % (
                    current_package['pkgname'], line))

            if IsMultiValued(key):
                if not current_package.get(key):
                    current_package[key] = []
                current_package[key].append(value)
            else:
                if not current_package.get(key):
                    current_package[key] = value
                else:
                    print('WARNING: overwriting attribute %s: %s -> %s' % (
                        key, current_package[key], value))

    return aurinfo


def ParseAurinfo(filename='.AURINFO'):
    with open(filename) as f:
        return ParseAurinfoFromIterable(f)


if __name__ == '__main__':
    pp = pprint.PrettyPrinter(indent=4)
    aurinfo = ParseAurinfo()
    for pkgname in aurinfo.GetPackageNames():
        print(">>> merged package: %s" % pkgname)
        pp.pprint(aurinfo.GetMergedPackage(pkgname))
        print()
<commit_msg>parser: add a validation mode<commit_after>#!/usr/bin/env python

from copy import deepcopy
import pprint
import sys

MULTIVALUED_ATTRS = set([
    'arch',
    'groups',
    'makedepends',
    'checkdepends',
    'optdepends',
    'depends',
    'provides',
    'conflicts',
    'replaces',
    'options',
    'license',
    'source',
    'noextract',
    'backup',
])

def IsMultiValued(attr):
    return attr in MULTIVALUED_ATTRS


class AurInfo(object):
    def __init__(self):
        self._pkgbase = {}
        self._packages = {}

    def GetPackageNames(self):
        return self._packages.keys()

    def GetMergedPackage(self, pkgname):
        package = deepcopy(self._pkgbase)
        package['pkgname'] = pkgname
        for k, v in self._packages.get(pkgname).items():
            package[k] = deepcopy(v)
        return package

    def AddPackage(self, pkgname):
        self._packages[pkgname] = {}
        return self._packages[pkgname]

    def SetPkgbase(self, pkgbasename):
        self._pkgbase = {'pkgname' : pkgbasename}
        return self._pkgbase

class ECatcherInterface(object):
    def Catch(self, lineno, error):
        raise NotImplementedError


class StderrECatcher(ECatcherInterface):
    def Catch(self, lineno, error):
        print('ERROR[%d]: %s' % (lineno, error), file=sys.stderr)


class CollectionECatcher(ECatcherInterface):
    def __init__(self):
        self._errors = []

    def Catch(self, lineno, error):
        self._errors.append((lineno, error))

    def HasErrors(self):
        return len(self._errors) > 0


def ParseAurinfoFromIterable(iterable, ecatcher=None):
    aurinfo = AurInfo()

    if ecatcher is None:
        ecatcher = StderrECatcher()

    current_package = None
    lineno = 0

    for line in iterable:
        lineno += 1
        line = line.rstrip()

        if not line:
            # end of package
            current_package = None
            continue

        if not line.startswith('\t'):
            # start of new package
            try:
                key, value = line.split(' = ', 1)
            except ValueError:
                ecatcher.Catch(lineno, 'unexpected header format in section=%s' %
                    current_package['pkgname'])
                continue

            if key == 'pkgbase':
                current_package = aurinfo.SetPkgbase(value)
            else:
                current_package = aurinfo.AddPackage(value)
        else:
            if current_package is None:
                ecatcher.Catch(lineno, 'package attribute found outside of '
                               'a package section')
                continue

            # package attribute
            try:
                key, value = line.lstrip('\t').split(' = ', 1)
            except ValueError:
                ecatcher.Catch(lineno, 'unexpected attribute format in '
                               'section=%s' % current_package['pkgname'])

            if IsMultiValued(key):
                if not current_package.get(key):
                    current_package[key] = []
                current_package[key].append(value)
            else:
                if not current_package.get(key):
                    current_package[key] = value
                else:
                    ecatcher.Catch(lineno, 'overwriting attribute '
                                   '%s: %s -> %s' % (key, current_package[key],
                                                     value))

    return aurinfo


def ParseAurinfo(filename='.AURINFO', ecatcher=None):
    with open(filename) as f:
        return ParseAurinfoFromIterable(f, ecatcher)


def ValidateAurinfo(filename='.AURINFO'):
    ecatcher = CollectionECatcher()
    ParseAurinfo(filename, ecatcher)
    return not ecatcher.HasErrors()


if __name__ == '__main__':
    pp = pprint.PrettyPrinter(indent=4)

    if len(sys.argv) == 1:
        print('error: not enough arguments')
        sys.exit(1)
    elif len(sys.argv) == 2:
        action = sys.argv[1]
        filename = '.AURINFO'
    else:
        action, filename = sys.argv[1:3]

    if action == 'parse':
        aurinfo = ParseAurinfo()
        for pkgname in aurinfo.GetPackageNames():
            print(">>> merged package: %s" % pkgname)
            pp.pprint(aurinfo.GetMergedPackage(pkgname))
            print()
    elif action == 'validate':
        sys.exit(not ValidateAurinfo(filename))
    else:
        print('unknown action: %s' % action)
        sys.exit(1)

{'fraction_non_alphanumeric': 0.10731256599788806, 'fraction_numerical': 0.0021119324181626186, 'mean_word_length': 1.9019532746074301, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3344350', 'n_tokens_mistral': 2248, 'n_tokens_neox': 2146, 'n_words': 504}