<commit_before>from modeltree.query import ModelTreeQuerySet
from avocado.meta.formatters import registry

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

class BaseExporter(object):
    "Base class for all exporters"

    preferred_formats = []

    def __init__(self, queryset, concepts):
        if not isinstance(queryset, ModelTreeQuerySet):
            queryset = queryset._clone(klass=ModelTreeQuerySet)

        self.queryset = queryset
        self.concepts = concepts

    def _get_raw_query(self, fields):
        return self.queryset.select(*fields, inclue_pk=False)

    def _read_row(self, row, params):
        for i, (keys, length, formatter) in enumerate(params):
            part, row = row[:length], row[length:]
            values = OrderedDict(zip(keys, part))
            yield formatter(values, self.preferred_formats)

    def _get_keys(self, fields):
        return [f.field_name for f in fields]

    def read(self, concepts=None):
        concepts = concepts or self.concepts
        params = []
        select_fields = []

        for concept in concepts:
            cfields = concept.conceptfields.select_related('field')
            fields = [c.field for c in cfields]
            select_fields.extend(f.field for f in fields)
            formatter = registry[concept.formatter](cfields)
            params.append((self._get_keys(fields), len(cfields), formatter))

        for row in iter(self._get_raw_query(select_fields)):
            yield self._read_row(row, params)

    def write(self, *args, **kwargs):
        raise NotImplemented<commit_msg>Add better parser for non-conflicting keys Fix #10, clean up internal API<commit_after>from modeltree.query import ModelTreeQuerySet
from avocado.meta.formatters import registry

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

class BaseExporter(object):
    "Base class for all exporters"

    preferred_formats = []

    def __init__(self, queryset, concepts):
        if not isinstance(queryset, ModelTreeQuerySet):
            queryset = queryset._clone(klass=ModelTreeQuerySet)

        self.queryset = queryset
        self.concepts = concepts

    def _get_rows(self, fields):
        return self.queryset.select(*fields, inclue_pk=False)

    def _format_row(self, row, params):
        for i, (keys, length, formatter) in enumerate(params):
            part, row = row[:length], row[length:]
            values = OrderedDict(zip(keys, part))
            yield formatter(values, self.preferred_formats)

    def _get_keys(self, fields):
        # Best case scenario, no conflicts, return as is. Otherwise
        # duplicates found will be suffixed with a '_N' where N is the
        # occurred position.
        keys = [f.field_name for f in fields]

        if len(set(keys)) != len(fields):
            cnts = {}
            for i, key in enumerate(keys):
                if keys.count(key) > 1:
                    cnts.setdefault(key, 0)
                    keys[i] = '{}_{}'.format(key, cnts[key])
                    cnts[key] += 1
        return keys

    def read(self, concepts=None):
        concepts = concepts or self.concepts
        params = []
        select_fields = []

        for concept in concepts:
            cfields = concept.conceptfields.select_related('field')
            fields = [c.field for c in cfields]
            select_fields.extend(f.field for f in fields)
            formatter = registry[concept.formatter](cfields)
            params.append((self._get_keys(fields), len(cfields), formatter))

        for row in iter(self._get_rows(select_fields)):
            yield self._format_row(row, params)

    def write(self, *args, **kwargs):
        raise NotImplemented
{'fraction_non_alphanumeric': 0.09312696747114375, 'fraction_numerical': 0.0013116474291710388, 'mean_word_length': 2.5937794533459, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6516150', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1048, 'n_words': 296}