<commit_before># -*- coding: utf-8 -*-
import copy
from xlwt import Workbook, easyxf
from itertools import groupby

from django.http import HttpResponse
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils.translation import ugettext_lazy as _
from django.db.models.fields import DateTimeField, DateField
from django.utils.encoding import force_unicode
from django.db.models import Q
from django import forms
from django.forms.models import fields_for_model
from django.db.models.related import RelatedObject
from django.db.models import ForeignKey
from django.conf import settings
from django.contrib.auth.models import User
from django.forms import MultipleChoiceField
from django.forms.widgets import SelectMultiple
from model_report.forms import ConfigForm, GroupByForm
from model_report.utils import base_label, ReportValue, ReportRow
from model_report.highcharts import HighchartRender
from model_report.widgets import RangeField
from model_report.export_pdf import render_to_pdf


import arial10


class FitSheetWrapper(object):
    """Try to fit columns to max size of any entry.
    To use, wrap this around a worksheet returned from the
    workbook's add_sheet method, like follows:

        sheet = FitSheetWrapper(book.add_sheet(sheet_name))

    The worksheet interface remains the same: this is a drop-in wrapper
    for auto-sizing columns.
    """
    def __init__(self, sheet):
        self.sheet = sheet
        self.widths = dict()
        self.heights = dict()

    def write(self, r, c, label='', *args, **kwargs):
        self.sheet.write(r, c, label, *args, **kwargs)
        self.sheet.row(r).collapse = True
        bold = False
        if args:
            style = args[0]
            bold = str(style.font.bold) in ('1', 'true', 'True')
        width = int(arial10.fitwidth(label, bold))
        if width > self.widths.get(c, 0):
            self.widths[c] = width
            self.sheet.col(c).width = width

        height = int(arial10.fitheight(label, bold))
        if height > self.heights.get(r, 0):
            self.heights[r] = height
            self.sheet.row(r).height = height

    def __getattr__(self, attr):
        return getattr(self.sheet, attr)


try:
    from collections import OrderedDict
except:
    OrderedDict = dict  # pyflakes:ignore


def autodiscover():
    """
    Auto-discover INSTALLED_APPS report.py modules and fail silently when
    not present. Borrowed form django.contrib.admin
    """

    from django.utils.importlib import import_module
    from django.utils.module_loading import module_has_submodule

    global reports

    for app in settings.INSTALLED_APPS:
        mod = import_module(app)
        # Attempt to import the app's admin module.
        try:
            before_import_registry = copy.copy(reports)
            import_module('%s.reports' % app)
        except:
            # Reset the model registry to the state before the last import as
            # this import will have to reoccur on the next request and this
            # could raise NotRegistered and AlreadyRegistered exceptions
            # (see #8245).
            reports = before_import_registry

            # Decide whether to bubble up this error. If the app just
            # doesn't have an admin module, we can ignore the error
            # attempting to import it, otherwise we want it to bubble up.
            if module_has_submodule(mod, 'reports'):
                raise


class ReportClassManager(object):
    """
    Class to handle registered reports class.
    """

    _register = OrderedDict()

    def __init__(self):
        self._register = OrderedDict()

    def register(self, slug, rclass):
        if slug in self._register:
            raise ValueError('Slug already exists: %s' % slug)
        setattr(rclass, 'slug', slug)
        self._register[slug] = rclass

    def get_report(self, slug):
        # return class
        return self._register.get(slug, None)

    def get_reports(self):
        # return clasess
        return self._register.values()


reports = ReportClassManager()


_cache_class = {}


def cache_return(fun):
    """
    Usages of this decorator have been removed from the ReportAdmin base class.

    Caching method returns gets in the way of customization at the implementation level
    now that report instances can be modified based on request data.
    """
    def wrap(self, *args, **kwargs):
        cache_field = '%s_%s' % (self.__class__.__name__, fun.func_name)
        if cache_field in _cache_class:
            return _cache_class[cache_field]
        result = fun(self, *args, **kwargs)
        _cache_class[cache_field] = result
        return result
    return wrap


class ReportAdmin(object):
    """
    Class to represent a Report.
    """

    fields = []
    """List of fields or lookup fields for query results to be listed."""

    model = None
    """Primary django model to query."""

    list_filter = ()
    """List of fields or lookup fields to filter data."""
    
    list_filter_widget = {}
    """Widget for list filter field"""
    
    list_filter_queryset = {}
    """ForeignKey custom queryset"""

    list_order_by = ()
    """List of fields or lookup fields to order data."""

    list_group_by = ()
    """List of fields or lookup fields to group data."""

    list_serie_fields = ()
    """List of fields to group by results in chart."""

    base_template_name = 'base.html'
    """Template file name to render the report."""

    template_name = 'model_report/report.html'
    """Template file name to render the report."""

    title = None
    """Title of the report."""

    type = 'report'
    """"report" for only report and  "chart" for report and chart graphic results."""

    group_totals = {}
    """Dictionary with field name as key and function to calculate their values."""

    report_totals = {}
    """Dictionary with field name as key and function to calculate their values."""

    override_field_values = {}
    """
    Dictionary with field name as key and function to parse their original values.

    ::

        override_field_values = {
            'men': men_format,
            'women': women_format
        }
    """
    override_field_formats = {}
    """Dictionary with field name as key and function to parse their value after :func:`override_field_values`."""

    override_field_labels = {}
    """Dictionary with field name as key and function to parse the column label."""

    override_field_choices = {}
    """#TODO"""

    override_field_filter_values = {}
    """#TODO"""

    override_group_value = {}
    """#TODO"""

    chart_types = ()
    """List of highchart types."""

    exports = ('excel', 'pdf')
    """Alternative render report as "pdf" or "csv"."""

    inlines = []
    """List of other's Report related to the main report."""

    query_set = None
    """#TODO"""

    always_show_full_username = False

    def __init__(self, parent_report=None, request=None):
        self.parent_report = parent_report
        self.request = request
        model_fields = []
        model_m2m_fields = []
        self.related_inline_field = None
        self.related_inline_accessor = None
        self.related_fields = []
        for field in self.get_query_field_names():
            try:
                m2mfields = []
                if '__' in field:  # IF field has lookup
                    pre_field = None
                    base_model = self.model
                    for field_lookup in field.split("__"):
                        if not pre_field:
                            pre_field = base_model._meta.get_field_by_name(field_lookup)[0]
                            if 'ManyToManyField' in unicode(pre_field) or isinstance(pre_field, RelatedObject):
                                m2mfields.append(pre_field)
                        elif isinstance(pre_field, RelatedObject):
                            base_model = pre_field.model
                            pre_field = base_model._meta.get_field_by_name(field_lookup)[0]
                        else:
                            if 'Date' in unicode(pre_field):
                                pre_field = pre_field
                            else:
                                base_model = pre_field.rel.to
                                pre_field = base_model._meta.get_field_by_name(field_lookup)[0]
                    model_field = pre_field
                else:
                    if not 'self.' in field:
                        model_field = self.model._meta.get_field_by_name(field)[0]
                    else:
                        get_attr = lambda s: getattr(s, field.split(".")[1])
                        get_attr.verbose_name = field
                        model_field = field
            except IndexError:
                raise ValueError('The field "%s" does not exist in model "%s".' % (field, self.model._meta.module_name))
            model_fields.append([model_field, field])
            if m2mfields:
                model_m2m_fields.append([model_field, field, len(model_fields) - 1, m2mfields])
        self.model_fields = model_fields
        self.model_m2m_fields = model_m2m_fields
        if parent_report:
            self.related_inline_field = [f for f, x in self.model._meta.get_fields_with_model() if f.rel and hasattr(f.rel, 'to') and f.rel.to is self.parent_report.model][0]
            self.related_inline_accessor = self.related_inline_field.related.get_accessor_name()
            self.related_fields = ["%s__%s" % (pfield.model._meta.module_name, attname) for pfield, attname in self.parent_report.model_fields if not isinstance(pfield, (str, unicode)) and  pfield.model == self.related_inline_field.rel.to]
            self.related_inline_filters = []

            for pfield, pattname in self.parent_report.model_fields:
                for cfield, cattname in self.model_fields:
                    try:
                        if pattname in cattname:
                            if pfield.model == cfield.model:
                                self.related_inline_filters.append([pattname, cattname, self.parent_report.get_fields().index(pattname)])
                    except Exception, e:
                        pass


    def _get_grouper_text(self, groupby_field, value):
        try:
            model_field = [mfield for mfield, field in self.model_fields if field == groupby_field][0]
        except:
            model_field = None
        value = self.get_grouper_text(value, groupby_field, model_field)
        if value is None or unicode(value) == u'None':
            if groupby_field is None or unicode(groupby_field) == u'None':
                value = force_unicode(_('Results'))
            else:
                value = force_unicode(_('Nothing'))
        return value

    def _get_value_text(self, index, value):
        try:
            model_field = self.model_fields[index][0]
        except:
            model_field = None

        value = self.get_value_text(value, index, model_field)
        if value is None or unicode(value) == u'None':
            value = ''
        if value == [None]:
            value = []
        return value

    def get_grouper_text(self, value, field, model_field):
        try:
            if not isinstance(model_field, (str, unicode)):
                obj = model_field.model(**{field: value})
                if hasattr(obj, 'get_%s_display' % field):
                    value = getattr(obj, 'get_%s_display' % field)()
        except:
            pass
        return value

    # @cache_return
    def get_m2m_field_names(self):
        return [field for ffield, field, index, mfield in self.model_m2m_fields]

    def get_value_text(self, value, index, model_field):
        try:
            if not isinstance(model_field, (str, unicode)):
                obj = model_field.model(**{model_field.name: value})
                if hasattr(obj, 'get_%s_display' % model_field.name):
                    return getattr(obj, 'get_%s_display' % model_field.name)()
        except:
            pass
        return value

    def get_empty_row_asdict(self, collection, default_value=[]):
        erow = {}
        for field in collection:
            erow[field] = copy.copy(default_value)
        return dict(copy.deepcopy(erow))

    def reorder_dictrow(self, dictrow):
        return [dictrow[field_name] for field_name in self.get_fields()]

    def get_fields(self):
        return [x for x in self.fields if not x in self.related_fields]

    def get_column_names(self, ignore_columns={}):
        """
        Return the list of columns
        """
        values = []
        for field, field_name in self.model_fields:
            if field_name in ignore_columns:
                continue
            caption = self.override_field_labels.get(field_name, base_label)(self, field)
            values.append(caption)
        return values

    # @cache_return
    def get_query_field_names(self):
        values = []
        for field in self.get_fields():
            if not 'self.' in field:
                values.append(field.split(".")[0])
            else:
                values.append(field)
        return values

    # @cache_return
    def get_query_set(self, filter_kwargs):
        """
        Return the the queryset
        """
        qs = self.model.objects.all()
        for k, v in filter_kwargs.items():
            if not v is None and v != '':
                if hasattr(v, 'values_list'):
                    v = v.values_list('pk', flat=True)
                    k = '%s__pk__in' % k.split("__")[0]
                elif isinstance(v, list):
                    if len(v) > 1:
                        k = '%s__in' % k
                    elif len(v) == 1:
                        if v[0] == '':
                            choices = []
                            for field in self.model_fields:
                                if field[1] == k:
                                    for c in field[0].choices:
                                        choices.append(c[0])
                            v = choices
                            k = '%s__in' % k
                        else:
                            v = v[0]
                    else:
                        pass
                qs = qs.filter(Q(**{k: v}))
        self.query_set = qs.distinct()
        return self.query_set

    def get_title(self):
        """
        Return the report title
        """
        title = self.title or None
        if not title:
            if not self.model:
                title = _('Unnamed report')
            else:
                title = force_unicode(self.model._meta.verbose_name_plural).lower().capitalize()
        return title

    def get_render_context(self, request, extra_context={}, by_row=None):
        context_request = request or self.request
        related_fields = []
        filter_related_fields = {}
        if self.parent_report and by_row:
            for mfield, cfield, index in self.related_inline_filters:
                filter_related_fields[cfield] = by_row[index].value

        try:
            form_groupby = self.get_form_groupby(context_request)
            form_filter = self.get_form_filter(context_request)
            form_config = self.get_form_config(context_request)<commit_msg>Removed not used variable         related_fields = []<commit_after># -*- coding: utf-8 -*-
import copy
from xlwt import Workbook, easyxf
from itertools import groupby

from django.http import HttpResponse
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils.translation import ugettext_lazy as _
from django.db.models.fields import DateTimeField, DateField
from django.utils.encoding import force_unicode
from django.db.models import Q
from django import forms
from django.forms.models import fields_for_model
from django.db.models.related import RelatedObject
from django.db.models import ForeignKey
from django.conf import settings
from django.contrib.auth.models import User
from django.forms import MultipleChoiceField
from django.forms.widgets import SelectMultiple
from model_report.forms import ConfigForm, GroupByForm
from model_report.utils import base_label, ReportValue, ReportRow
from model_report.highcharts import HighchartRender
from model_report.widgets import RangeField
from model_report.export_pdf import render_to_pdf


import arial10


class FitSheetWrapper(object):
    """Try to fit columns to max size of any entry.
    To use, wrap this around a worksheet returned from the
    workbook's add_sheet method, like follows:

        sheet = FitSheetWrapper(book.add_sheet(sheet_name))

    The worksheet interface remains the same: this is a drop-in wrapper
    for auto-sizing columns.
    """
    def __init__(self, sheet):
        self.sheet = sheet
        self.widths = dict()
        self.heights = dict()

    def write(self, r, c, label='', *args, **kwargs):
        self.sheet.write(r, c, label, *args, **kwargs)
        self.sheet.row(r).collapse = True
        bold = False
        if args:
            style = args[0]
            bold = str(style.font.bold) in ('1', 'true', 'True')
        width = int(arial10.fitwidth(label, bold))
        if width > self.widths.get(c, 0):
            self.widths[c] = width
            self.sheet.col(c).width = width

        height = int(arial10.fitheight(label, bold))
        if height > self.heights.get(r, 0):
            self.heights[r] = height
            self.sheet.row(r).height = height

    def __getattr__(self, attr):
        return getattr(self.sheet, attr)


try:
    from collections import OrderedDict
except:
    OrderedDict = dict  # pyflakes:ignore


def autodiscover():
    """
    Auto-discover INSTALLED_APPS report.py modules and fail silently when
    not present. Borrowed form django.contrib.admin
    """

    from django.utils.importlib import import_module
    from django.utils.module_loading import module_has_submodule

    global reports

    for app in settings.INSTALLED_APPS:
        mod = import_module(app)
        # Attempt to import the app's admin module.
        try:
            before_import_registry = copy.copy(reports)
            import_module('%s.reports' % app)
        except:
            # Reset the model registry to the state before the last import as
            # this import will have to reoccur on the next request and this
            # could raise NotRegistered and AlreadyRegistered exceptions
            # (see #8245).
            reports = before_import_registry

            # Decide whether to bubble up this error. If the app just
            # doesn't have an admin module, we can ignore the error
            # attempting to import it, otherwise we want it to bubble up.
            if module_has_submodule(mod, 'reports'):
                raise


class ReportClassManager(object):
    """
    Class to handle registered reports class.
    """

    _register = OrderedDict()

    def __init__(self):
        self._register = OrderedDict()

    def register(self, slug, rclass):
        if slug in self._register:
            raise ValueError('Slug already exists: %s' % slug)
        setattr(rclass, 'slug', slug)
        self._register[slug] = rclass

    def get_report(self, slug):
        # return class
        return self._register.get(slug, None)

    def get_reports(self):
        # return clasess
        return self._register.values()


reports = ReportClassManager()


_cache_class = {}


def cache_return(fun):
    """
    Usages of this decorator have been removed from the ReportAdmin base class.

    Caching method returns gets in the way of customization at the implementation level
    now that report instances can be modified based on request data.
    """
    def wrap(self, *args, **kwargs):
        cache_field = '%s_%s' % (self.__class__.__name__, fun.func_name)
        if cache_field in _cache_class:
            return _cache_class[cache_field]
        result = fun(self, *args, **kwargs)
        _cache_class[cache_field] = result
        return result
    return wrap


class ReportAdmin(object):
    """
    Class to represent a Report.
    """

    fields = []
    """List of fields or lookup fields for query results to be listed."""

    model = None
    """Primary django model to query."""

    list_filter = ()
    """List of fields or lookup fields to filter data."""
    
    list_filter_widget = {}
    """Widget for list filter field"""
    
    list_filter_queryset = {}
    """ForeignKey custom queryset"""

    list_order_by = ()
    """List of fields or lookup fields to order data."""

    list_group_by = ()
    """List of fields or lookup fields to group data."""

    list_serie_fields = ()
    """List of fields to group by results in chart."""

    base_template_name = 'base.html'
    """Template file name to render the report."""

    template_name = 'model_report/report.html'
    """Template file name to render the report."""

    title = None
    """Title of the report."""

    type = 'report'
    """"report" for only report and  "chart" for report and chart graphic results."""

    group_totals = {}
    """Dictionary with field name as key and function to calculate their values."""

    report_totals = {}
    """Dictionary with field name as key and function to calculate their values."""

    override_field_values = {}
    """
    Dictionary with field name as key and function to parse their original values.

    ::

        override_field_values = {
            'men': men_format,
            'women': women_format
        }
    """
    override_field_formats = {}
    """Dictionary with field name as key and function to parse their value after :func:`override_field_values`."""

    override_field_labels = {}
    """Dictionary with field name as key and function to parse the column label."""

    override_field_choices = {}
    """#TODO"""

    override_field_filter_values = {}
    """#TODO"""

    override_group_value = {}
    """#TODO"""

    chart_types = ()
    """List of highchart types."""

    exports = ('excel', 'pdf')
    """Alternative render report as "pdf" or "csv"."""

    inlines = []
    """List of other's Report related to the main report."""

    query_set = None
    """#TODO"""

    always_show_full_username = False

    def __init__(self, parent_report=None, request=None):
        self.parent_report = parent_report
        self.request = request
        model_fields = []
        model_m2m_fields = []
        self.related_inline_field = None
        self.related_inline_accessor = None
        self.related_fields = []
        for field in self.get_query_field_names():
            try:
                m2mfields = []
                if '__' in field:  # IF field has lookup
                    pre_field = None
                    base_model = self.model
                    for field_lookup in field.split("__"):
                        if not pre_field:
                            pre_field = base_model._meta.get_field_by_name(field_lookup)[0]
                            if 'ManyToManyField' in unicode(pre_field) or isinstance(pre_field, RelatedObject):
                                m2mfields.append(pre_field)
                        elif isinstance(pre_field, RelatedObject):
                            base_model = pre_field.model
                            pre_field = base_model._meta.get_field_by_name(field_lookup)[0]
                        else:
                            if 'Date' in unicode(pre_field):
                                pre_field = pre_field
                            else:
                                base_model = pre_field.rel.to
                                pre_field = base_model._meta.get_field_by_name(field_lookup)[0]
                    model_field = pre_field
                else:
                    if not 'self.' in field:
                        model_field = self.model._meta.get_field_by_name(field)[0]
                    else:
                        get_attr = lambda s: getattr(s, field.split(".")[1])
                        get_attr.verbose_name = field
                        model_field = field
            except IndexError:
                raise ValueError('The field "%s" does not exist in model "%s".' % (field, self.model._meta.module_name))
            model_fields.append([model_field, field])
            if m2mfields:
                model_m2m_fields.append([model_field, field, len(model_fields) - 1, m2mfields])
        self.model_fields = model_fields
        self.model_m2m_fields = model_m2m_fields
        if parent_report:
            self.related_inline_field = [f for f, x in self.model._meta.get_fields_with_model() if f.rel and hasattr(f.rel, 'to') and f.rel.to is self.parent_report.model][0]
            self.related_inline_accessor = self.related_inline_field.related.get_accessor_name()
            self.related_fields = ["%s__%s" % (pfield.model._meta.module_name, attname) for pfield, attname in self.parent_report.model_fields if not isinstance(pfield, (str, unicode)) and  pfield.model == self.related_inline_field.rel.to]
            self.related_inline_filters = []

            for pfield, pattname in self.parent_report.model_fields:
                for cfield, cattname in self.model_fields:
                    try:
                        if pattname in cattname:
                            if pfield.model == cfield.model:
                                self.related_inline_filters.append([pattname, cattname, self.parent_report.get_fields().index(pattname)])
                    except Exception, e:
                        pass


    def _get_grouper_text(self, groupby_field, value):
        try:
            model_field = [mfield for mfield, field in self.model_fields if field == groupby_field][0]
        except:
            model_field = None
        value = self.get_grouper_text(value, groupby_field, model_field)
        if value is None or unicode(value) == u'None':
            if groupby_field is None or unicode(groupby_field) == u'None':
                value = force_unicode(_('Results'))
            else:
                value = force_unicode(_('Nothing'))
        return value

    def _get_value_text(self, index, value):
        try:
            model_field = self.model_fields[index][0]
        except:
            model_field = None

        value = self.get_value_text(value, index, model_field)
        if value is None or unicode(value) == u'None':
            value = ''
        if value == [None]:
            value = []
        return value

    def get_grouper_text(self, value, field, model_field):
        try:
            if not isinstance(model_field, (str, unicode)):
                obj = model_field.model(**{field: value})
                if hasattr(obj, 'get_%s_display' % field):
                    value = getattr(obj, 'get_%s_display' % field)()
        except:
            pass
        return value

    # @cache_return
    def get_m2m_field_names(self):
        return [field for ffield, field, index, mfield in self.model_m2m_fields]

    def get_value_text(self, value, index, model_field):
        try:
            if not isinstance(model_field, (str, unicode)):
                obj = model_field.model(**{model_field.name: value})
                if hasattr(obj, 'get_%s_display' % model_field.name):
                    return getattr(obj, 'get_%s_display' % model_field.name)()
        except:
            pass
        return value

    def get_empty_row_asdict(self, collection, default_value=[]):
        erow = {}
        for field in collection:
            erow[field] = copy.copy(default_value)
        return dict(copy.deepcopy(erow))

    def reorder_dictrow(self, dictrow):
        return [dictrow[field_name] for field_name in self.get_fields()]

    def get_fields(self):
        return [x for x in self.fields if not x in self.related_fields]

    def get_column_names(self, ignore_columns={}):
        """
        Return the list of columns
        """
        values = []
        for field, field_name in self.model_fields:
            if field_name in ignore_columns:
                continue
            caption = self.override_field_labels.get(field_name, base_label)(self, field)
            values.append(caption)
        return values

    # @cache_return
    def get_query_field_names(self):
        values = []
        for field in self.get_fields():
            if not 'self.' in field:
                values.append(field.split(".")[0])
            else:
                values.append(field)
        return values

    # @cache_return
    def get_query_set(self, filter_kwargs):
        """
        Return the the queryset
        """
        qs = self.model.objects.all()
        for k, v in filter_kwargs.items():
            if not v is None and v != '':
                if hasattr(v, 'values_list'):
                    v = v.values_list('pk', flat=True)
                    k = '%s__pk__in' % k.split("__")[0]
                elif isinstance(v, list):
                    if len(v) > 1:
                        k = '%s__in' % k
                    elif len(v) == 1:
                        if v[0] == '':
                            choices = []
                            for field in self.model_fields:
                                if field[1] == k:
                                    for c in field[0].choices:
                                        choices.append(c[0])
                            v = choices
                            k = '%s__in' % k
                        else:
                            v = v[0]
                    else:
                        pass
                qs = qs.filter(Q(**{k: v}))
        self.query_set = qs.distinct()
        return self.query_set

    def get_title(self):
        """
        Return the report title
        """
        title = self.title or None
        if not title:
            if not self.model:
                title = _('Unnamed report')
            else:
                title = force_unicode(self.model._meta.verbose_name_plural).lower().capitalize()
        return title

    def get_render_context(self, request, extra_context={}, by_row=None):
        context_request = request or self.request
        filter_related_fields = {}
        if self.parent_report and by_row:
            for mfield, cfield, index in self.related_inline_filters:
                filter_related_fields[cfield] = by_row[index].value

        try:
            form_groupby = self.get_form_groupby(context_request)
            form_filter = self.get_form_filter(context_request)
            form_config = self.get_form_config(context_request)
{'fraction_non_alphanumeric': 0.09131084681874309, 'fraction_numerical': 0.0027945668421394685, 'mean_word_length': 2.0000974848898423, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4083160', 'n_tokens_mistral': 8628, 'n_tokens_neox': 8286, 'n_words': 2379}