<commit_before>import logging
import io
import os
import zipfile
import requests
import base64
from contextlib import closing
from pathlib import Path

import yaml

from toposort import toposort_flatten

from .client import client
from .constraints import parse as parse_constraints, parse_storage_constraint, parse_device_constraint
from .errors import JujuError
from . import utils, jasyncio, charmhub
from .origin import Channel
from .url import Schema, URL

log = logging.getLogger(__name__)


class BundleHandler:
    """
    Handle bundles by using the API to translate bundle YAML into a plan of
    steps and then dispatching each of those using the API.
    """
    def __init__(self, model, trusted=False, forced=False):
        self.model = model
        self.trusted = trusted
        self.forced = forced
        self.bundle = None
        self.overlays = []
        self.overlay_removed_charms = set()

        self.charmstore = model.charmstore
        self.plan = []
        self.references = {}
        self._units_by_app = {}
        self.origins = {}

        for unit_name, unit in model.units.items():
            app_units = self._units_by_app.setdefault(unit.application, [])
            app_units.append(unit_name)

        self.bundle_facade = client.BundleFacade.from_connection(
            model.connection())
        self.client_facade = client.ClientFacade.from_connection(
            model.connection())
        self.app_facade = client.ApplicationFacade.from_connection(
            model.connection())
        self.ann_facade = client.AnnotationsFacade.from_connection(
            model.connection())
        self.machine_manager_facade = client.MachineManagerFacade.from_connection(
            model.connection())

        # Feature detect if we have the new charms facade, otherwise fallback
        # to the client facade, when making calls.
        best_facade_version = client.CharmsFacade.best_facade_version(model.connection())
        if best_facade_version is not None and best_facade_version > 2:
            self.charms_facade = client.CharmsFacade.from_connection(
                model.connection())
        else:
            self.charms_facade = None

        # This describes all the change types that the BundleHandler supports.
        change_type_cls = [AddApplicationChange,
                           AddCharmChange,
                           AddMachineChange,
                           AddRelationChange,
                           AddUnitChange,
                           CreateOfferChange,
                           ConsumeOfferChange,
                           ExposeChange,
                           ScaleChange,
                           SetAnnotationsChange]
        self.change_types = {}
        for change_cls in change_type_cls:
            self.change_types[change_cls.method()] = change_cls

    async def _validate_bundle(self, bundle):
        """Validate the bundle for known issues, raises an error if it
        encounters a known problem
        """
        apps_dict = bundle.get('applications', {})
        for app_name in self.applications:
            app_dict = apps_dict[app_name]
            app_trusted = app_dict.get('trust')
            if (not self.trusted and not self.forced) and app_trusted:
                raise JujuError(
                    "Bundle cannot be deployed without trusting applications with your cloud credentials.\n"
                    "Please repeat the deploy command with the --trust argument if you consent to trust the following application\n"
                    " - {}\n".format(app_name)
                )
        return bundle

    async def _handle_local_charms(self, bundle, bundle_dir):
        """Search for references to local charms (i.e. filesystem paths)
        in the bundle. Upload the local charms to the model, and replace
        the filesystem paths with appropriate 'local:' paths in the bundle.

        Return the modified bundle.

        :param dict bundle: Bundle dictionary
        :return: Modified bundle dictionary

        """
        apps, args = [], []

        default_series = bundle.get('series')
        apps_dict = bundle.get('applications', {})
        for app_name in self.applications:
            app_dict = apps_dict[app_name]
            charm_dir = app_dict['charm']
            try:
                charm_path = (bundle_dir / charm_dir).resolve()
                if not (charm_path.is_dir() or
                        (charm_path.is_file() and
                         charm_path.suffix in ('.charm', '.zip'))):
                    continue
                charm_dir = str(charm_path)
            except ValueError:
                pass
            except FileNotFoundError:
                continue
            series = (
                app_dict.get('series') or
                default_series or
                await get_charm_series(charm_dir, self.model)
            )
            if not series:
                raise JujuError(
                    "Couldn't determine series for charm at {}. "
                    "Add a 'series' key to the bundle.".format(charm_dir))
            # Keep track of what we need to update. We keep a list of apps
            # that need to be updated, and a corresponding list of args
            # needed to update those apps.
            apps.append(app_name)
            args.append((charm_dir, series))

        if apps:
            # If we have apps to update, spawn all the coroutines concurrently
            # and wait for them to finish.
            charm_urls = await jasyncio.gather(*[
                self.model.add_local_charm_dir(*params)
                for params in args
            ])

            # Update the 'charm:' entry for each app with the new 'local:' url.
            for app_name, charm_url, (charm_dir, _) in zip(apps, charm_urls, args):
                resources = await self.model.add_local_resources(
                    app_name,
                    charm_url,
                    utils.get_local_charm_metadata(charm_dir),
                    resources=bundle.get('applications', {app_name: {}})[app_name].get("resources", {}),
                )
                apps_dict[app_name]['charm'] = charm_url
                apps_dict[app_name]["resources"] = resources

        return bundle

    def _resolve_include_file_config(self, bundle_dir):
        """if any of the applications (including the ones in the overlays)
        have "config: include-file:..." or "config:
        include-base64:...", then we have to resolve and inline them
        into the bundle here because they're all files with local
        relative paths, so backend can't handle them.

        """
        bundle_apps = [self.bundle.get('applications', {})]
        overlay_apps = [overlay.get('applications', {}) for overlay in self.overlays]

        for apps in bundle_apps + overlay_apps:
            for app_name, app in apps.items():

                if app and 'options' in app:
                    if 'config' in app['options'] and app['options']['config'].startswith('include-file'):
                        # resolve the file
                        if not bundle_dir:
                            raise NotImplementedError('unable to resolve paths for config:include-file for non-local charms')
                        try:
                            config_path = (bundle_dir / Path(app['options']['config'].split('//')[1])).resolve()
                        except IndexError:
                            raise JujuError('the path for the included file should start with // and be relative to the bundle')
                        if not config_path.exists():
                            raise JujuError('unable to locate config file : %s for : %s' % (config_path, app_name))

                        # get the contents of the file
                        config_contents = yaml.safe_load(config_path.read_text())

                        # inline the configurations for the current app into
                        # the app['options']
                        for key, val in config_contents[app_name].items():
                            app['options'][key] = val

                        # remove the 'include-file' config
                        app['options'].pop('config')

                    for option_key, option_val in app['options'].items():
                        if isinstance(option_val, str) and option_val.startswith('include-base64'):
                            # resolve the file
                            if not bundle_dir:
                                raise NotImplementedError('unable to resolve paths for config:include-base64 for non-local charms')
                            try:
                                base64_path = (bundle_dir / Path(option_val.split('//')[1])).resolve()
                            except IndexError:
                                raise JujuError('the path for the included base64 file should start with // and be relative to the bundle')

                            if not base64_path.exists():
                                raise JujuError('unable to locate the base64 file : %s for : %s' % (base64_path, app_name))

                            # inline the base64 encoded config value
                            base64_contents = base64.b64decode(base64_path.read_text())
                            app['options'][option_key] = base64_contents

        return self.bundle, self.overlays

    async def fetch_plan(self, charm_url, origin, overlays=[]):
        entity_id = charm_url.path()
        is_local = Schema.LOCAL.matches(charm_url.schema)
        bundle_dir = None

        if is_local and os.path.isfile(entity_id):
            bundle_yaml = Path(entity_id).read_text()
            bundle_dir = Path(entity_id).parent
        elif is_local and os.path.isdir(entity_id):
            bundle_yaml = (Path(entity_id) / "bundle.yaml").read_text()
            bundle_dir = Path(entity_id)

        if Schema.CHARM_STORE.matches(charm_url.schema):
            bundle_yaml = await self.charmstore.files(entity_id,
                                                      filename='bundle.yaml',
                                                      read_file=True)
        elif Schema.CHARM_HUB.matches(charm_url.schema):
            bundle_yaml = await self._download_bundle(charm_url, origin)

        if not bundle_yaml:
            raise JujuError('empty bundle, nothing to deploy')

        _bundles = [b for b in yaml.safe_load_all(bundle_yaml)]
        self.overlays = _bundles[1:]
        self.bundle = _bundles[0]

        if overlays != []:
            for overlay_yaml_path in overlays:
                try:
                    overlay_contents = Path(overlay_yaml_path).read_text()
                except (OSError, IOError) as e:
                    raise JujuError('unable to open overlay %s \n %s' % (overlay_yaml_path, e))
                self.overlays.extend(yaml.safe_load_all(overlay_contents))

        # gather the names of the removed charms so model.deploy
        # wouldn't wait for them to appear in the model
        for overlay in self.overlays:
            overlay_apps = overlay.get('applications', {})
            for charm_name, val in overlay_apps.items():
                if val is None:
                    self.overlay_removed_charms.add(charm_name)

        self.bundle = await self._validate_bundle(self.bundle)

        if is_local:
            self.bundle = await self._handle_local_charms(self.bundle, bundle_dir)

        self.bundle, self.overlays = self._resolve_include_file_config(bundle_dir)

        _yaml_data = [yaml.dump(self.bundle)]
        for overlay in self.overlays:
            _yaml_data.append(yaml.dump(overlay).replace('null', ''))
        yaml_data = "---\n".join(_yaml_data)

        self.plan = await self.bundle_facade.GetChanges(
            bundleurl=entity_id,
            yaml=yaml_data)

        if self.plan.errors:
            raise JujuError(self.plan.errors)

    async def _download_bundle(self, charm_url, origin):
        if self.charms_facade is None:
            raise JujuError('unable to download bundle for {} using the new charms facade. Upgrade controller to proceed.'.format(charm_url))

        resp = await self.charms_facade.GetDownloadInfos(entities=[{
            'charm-url': str(charm_url),
            'charm-origin': {
                'source': origin.source,
                'type': origin.type_,
                'id': origin.id_,
                'hash': origin.hash_,
                'revision': origin.revision,
                'risk': origin.risk,
                'track': origin.track,
                'architecture': origin.architecture,
                'os': origin.os,
                'series': origin.series,
            }
        }])
        if len(resp.results) != 1:
            raise JujuError("expected one result, received {}".format(resp.results))

        result = resp.results[0]
        if not result.url:
            raise JujuError("no url found for bundle {}".format(charm_url.name))

        bundle_resp = requests.get(result.url)
        bundle_resp.raise_for_status()

        with closing(bundle_resp), zipfile.ZipFile(io.BytesIO(bundle_resp.content)) as archive:
            return self._get_bundle_yaml(archive)

    def _get_bundle_yaml(self, archive):
        for member in archive.infolist():
            if member.filename == "bundle.yaml":
                return archive.read(member)
        raise JujuError("bundle.yaml not found")

    async def _resolve_charms(self):
        deployed = dict()

        specs = self.applications_specs

        for name in self.applications:
            spec = specs[name]
            app = self.model.applications.get(name, None)

            cons = None
            if app is not None:
                deployed[name] = name

                if is_local_charm(spec['charm']):
                    spec['charm'] = self.model.applications[name]
                    continue
                if spec['charm'] == app.charm_url:
                    continue

                cons = await app.get_constraints()

            if is_local_charm(spec['charm']):
                continue

            charm_url = URL.parse(spec['charm'])
            channel = None
            track, risk = '', ''
            if 'channel' in spec:
                channel = Channel.parse(spec['channel'])
                track, risk = channel.track, channel.risk
            if self.charms_facade is not None:
                if cons is not None and cons['arch'] != '':
                    architecture = cons['arch']
                else:
                    architecture = await self.model._resolve_architecture(charm_url)

                origin = client.CharmOrigin(source="charm-hub",
                                            architecture=architecture,
                                            risk=risk,
                                            track=track)
                charm_url, charm_origin, _ = await self.model._resolve_charm(charm_url, origin)

                spec['charm'] = str(charm_url)
            else:
                results = await self.model.charmstore.entity(str(charm_url))
                charm_url = results.get('Id', charm_url)
                charm_origin = client.CharmOrigin(source="charm-store",
                                                  risk=risk,
                                                  track=track)

            if str(channel) not in self.origins:
                self.origins[str(charm_url)] = {}<commit_msg>Set CharmOrigin fields correctly based on client version<commit_after>import logging
import io
import os
import zipfile
import requests
import base64
from contextlib import closing
from pathlib import Path

import yaml

from toposort import toposort_flatten

from .client import client
from .constraints import parse as parse_constraints, parse_storage_constraint, parse_device_constraint
from .errors import JujuError
from . import utils, jasyncio, charmhub
from .origin import Channel
from .url import Schema, URL

log = logging.getLogger(__name__)


class BundleHandler:
    """
    Handle bundles by using the API to translate bundle YAML into a plan of
    steps and then dispatching each of those using the API.
    """
    def __init__(self, model, trusted=False, forced=False):
        self.model = model
        self.trusted = trusted
        self.forced = forced
        self.bundle = None
        self.overlays = []
        self.overlay_removed_charms = set()

        self.charmstore = model.charmstore
        self.plan = []
        self.references = {}
        self._units_by_app = {}
        self.origins = {}

        for unit_name, unit in model.units.items():
            app_units = self._units_by_app.setdefault(unit.application, [])
            app_units.append(unit_name)

        self.bundle_facade = client.BundleFacade.from_connection(
            model.connection())
        self.client_facade = client.ClientFacade.from_connection(
            model.connection())
        self.app_facade = client.ApplicationFacade.from_connection(
            model.connection())
        self.ann_facade = client.AnnotationsFacade.from_connection(
            model.connection())
        self.machine_manager_facade = client.MachineManagerFacade.from_connection(
            model.connection())

        # Feature detect if we have the new charms facade, otherwise fallback
        # to the client facade, when making calls.
        best_facade_version = client.CharmsFacade.best_facade_version(model.connection())
        if best_facade_version is not None and best_facade_version > 2:
            self.charms_facade = client.CharmsFacade.from_connection(
                model.connection())
        else:
            self.charms_facade = None

        # This describes all the change types that the BundleHandler supports.
        change_type_cls = [AddApplicationChange,
                           AddCharmChange,
                           AddMachineChange,
                           AddRelationChange,
                           AddUnitChange,
                           CreateOfferChange,
                           ConsumeOfferChange,
                           ExposeChange,
                           ScaleChange,
                           SetAnnotationsChange]
        self.change_types = {}
        for change_cls in change_type_cls:
            self.change_types[change_cls.method()] = change_cls

    async def _validate_bundle(self, bundle):
        """Validate the bundle for known issues, raises an error if it
        encounters a known problem
        """
        apps_dict = bundle.get('applications', {})
        for app_name in self.applications:
            app_dict = apps_dict[app_name]
            app_trusted = app_dict.get('trust')
            if (not self.trusted and not self.forced) and app_trusted:
                raise JujuError(
                    "Bundle cannot be deployed without trusting applications with your cloud credentials.\n"
                    "Please repeat the deploy command with the --trust argument if you consent to trust the following application\n"
                    " - {}\n".format(app_name)
                )
        return bundle

    async def _handle_local_charms(self, bundle, bundle_dir):
        """Search for references to local charms (i.e. filesystem paths)
        in the bundle. Upload the local charms to the model, and replace
        the filesystem paths with appropriate 'local:' paths in the bundle.

        Return the modified bundle.

        :param dict bundle: Bundle dictionary
        :return: Modified bundle dictionary

        """
        apps, args = [], []

        default_series = bundle.get('series')
        apps_dict = bundle.get('applications', {})
        for app_name in self.applications:
            app_dict = apps_dict[app_name]
            charm_dir = app_dict['charm']
            try:
                charm_path = (bundle_dir / charm_dir).resolve()
                if not (charm_path.is_dir() or
                        (charm_path.is_file() and
                         charm_path.suffix in ('.charm', '.zip'))):
                    continue
                charm_dir = str(charm_path)
            except ValueError:
                pass
            except FileNotFoundError:
                continue
            series = (
                app_dict.get('series') or
                default_series or
                await get_charm_series(charm_dir, self.model)
            )
            if not series:
                raise JujuError(
                    "Couldn't determine series for charm at {}. "
                    "Add a 'series' key to the bundle.".format(charm_dir))
            # Keep track of what we need to update. We keep a list of apps
            # that need to be updated, and a corresponding list of args
            # needed to update those apps.
            apps.append(app_name)
            args.append((charm_dir, series))

        if apps:
            # If we have apps to update, spawn all the coroutines concurrently
            # and wait for them to finish.
            charm_urls = await jasyncio.gather(*[
                self.model.add_local_charm_dir(*params)
                for params in args
            ])

            # Update the 'charm:' entry for each app with the new 'local:' url.
            for app_name, charm_url, (charm_dir, _) in zip(apps, charm_urls, args):
                resources = await self.model.add_local_resources(
                    app_name,
                    charm_url,
                    utils.get_local_charm_metadata(charm_dir),
                    resources=bundle.get('applications', {app_name: {}})[app_name].get("resources", {}),
                )
                apps_dict[app_name]['charm'] = charm_url
                apps_dict[app_name]["resources"] = resources

        return bundle

    def _resolve_include_file_config(self, bundle_dir):
        """if any of the applications (including the ones in the overlays)
        have "config: include-file:..." or "config:
        include-base64:...", then we have to resolve and inline them
        into the bundle here because they're all files with local
        relative paths, so backend can't handle them.

        """
        bundle_apps = [self.bundle.get('applications', {})]
        overlay_apps = [overlay.get('applications', {}) for overlay in self.overlays]

        for apps in bundle_apps + overlay_apps:
            for app_name, app in apps.items():

                if app and 'options' in app:
                    if 'config' in app['options'] and app['options']['config'].startswith('include-file'):
                        # resolve the file
                        if not bundle_dir:
                            raise NotImplementedError('unable to resolve paths for config:include-file for non-local charms')
                        try:
                            config_path = (bundle_dir / Path(app['options']['config'].split('//')[1])).resolve()
                        except IndexError:
                            raise JujuError('the path for the included file should start with // and be relative to the bundle')
                        if not config_path.exists():
                            raise JujuError('unable to locate config file : %s for : %s' % (config_path, app_name))

                        # get the contents of the file
                        config_contents = yaml.safe_load(config_path.read_text())

                        # inline the configurations for the current app into
                        # the app['options']
                        for key, val in config_contents[app_name].items():
                            app['options'][key] = val

                        # remove the 'include-file' config
                        app['options'].pop('config')

                    for option_key, option_val in app['options'].items():
                        if isinstance(option_val, str) and option_val.startswith('include-base64'):
                            # resolve the file
                            if not bundle_dir:
                                raise NotImplementedError('unable to resolve paths for config:include-base64 for non-local charms')
                            try:
                                base64_path = (bundle_dir / Path(option_val.split('//')[1])).resolve()
                            except IndexError:
                                raise JujuError('the path for the included base64 file should start with // and be relative to the bundle')

                            if not base64_path.exists():
                                raise JujuError('unable to locate the base64 file : %s for : %s' % (base64_path, app_name))

                            # inline the base64 encoded config value
                            base64_contents = base64.b64decode(base64_path.read_text())
                            app['options'][option_key] = base64_contents

        return self.bundle, self.overlays

    async def fetch_plan(self, charm_url, origin, overlays=[]):
        entity_id = charm_url.path()
        is_local = Schema.LOCAL.matches(charm_url.schema)
        bundle_dir = None

        if is_local and os.path.isfile(entity_id):
            bundle_yaml = Path(entity_id).read_text()
            bundle_dir = Path(entity_id).parent
        elif is_local and os.path.isdir(entity_id):
            bundle_yaml = (Path(entity_id) / "bundle.yaml").read_text()
            bundle_dir = Path(entity_id)

        if Schema.CHARM_STORE.matches(charm_url.schema):
            bundle_yaml = await self.charmstore.files(entity_id,
                                                      filename='bundle.yaml',
                                                      read_file=True)
        elif Schema.CHARM_HUB.matches(charm_url.schema):
            bundle_yaml = await self._download_bundle(charm_url, origin)

        if not bundle_yaml:
            raise JujuError('empty bundle, nothing to deploy')

        _bundles = [b for b in yaml.safe_load_all(bundle_yaml)]
        self.overlays = _bundles[1:]
        self.bundle = _bundles[0]

        if overlays != []:
            for overlay_yaml_path in overlays:
                try:
                    overlay_contents = Path(overlay_yaml_path).read_text()
                except (OSError, IOError) as e:
                    raise JujuError('unable to open overlay %s \n %s' % (overlay_yaml_path, e))
                self.overlays.extend(yaml.safe_load_all(overlay_contents))

        # gather the names of the removed charms so model.deploy
        # wouldn't wait for them to appear in the model
        for overlay in self.overlays:
            overlay_apps = overlay.get('applications', {})
            for charm_name, val in overlay_apps.items():
                if val is None:
                    self.overlay_removed_charms.add(charm_name)

        self.bundle = await self._validate_bundle(self.bundle)

        if is_local:
            self.bundle = await self._handle_local_charms(self.bundle, bundle_dir)

        self.bundle, self.overlays = self._resolve_include_file_config(bundle_dir)

        _yaml_data = [yaml.dump(self.bundle)]
        for overlay in self.overlays:
            _yaml_data.append(yaml.dump(overlay).replace('null', ''))
        yaml_data = "---\n".join(_yaml_data)

        self.plan = await self.bundle_facade.GetChanges(
            bundleurl=entity_id,
            yaml=yaml_data)

        if self.plan.errors:
            raise JujuError(self.plan.errors)

    async def _download_bundle(self, charm_url, origin):
        if self.charms_facade is None:
            raise JujuError('unable to download bundle for {} using the new charms facade. Upgrade controller to proceed.'.format(charm_url))

        id = origin.id_ if origin.id_ else ''
        hash = origin.hash_ if origin.hash_ else ''
        charm_origin = {
            'source': origin.source,
            'type': origin.type_,
            'id': id,
            'hash': hash,
            'revision': origin.revision,
            'risk': origin.risk,
            'track': origin.track,
            'architecture': origin.architecture,
        }
        if self.model.connection().is_using_old_client:
            charm_origin['os'] = origin.os
            charm_origin['series'] = origin.series
        else:
            charm_origin['base'] = origin.base

        resp = await self.charms_facade.GetDownloadInfos(entities=[{
            'charm-url': str(charm_url),
            'charm-origin': charm_origin
        }])
        if len(resp.results) != 1:
            raise JujuError("expected one result, received {}".format(resp.results))

        result = resp.results[0]
        if not result.url:
            raise JujuError("no url found for bundle {}".format(charm_url.name))

        bundle_resp = requests.get(result.url)
        bundle_resp.raise_for_status()

        with closing(bundle_resp), zipfile.ZipFile(io.BytesIO(bundle_resp.content)) as archive:
            return self._get_bundle_yaml(archive)

    def _get_bundle_yaml(self, archive):
        for member in archive.infolist():
            if member.filename == "bundle.yaml":
                return archive.read(member)
        raise JujuError("bundle.yaml not found")

    async def _resolve_charms(self):
        deployed = dict()

        specs = self.applications_specs

        for name in self.applications:
            spec = specs[name]
            app = self.model.applications.get(name, None)

            cons = None
            if app is not None:
                deployed[name] = name

                if is_local_charm(spec['charm']):
                    spec['charm'] = self.model.applications[name]
                    continue
                if spec['charm'] == app.charm_url:
                    continue

                cons = await app.get_constraints()

            if is_local_charm(spec['charm']):
                continue

            charm_url = URL.parse(spec['charm'])
            channel = None
            track, risk = '', ''
            if 'channel' in spec:
                channel = Channel.parse(spec['channel'])
                track, risk = channel.track, channel.risk
            if self.charms_facade is not None:
                if cons is not None and cons['arch'] != '':
                    architecture = cons['arch']
                else:
                    architecture = await self.model._resolve_architecture(charm_url)

                origin = client.CharmOrigin(source="charm-hub",
                                            architecture=architecture,
                                            risk=risk,
                                            track=track)
                charm_url, charm_origin, _ = await self.model._resolve_charm(charm_url, origin)
                # import pdb;pdb.set_trace()
                spec['charm'] = str(charm_url)
            else:
                results = await self.model.charmstore.entity(str(charm_url))
                charm_url = results.get('Id', charm_url)
                charm_origin = client.CharmOrigin(source="charm-store",
                                                  risk=risk,
                                                  track=track)

            if str(channel) not in self.origins:
                self.origins[str(charm_url)] = {}
{'fraction_non_alphanumeric': 0.08487482945711838, 'fraction_numerical': 0.002347939207411873, 'mean_word_length': 1.700077101002313, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1146932', 'n_tokens_mistral': 8309, 'n_tokens_neox': 7933, 'n_words': 2132}