<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import os

import flask
import jsonschema
import netaddr
from neutronclient.common import exceptions as n_exceptions
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_utils import excutils

from kuryr import app
from kuryr import binding
from kuryr.common import config
from kuryr.common import constants
from kuryr.common import exceptions
from kuryr import schemata
from kuryr import utils

MANDATORY_NEUTRON_EXTENSION = "subnet_allocation"

cfg.CONF.import_group('neutron_client', 'kuryr.common.config')
cfg.CONF.import_group('keystone_client', 'kuryr.common.config')

keystone_conf = cfg.CONF.keystone_client
username = keystone_conf.admin_user
tenant_name = keystone_conf.admin_tenant_name
password = keystone_conf.admin_password
auth_token = keystone_conf.admin_token
auth_uri = keystone_conf.auth_uri.rstrip('/')

neutron_uri = cfg.CONF.neutron_client.neutron_uri
enable_dhcp = cfg.CONF.neutron_client.enable_dhcp

if username and password:
    # Authenticate with password crentials
    app.neutron = utils.get_neutron_client(
        url=neutron_uri, username=username, tenant_name=tenant_name,
        password=password, auth_url=auth_uri)
else:
    app.neutron = utils.get_neutron_client_simple(
        url=neutron_uri, token=auth_token)


def check_for_neutron_ext_support():
    """Validates for mandatory extension support availability in neutron.
    """
    try:
        app.neutron.show_extension(MANDATORY_NEUTRON_EXTENSION)
    except n_exceptions.NeutronClientException as e:
        if e.status_code == n_exceptions.NotFound.status_code:
            raise exceptions.MandatoryApiMissing(
                            "Neutron extension with alias '{0}' not found"
                            .format(MANDATORY_NEUTRON_EXTENSION))


# TODO(tfukushima): Retrieve the following subnet names from the config file.
SUBNET_POOLS_V4 = [
    p.strip() for p in os.environ.get('SUBNET_POOLS_V4', 'kuryr').split(',')]
SUBNET_POOLS_V6 = [
    p.strip() for p in os.environ.get('SUBNET_POOLS_V6', 'kuryr6').split(',')]

app.neutron.format = 'json'


def _cache_default_subnetpool_ids(app):
    """Caches IDs of the default subnetpools as app.DEFAULT_POOL_IDS."""
    if not hasattr(app, 'DEFAULT_POOL_IDS'):
        default_subnetpool_id_set = set()
        try:
            subnetpool_names = SUBNET_POOLS_V4 + SUBNET_POOLS_V6
            for subnetpool_name in subnetpool_names:
                subnetpools = app.neutron.list_subnetpools(
                    name=subnetpool_name)
                for subnetpool in subnetpools['subnetpools']:
                    default_subnetpool_id_set.add(subnetpool['id'])
        except n_exceptions.NeutronClientException as ex:
            app.logger.error("Error happened during retrieving the default "
                             "subnet pools.".format(ex))
        app.DEFAULT_POOL_IDS = frozenset(default_subnetpool_id_set)


def _get_networks_by_attrs(**attrs):
    networks = app.neutron.list_networks(**attrs)
    if len(networks.get('networks', [])) > 1:
        raise exceptions.DuplicatedResourceException(
            "Multiple Neutron networks exist for the params {0}"
            .format(', '.join(['{0}={1}'.format(k, v)
                               for k, v in attrs.items()])))
    return networks['networks']


def _get_subnets_by_attrs(**attrs):
    subnets = app.neutron.list_subnets(**attrs)
    if len(subnets.get('subnets', [])) > 2:  # subnets for IPv4 and/or IPv6
        raise exceptions.DuplicatedResourceException(
            "Multiple Neutron subnets exist for the params {0} "
            .format(', '.join(['{0}={1}'.format(k, v)
                               for k, v in attrs.items()])))
    return subnets['subnets']


def _get_ports_by_attrs(**attrs):
    ports = app.neutron.list_ports(**attrs)
    if len(ports.get('ports', [])) > 1:
        raise exceptions.DuplicatedResourceException(
            "Multiple Neutron ports exist for the params {0} "
            .format(', '.join(['{0}={1}'.format(k, v)
                               for k, v in attrs.items()])))
    return ports['ports']


def _get_subnetpools_by_attrs(**attrs):
    subnetpools = app.neutron.list_subnetpools(**attrs)
    if len(subnetpools.get('subnetpools', [])) > 1:
        raise exceptions.DuplicatedResourceException(
            "Multiple Neutron subnetspool exist for the params {0} "
            .format(', '.join(['{0}={1}'.format(k, v)
                               for k, v in attrs.items()])))
    return subnetpools['subnetpools']


def _process_subnet(neutron_network_id, endpoint_id, interface_cidr,
                    new_subnets, existing_subnets, pool_id=None):
    cidr = netaddr.IPNetwork(interface_cidr)
    subnet_network = str(cidr.network)
    subnet_cidr = '/'.join([subnet_network,
                            str(cidr.prefixlen)])
    subnets = _get_subnets_by_attrs(
        network_id=neutron_network_id, cidr=subnet_cidr)
    if subnets:
        existing_subnets += subnets
    else:
        new_subnet = {
            'name': '-'.join([endpoint_id, subnet_network]),
            'network_id': neutron_network_id,
            'ip_version': cidr.version,
            'cidr': subnet_cidr,
            'enable_dhcp': enable_dhcp,
        }
        if pool_id:
            del new_subnet['cidr']
            new_subnet['subnetpool_id'] = pool_id

        new_subnets.append(new_subnet)


def _get_or_create_subnet_by_pools(subnetpool_names, neutron_network_id,
                                   endpoint_id, new_subnets, existing_subnets):
    for subnetpool_name in subnetpool_names:
        pools = _get_subnetpools_by_attrs(name=subnetpool_name)
        if pools:
            pool = pools[0]
            prefixes = pool['prefixes']
            for prefix in prefixes:
                _process_subnet(neutron_network_id, endpoint_id, prefix,
                                new_subnets, existing_subnets,
                                pool_id=pool['id'])
    if not (new_subnets or existing_subnets):
        raise exceptions.NoResourceException(
            "No subnetpools with name {0} is found."
            .format(', '.join(subnetpool_names)))


def _handle_allocation_from_pools(neutron_network_id, endpoint_id,
                                  new_subnets, existing_subnets):
    _get_or_create_subnet_by_pools(SUBNET_POOLS_V4, neutron_network_id,
                                   endpoint_id, new_subnets, existing_subnets)
    _get_or_create_subnet_by_pools(SUBNET_POOLS_V6, neutron_network_id,
                                   endpoint_id, new_subnets, existing_subnets)

    created_subnets_response = {'subnets': []}
    if new_subnets:
        created_subnets_response = app.neutron.create_subnet(
            {'subnets': new_subnets})

    return created_subnets_response


def _handle_explicit_allocation(neutron_network_id, endpoint_id,
                                interface_cidrv4, interface_cidrv6,
                                new_subnets, existing_subnets):
    if interface_cidrv4:
        _process_subnet(neutron_network_id, endpoint_id, interface_cidrv4,
                        new_subnets, existing_subnets)

    if interface_cidrv6:
        _process_subnet(neutron_network_id, endpoint_id, interface_cidrv6,
                        new_subnets, existing_subnets)

    created_subnets_response = {'subnets': []}
    if new_subnets:
        # Bulk create operation of subnets
        created_subnets_response = app.neutron.create_subnet(
            {'subnets': new_subnets})

    return created_subnets_response


def _process_interface_address(port_dict, subnets_dict_by_id,
                               response_interface):
    assigned_address = port_dict['ip_address']
    subnet_id = port_dict['subnet_id']
    subnet = subnets_dict_by_id[subnet_id]
    cidr = netaddr.IPNetwork(subnet['cidr'])
    assigned_address += '/' + str(cidr.prefixlen)
    if cidr.version == 4:
        response_interface['Address'] = assigned_address
    else:
        response_interface['AddressIPv6'] = assigned_address


def _create_subnets_and_or_port(interface, neutron_network_id, endpoint_id):
    response_interface = {}
    existing_subnets = []
    created_subnets_response = {'subnets': []}
    # v4 and v6 Subnets for bulk creation.
    new_subnets = []

    interface_cidrv4 = interface.get('Address', '')
    interface_cidrv6 = interface.get('AddressIPv6', '')
    interface_mac = interface.get('MacAddress', '')

    if interface_cidrv4 or interface_cidrv6:
        created_subnets_response = _handle_explicit_allocation(
            neutron_network_id, endpoint_id, interface_cidrv4,
            interface_cidrv6, new_subnets, existing_subnets)
    else:
        app.logger.info("Retrieving or creating subnets with the default "
                        "subnetpool because Address and AddressIPv6 are "
                        "not given.")
        created_subnets_response = _handle_allocation_from_pools(
            neutron_network_id, endpoint_id, new_subnets, existing_subnets)

    try:
        port = {
            'name': '-'.join([endpoint_id, 'port']),
            'admin_state_up': True,
            'network_id': neutron_network_id,
            'device_owner': constants.DEVICE_OWNER,
            'device_id': endpoint_id,
        }
        if interface_mac:
            port['mac_address'] = interface_mac
        created_subnets = created_subnets_response.get('subnets', [])
        all_subnets = created_subnets + existing_subnets
        fixed_ips = port['fixed_ips'] = []
        for subnet in all_subnets:
            fixed_ip = {'subnet_id': subnet['id']}
            if interface_cidrv4 or interface_cidrv6:
                if subnet['ip_version'] == 4 and interface_cidrv4:
                    cidr = netaddr.IPNetwork(interface_cidrv4)
                elif subnet['ip_version'] == 6 and interface_cidrv6:
                    cidr = netaddr.IPNetwork(interface_cidrv6)
                subnet_cidr = '/'.join([str(cidr.network),
                                        str(cidr.prefixlen)])
                if subnet['cidr'] != subnet_cidr:
                    continue
                fixed_ip['ip_address'] = str(cidr.ip)
            fixed_ips.append(fixed_ip)
        created_port = app.neutron.create_port({'port': port})
        created_port = created_port['port']

        created_fixed_ips = created_port['fixed_ips']
        subnets_dict_by_id = {subnet['id']: subnet
                              for subnet in all_subnets}
        if not interface_mac:
            response_interface['MacAddress'] = created_port['mac_address']

        if not (interface_cidrv4 or interface_cidrv6):
            if 'ip_address' in created_port:
                _process_interface_address(
                    created_port, subnets_dict_by_id, response_interface)
            for fixed_ip in created_fixed_ips:
                _process_interface_address(
                    fixed_ip, subnets_dict_by_id, response_interface)
    except n_exceptions.NeutronClientException as ex:
        app.logger.error("Error happend during creating a "
                         "Neutron port: {0}".format(ex))
        # Rollback the subnets creation
        for subnet in created_subnets:
            app.neutron.delete_subnet(subnet['id'])
        raise
    return response_interface


@app.route('/Plugin.Activate', methods=['POST'])
def plugin_activate():
    return flask.jsonify(constants.SCHEMA['PLUGIN_ACTIVATE'])


@app.route('/NetworkDriver.GetCapabilities', methods=['POST'])
def plugin_scope():
    capabilities = {'Scope': cfg.CONF.capability_scope}
    return flask.jsonify(capabilities)


@app.route('/NetworkDriver.DiscoverNew', methods=['POST'])
def network_driver_discover_new():
    """The callback function for the DiscoverNew notification.

    The DiscoverNew notification includes the type of the
    resource that has been newly discovered and possibly other
    information associated with the resource.

    See the following link for more details about the spec:
<commit_msg>Add docstring for plugin_activate<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import os

import flask
import jsonschema
import netaddr
from neutronclient.common import exceptions as n_exceptions
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_utils import excutils

from kuryr import app
from kuryr import binding
from kuryr.common import config
from kuryr.common import constants
from kuryr.common import exceptions
from kuryr import schemata
from kuryr import utils

MANDATORY_NEUTRON_EXTENSION = "subnet_allocation"

cfg.CONF.import_group('neutron_client', 'kuryr.common.config')
cfg.CONF.import_group('keystone_client', 'kuryr.common.config')

keystone_conf = cfg.CONF.keystone_client
username = keystone_conf.admin_user
tenant_name = keystone_conf.admin_tenant_name
password = keystone_conf.admin_password
auth_token = keystone_conf.admin_token
auth_uri = keystone_conf.auth_uri.rstrip('/')

neutron_uri = cfg.CONF.neutron_client.neutron_uri
enable_dhcp = cfg.CONF.neutron_client.enable_dhcp

if username and password:
    # Authenticate with password crentials
    app.neutron = utils.get_neutron_client(
        url=neutron_uri, username=username, tenant_name=tenant_name,
        password=password, auth_url=auth_uri)
else:
    app.neutron = utils.get_neutron_client_simple(
        url=neutron_uri, token=auth_token)


def check_for_neutron_ext_support():
    """Validates for mandatory extension support availability in neutron.
    """
    try:
        app.neutron.show_extension(MANDATORY_NEUTRON_EXTENSION)
    except n_exceptions.NeutronClientException as e:
        if e.status_code == n_exceptions.NotFound.status_code:
            raise exceptions.MandatoryApiMissing(
                            "Neutron extension with alias '{0}' not found"
                            .format(MANDATORY_NEUTRON_EXTENSION))


# TODO(tfukushima): Retrieve the following subnet names from the config file.
SUBNET_POOLS_V4 = [
    p.strip() for p in os.environ.get('SUBNET_POOLS_V4', 'kuryr').split(',')]
SUBNET_POOLS_V6 = [
    p.strip() for p in os.environ.get('SUBNET_POOLS_V6', 'kuryr6').split(',')]

app.neutron.format = 'json'


def _cache_default_subnetpool_ids(app):
    """Caches IDs of the default subnetpools as app.DEFAULT_POOL_IDS."""
    if not hasattr(app, 'DEFAULT_POOL_IDS'):
        default_subnetpool_id_set = set()
        try:
            subnetpool_names = SUBNET_POOLS_V4 + SUBNET_POOLS_V6
            for subnetpool_name in subnetpool_names:
                subnetpools = app.neutron.list_subnetpools(
                    name=subnetpool_name)
                for subnetpool in subnetpools['subnetpools']:
                    default_subnetpool_id_set.add(subnetpool['id'])
        except n_exceptions.NeutronClientException as ex:
            app.logger.error("Error happened during retrieving the default "
                             "subnet pools.".format(ex))
        app.DEFAULT_POOL_IDS = frozenset(default_subnetpool_id_set)


def _get_networks_by_attrs(**attrs):
    networks = app.neutron.list_networks(**attrs)
    if len(networks.get('networks', [])) > 1:
        raise exceptions.DuplicatedResourceException(
            "Multiple Neutron networks exist for the params {0}"
            .format(', '.join(['{0}={1}'.format(k, v)
                               for k, v in attrs.items()])))
    return networks['networks']


def _get_subnets_by_attrs(**attrs):
    subnets = app.neutron.list_subnets(**attrs)
    if len(subnets.get('subnets', [])) > 2:  # subnets for IPv4 and/or IPv6
        raise exceptions.DuplicatedResourceException(
            "Multiple Neutron subnets exist for the params {0} "
            .format(', '.join(['{0}={1}'.format(k, v)
                               for k, v in attrs.items()])))
    return subnets['subnets']


def _get_ports_by_attrs(**attrs):
    ports = app.neutron.list_ports(**attrs)
    if len(ports.get('ports', [])) > 1:
        raise exceptions.DuplicatedResourceException(
            "Multiple Neutron ports exist for the params {0} "
            .format(', '.join(['{0}={1}'.format(k, v)
                               for k, v in attrs.items()])))
    return ports['ports']


def _get_subnetpools_by_attrs(**attrs):
    subnetpools = app.neutron.list_subnetpools(**attrs)
    if len(subnetpools.get('subnetpools', [])) > 1:
        raise exceptions.DuplicatedResourceException(
            "Multiple Neutron subnetspool exist for the params {0} "
            .format(', '.join(['{0}={1}'.format(k, v)
                               for k, v in attrs.items()])))
    return subnetpools['subnetpools']


def _process_subnet(neutron_network_id, endpoint_id, interface_cidr,
                    new_subnets, existing_subnets, pool_id=None):
    cidr = netaddr.IPNetwork(interface_cidr)
    subnet_network = str(cidr.network)
    subnet_cidr = '/'.join([subnet_network,
                            str(cidr.prefixlen)])
    subnets = _get_subnets_by_attrs(
        network_id=neutron_network_id, cidr=subnet_cidr)
    if subnets:
        existing_subnets += subnets
    else:
        new_subnet = {
            'name': '-'.join([endpoint_id, subnet_network]),
            'network_id': neutron_network_id,
            'ip_version': cidr.version,
            'cidr': subnet_cidr,
            'enable_dhcp': enable_dhcp,
        }
        if pool_id:
            del new_subnet['cidr']
            new_subnet['subnetpool_id'] = pool_id

        new_subnets.append(new_subnet)


def _get_or_create_subnet_by_pools(subnetpool_names, neutron_network_id,
                                   endpoint_id, new_subnets, existing_subnets):
    for subnetpool_name in subnetpool_names:
        pools = _get_subnetpools_by_attrs(name=subnetpool_name)
        if pools:
            pool = pools[0]
            prefixes = pool['prefixes']
            for prefix in prefixes:
                _process_subnet(neutron_network_id, endpoint_id, prefix,
                                new_subnets, existing_subnets,
                                pool_id=pool['id'])
    if not (new_subnets or existing_subnets):
        raise exceptions.NoResourceException(
            "No subnetpools with name {0} is found."
            .format(', '.join(subnetpool_names)))


def _handle_allocation_from_pools(neutron_network_id, endpoint_id,
                                  new_subnets, existing_subnets):
    _get_or_create_subnet_by_pools(SUBNET_POOLS_V4, neutron_network_id,
                                   endpoint_id, new_subnets, existing_subnets)
    _get_or_create_subnet_by_pools(SUBNET_POOLS_V6, neutron_network_id,
                                   endpoint_id, new_subnets, existing_subnets)

    created_subnets_response = {'subnets': []}
    if new_subnets:
        created_subnets_response = app.neutron.create_subnet(
            {'subnets': new_subnets})

    return created_subnets_response


def _handle_explicit_allocation(neutron_network_id, endpoint_id,
                                interface_cidrv4, interface_cidrv6,
                                new_subnets, existing_subnets):
    if interface_cidrv4:
        _process_subnet(neutron_network_id, endpoint_id, interface_cidrv4,
                        new_subnets, existing_subnets)

    if interface_cidrv6:
        _process_subnet(neutron_network_id, endpoint_id, interface_cidrv6,
                        new_subnets, existing_subnets)

    created_subnets_response = {'subnets': []}
    if new_subnets:
        # Bulk create operation of subnets
        created_subnets_response = app.neutron.create_subnet(
            {'subnets': new_subnets})

    return created_subnets_response


def _process_interface_address(port_dict, subnets_dict_by_id,
                               response_interface):
    assigned_address = port_dict['ip_address']
    subnet_id = port_dict['subnet_id']
    subnet = subnets_dict_by_id[subnet_id]
    cidr = netaddr.IPNetwork(subnet['cidr'])
    assigned_address += '/' + str(cidr.prefixlen)
    if cidr.version == 4:
        response_interface['Address'] = assigned_address
    else:
        response_interface['AddressIPv6'] = assigned_address


def _create_subnets_and_or_port(interface, neutron_network_id, endpoint_id):
    response_interface = {}
    existing_subnets = []
    created_subnets_response = {'subnets': []}
    # v4 and v6 Subnets for bulk creation.
    new_subnets = []

    interface_cidrv4 = interface.get('Address', '')
    interface_cidrv6 = interface.get('AddressIPv6', '')
    interface_mac = interface.get('MacAddress', '')

    if interface_cidrv4 or interface_cidrv6:
        created_subnets_response = _handle_explicit_allocation(
            neutron_network_id, endpoint_id, interface_cidrv4,
            interface_cidrv6, new_subnets, existing_subnets)
    else:
        app.logger.info("Retrieving or creating subnets with the default "
                        "subnetpool because Address and AddressIPv6 are "
                        "not given.")
        created_subnets_response = _handle_allocation_from_pools(
            neutron_network_id, endpoint_id, new_subnets, existing_subnets)

    try:
        port = {
            'name': '-'.join([endpoint_id, 'port']),
            'admin_state_up': True,
            'network_id': neutron_network_id,
            'device_owner': constants.DEVICE_OWNER,
            'device_id': endpoint_id,
        }
        if interface_mac:
            port['mac_address'] = interface_mac
        created_subnets = created_subnets_response.get('subnets', [])
        all_subnets = created_subnets + existing_subnets
        fixed_ips = port['fixed_ips'] = []
        for subnet in all_subnets:
            fixed_ip = {'subnet_id': subnet['id']}
            if interface_cidrv4 or interface_cidrv6:
                if subnet['ip_version'] == 4 and interface_cidrv4:
                    cidr = netaddr.IPNetwork(interface_cidrv4)
                elif subnet['ip_version'] == 6 and interface_cidrv6:
                    cidr = netaddr.IPNetwork(interface_cidrv6)
                subnet_cidr = '/'.join([str(cidr.network),
                                        str(cidr.prefixlen)])
                if subnet['cidr'] != subnet_cidr:
                    continue
                fixed_ip['ip_address'] = str(cidr.ip)
            fixed_ips.append(fixed_ip)
        created_port = app.neutron.create_port({'port': port})
        created_port = created_port['port']

        created_fixed_ips = created_port['fixed_ips']
        subnets_dict_by_id = {subnet['id']: subnet
                              for subnet in all_subnets}
        if not interface_mac:
            response_interface['MacAddress'] = created_port['mac_address']

        if not (interface_cidrv4 or interface_cidrv6):
            if 'ip_address' in created_port:
                _process_interface_address(
                    created_port, subnets_dict_by_id, response_interface)
            for fixed_ip in created_fixed_ips:
                _process_interface_address(
                    fixed_ip, subnets_dict_by_id, response_interface)
    except n_exceptions.NeutronClientException as ex:
        app.logger.error("Error happend during creating a "
                         "Neutron port: {0}".format(ex))
        # Rollback the subnets creation
        for subnet in created_subnets:
            app.neutron.delete_subnet(subnet['id'])
        raise
    return response_interface


@app.route('/Plugin.Activate', methods=['POST'])
def plugin_activate():
    """Returns the list of the implemented drivers.

    This function returns the list of the implemented drivers defaults to
    ``[NetworkDriver]`` in the handshake of the remote driver, which happens
    right before the first request against Kuryr.

    See the following link for more details about the spec:

      https://github.com/docker/libnetwork/blob/master/docs/remote.md#handshake  # noqa
    """
    return flask.jsonify(constants.SCHEMA['PLUGIN_ACTIVATE'])


@app.route('/NetworkDriver.GetCapabilities', methods=['POST'])
def plugin_scope():
    capabilities = {'Scope': cfg.CONF.capability_scope}
    return flask.jsonify(capabilities)


@app.route('/NetworkDriver.DiscoverNew', methods=['POST'])
def network_driver_discover_new():
    """The callback function for the DiscoverNew notification.

    The DiscoverNew notification includes the type of the
    resource that has been newly discovered and possibly other
    information associated with the resource.

    See the following link for more details about the spec:

{'fraction_non_alphanumeric': 0.0900988557639916, 'fraction_numerical': 0.004903868607456994, 'mean_word_length': 2.756029820201725, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7108961', 'n_tokens_mistral': 8007, 'n_tokens_neox': 7722, 'n_words': 1640}