<commit_before>/*
 * Copyright (C) 2017 Square, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.squareup.inject.assisted.processor

import com.google.auto.common.MoreTypes
import com.google.auto.service.AutoService
import com.squareup.inject.assisted.Assisted
import com.squareup.inject.assisted.AssistedInject
import com.squareup.inject.assisted.processor.internal.BindingKey
import com.squareup.inject.assisted.processor.internal.BindingKey.Use.ASSISTED
import com.squareup.inject.assisted.processor.internal.BindingKey.Use.PROVIDED
import com.squareup.inject.assisted.processor.internal.Key
import com.squareup.inject.assisted.processor.internal.applyEach
import com.squareup.inject.assisted.processor.internal.applyEachIndexed
import com.squareup.inject.assisted.processor.internal.cast
import com.squareup.inject.assisted.processor.internal.duplicates
import com.squareup.inject.assisted.processor.internal.findElementsAnnotatedWith
import com.squareup.inject.assisted.processor.internal.hasAnnotation
import com.squareup.javapoet.ClassName
import com.squareup.javapoet.JavaFile
import com.squareup.javapoet.MethodSpec
import com.squareup.javapoet.ParameterizedTypeName
import com.squareup.javapoet.TypeName
import com.squareup.javapoet.TypeSpec
import com.squareup.javapoet.TypeVariableName
import javax.annotation.processing.AbstractProcessor
import javax.annotation.processing.Filer
import javax.annotation.processing.Messager
import javax.annotation.processing.ProcessingEnvironment
import javax.annotation.processing.Processor
import javax.annotation.processing.RoundEnvironment
import javax.lang.model.SourceVersion
import javax.lang.model.element.Element
import javax.lang.model.element.ElementKind.CLASS
import javax.lang.model.element.ElementKind.CONSTRUCTOR
import javax.lang.model.element.ElementKind.INTERFACE
import javax.lang.model.element.ExecutableElement
import javax.lang.model.element.Modifier.FINAL
import javax.lang.model.element.Modifier.PRIVATE
import javax.lang.model.element.Modifier.PUBLIC
import javax.lang.model.element.Modifier.STATIC
import javax.lang.model.element.TypeElement
import javax.lang.model.util.Types
import javax.tools.Diagnostic.Kind.ERROR

@AutoService(Processor::class)
class AssistedInjectProcessor : AbstractProcessor() {
  override fun getSupportedSourceVersion() = SourceVersion.latest()
  override fun getSupportedAnnotationTypes() = setOf(
      AssistedInject::class.java.canonicalName,
      AssistedInject.Factory::class.java.canonicalName)

  override fun init(env: ProcessingEnvironment) {
    super.init(env)
    this.types = env.typeUtils
    this.messager = env.messager
    this.filer = env.filer
  }

  private lateinit var types: Types
  private lateinit var messager: Messager
  private lateinit var filer: Filer

  override fun process(annotations: Set<TypeElement>, roundEnv: RoundEnvironment): Boolean {
    val factoryTypes = roundEnv.findElementsAnnotatedWith<AssistedInject.Factory>()
    factoryTypes
        .filterNot { it.enclosingElement.kind == CLASS }
        .forEach {
          messager.printMessage(ERROR, "@AssistedInject.Factory must be declared as a nested type.", it)
        }

    // Grab types with only an @AssistedInject.Factory so we can detect missing constructor annotations.
    val typeWrappersWithFactories = factoryTypes
        .map { it.enclosingElement }
        .filter { it.kind == CLASS }
        .map { MoreTypes.equivalence().wrap(it.asType()) }

    // Grab types with only @AssistedInject so we can detect missing @AssistedInject.Factory types.
    val typeWrappersWithAssisted = roundEnv.findElementsAnnotatedWith<AssistedInject>()
        .map { it.enclosingElement as TypeElement }
        .map { MoreTypes.equivalence().wrap(it.asType()) }

    (typeWrappersWithFactories + typeWrappersWithAssisted)
        .toSet()
        .map { types.asElement(it.get()) as TypeElement }
        .forEach {
          try {
            process(it)
          } catch (e: StopProcessingException) {
            messager.printMessage(ERROR, e.message, e.originatingElement)
          } catch (e: Exception) {
            messager.printMessage(ERROR, "Uncaught error: ${e.message}")
          }
        }

    return false
  }

  private fun process(type: TypeElement) {
    val constructor = findAssistedConstructor(type)

    val bindingKeys = parseBindingKeys(constructor)
    val providedKeys = providedKeys(constructor, bindingKeys)
    val assistedKeys = assistedKeys(constructor, bindingKeys)

    val factoryType = findFactoryType(type)
    val factoryMethod = findFactoryMethod(factoryType, type)
    validateFactoryKeys(factoryMethod, assistedKeys.map(BindingKey::key).toSet())

    val typeName = TypeName.get(type.asType())
    val factoryName = ClassName.get(factoryType)
    val generatedName = ClassName.get(type).peerClass(type.simpleName.toString() + SUFFIX)
    val generatedSpec = TypeSpec.classBuilder(generatedName)
        .addModifiers(PUBLIC, FINAL)
        .addSuperinterface(factoryName)
        .addOriginatingElement(type)
        .addOriginatingElement(factoryType)
        .applyEach(providedKeys) {
          addField(it.providerType.withoutAnnotations(), it.name, PRIVATE, FINAL)
        }
        .addMethod(MethodSpec.constructorBuilder()
            .addModifiers(PUBLIC)
            .addAnnotation(INJECT)
            .applyEach(providedKeys) {
              addParameter(it.providerType, it.name)
              addStatement("this.$1N = $1N", it.name)
            }
            .build())
        .addMethod(MethodSpec.methodBuilder(factoryMethod.simpleName.toString())
            .addAnnotation(Override::class.java)
            .addModifiers(PUBLIC)
            .returns(typeName)
            .apply {
              if (typeName is ParameterizedTypeName) {
                addTypeVariables(typeName.typeArguments.filterIsInstance<TypeVariableName>())
              }
            }
            .applyEach(assistedKeys) { key ->
              addParameter(key.type, key.name)
            }
            .addCode("$[return new \$T(\n", typeName)
            .applyEachIndexed(bindingKeys) { index, key ->
              if (index > 0) addCode(",\n")
              addCode(key.bindingResolveCode())
            }
            .addCode(");$]\n")
            .build())
        .build()

    JavaFile.builder(generatedName.packageName(), generatedSpec)
        .addFileComment("Generated by @AssistedInject. Do not modify!")
        .build()
        .writeTo(filer)
  }

  private fun findAssistedConstructor(type: TypeElement): ExecutableElement {
    val constructors = type.enclosedElements
        .filter { it.kind == CONSTRUCTOR }
        .cast<ExecutableElement>()
    if (constructors.size == 1) {
      // Only one constructor (common case). Use it! This also allows better error messages to be
      // produced in the case where you have no @Assisted parameters on this constructor or are
      // missing @AssistedInject.
      return constructors.single()
    }

    val assistedConstructors = constructors.filter { it.hasAnnotation<AssistedInject>() }
    if (assistedConstructors.isEmpty()) {
      throw StopProcessingException(
          "Assisted injection requires an @AssistedInject-annotated constructor " +
              "with at least one @Assisted parameter.", type)
    }
    if (assistedConstructors.size > 1) {
      throw StopProcessingException("Multiple @AssistedInject-annotated constructors found.", type)
    }
    return assistedConstructors.single()
  }

  private fun parseBindingKeys(method: ExecutableElement) = method.parameters
      .map { BindingKey(it, if (it.hasAnnotation<Assisted>()) ASSISTED else PROVIDED) }

  private fun providedKeys(method: ExecutableElement, keys: List<BindingKey>): List<BindingKey> {
    val providedKeys = keys.filter { it.use == PROVIDED }
    if (providedKeys.isEmpty()) {
      throw StopProcessingException(
          "Assisted injection requires at least one non-@Assisted parameter.", method)
    }
    val duplicateKeys = providedKeys.duplicates()
    if (duplicateKeys.isNotEmpty()) {
      throw StopProcessingException(
          "Duplicate non-@Assisted parameters declared. Forget a qualifier annotation?"
              + duplicateKeys.toSet().joinToString("\n * ", prefix = "\n * "),
          method)
    }
    return providedKeys
  }

  private fun assistedKeys(method: ExecutableElement, keys: List<BindingKey>): List<BindingKey> {
    val assistedKeys = keys.filter { it.use == ASSISTED }
    if (assistedKeys.isEmpty()) {
      throw StopProcessingException(
          "Assisted injection requires at least one @Assisted parameter.", method)
    }
    val duplicateKeys = assistedKeys.duplicates()
    if (duplicateKeys.isNotEmpty()) {
      throw StopProcessingException(
          "Duplicate @Assisted parameters declared. Forget a qualifier annotation?"
              + duplicateKeys.toSet().joinToString("\n * ", prefix = "\n * "),
          method)
    }
    return assistedKeys
  }

  private fun findFactoryType(type: TypeElement): TypeElement {
    val types = type.enclosedElements
        .filterIsInstance<TypeElement>()
        .filter { it.hasAnnotation<AssistedInject.Factory>() }
    if (types.isEmpty()) {
      throw StopProcessingException("No nested @AssistedInject.Factory found.", type)
    }
    if (types.size > 1) {
      throw StopProcessingException("Multiple @AssistedInject.Factory types found.", type)
    }
    val factory = types.single()
    if (factory.kind != INTERFACE) {
      throw StopProcessingException("@AssistedInject.Factory must be an interface.", factory)
    }
    return factory
  }

  private fun findFactoryMethod(factory: TypeElement, type: TypeElement): ExecutableElement {
    val methods = factory.enclosedElements
        .filterIsInstance<ExecutableElement>() // Ignore non-method elements like constants.
        .filterNot { it.isDefault } // Ignore default methods for convenience overloads.
        .filterNot { STATIC in it.modifiers } // Ignore static helper methods.
        .filterNot { PRIVATE in it.modifiers } // Ignore private helper methods for default methods.
    if (methods.isEmpty()) {
      throw StopProcessingException("Factory interface does not define a factory method.", factory)
    }
    if (methods.size > 1) {
      throw StopProcessingException("Factory interface defines multiple factory methods.", factory)
    }
    val method = methods.single()
    //if (!types.isAssignable(type.asType(), method.returnType)) {
    //  throw StopProcessingException("Factory method returns incorrect type. "
    //      + "Must be ${type.simpleName} or one of its supertypes.", method)
    //}
    return method
  }

  private fun validateFactoryKeys(method: ExecutableElement, expectedKeys: Set<Key>) {
    val keys = method.parameters.map(::Key).toSet()
    if (keys != expectedKeys) {
      var message = "Factory method parameters do not match constructor @Assisted parameters."

      val missingKeys = expectedKeys - keys
      if (missingKeys.isNotEmpty()) {
        message += missingKeys.joinToString("\n * ", prefix = "\n\nMissing:\n * ")
      }

      val unknownKeys = keys - expectedKeys
      if (unknownKeys.isNotEmpty()) {
        message += unknownKeys.joinToString("\n * ", prefix = "\n\nUnknown:\n * ")
      }

      throw StopProcessingException(message, method)
    }
  }

  companion object {
    const val SUFFIX = "_AssistedFactory"
    private val INJECT = ClassName.get("javax.inject", "Inject")
  }
}

internal class StopProcessingException(
    message: String,
    val originatingElement: Element? = null
) : Exception(message)
<commit_msg>Use an intermediate model before code gen<commit_after>/*
 * Copyright (C) 2017 Square, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.squareup.inject.assisted.processor

import com.google.auto.common.MoreTypes
import com.google.auto.service.AutoService
import com.squareup.inject.assisted.Assisted
import com.squareup.inject.assisted.AssistedInject
import com.squareup.inject.assisted.processor.internal.BindingKey
import com.squareup.inject.assisted.processor.internal.BindingKey.Use.ASSISTED
import com.squareup.inject.assisted.processor.internal.BindingKey.Use.PROVIDED
import com.squareup.inject.assisted.processor.internal.Key
import com.squareup.inject.assisted.processor.internal.applyEach
import com.squareup.inject.assisted.processor.internal.applyEachIndexed
import com.squareup.inject.assisted.processor.internal.cast
import com.squareup.inject.assisted.processor.internal.duplicates
import com.squareup.inject.assisted.processor.internal.findElementsAnnotatedWith
import com.squareup.inject.assisted.processor.internal.hasAnnotation
import com.squareup.javapoet.ClassName
import com.squareup.javapoet.JavaFile
import com.squareup.javapoet.MethodSpec
import com.squareup.javapoet.ParameterizedTypeName
import com.squareup.javapoet.TypeName
import com.squareup.javapoet.TypeSpec
import com.squareup.javapoet.TypeVariableName
import javax.annotation.processing.AbstractProcessor
import javax.annotation.processing.Filer
import javax.annotation.processing.Messager
import javax.annotation.processing.ProcessingEnvironment
import javax.annotation.processing.Processor
import javax.annotation.processing.RoundEnvironment
import javax.lang.model.SourceVersion
import javax.lang.model.element.Element
import javax.lang.model.element.ElementKind.CLASS
import javax.lang.model.element.ElementKind.CONSTRUCTOR
import javax.lang.model.element.ElementKind.INTERFACE
import javax.lang.model.element.ExecutableElement
import javax.lang.model.element.Modifier.FINAL
import javax.lang.model.element.Modifier.PRIVATE
import javax.lang.model.element.Modifier.PUBLIC
import javax.lang.model.element.Modifier.STATIC
import javax.lang.model.element.TypeElement
import javax.lang.model.util.Types
import javax.tools.Diagnostic.Kind.ERROR

@AutoService(Processor::class)
class AssistedInjectProcessor : AbstractProcessor() {
  override fun getSupportedSourceVersion() = SourceVersion.latest()
  override fun getSupportedAnnotationTypes() = setOf(
      AssistedInject::class.java.canonicalName,
      AssistedInject.Factory::class.java.canonicalName)

  override fun init(env: ProcessingEnvironment) {
    super.init(env)
    this.types = env.typeUtils
    this.messager = env.messager
    this.filer = env.filer
  }

  private lateinit var types: Types
  private lateinit var messager: Messager
  private lateinit var filer: Filer

  override fun process(annotations: Set<TypeElement>, roundEnv: RoundEnvironment): Boolean {
    val factoryTypes = roundEnv.findElementsAnnotatedWith<AssistedInject.Factory>()
    factoryTypes
        .filterNot { it.enclosingElement.kind == CLASS }
        .forEach {
          messager.printMessage(ERROR, "@AssistedInject.Factory must be declared as a nested type.", it)
        }

    // Grab types with only an @AssistedInject.Factory so we can detect missing constructor annotations.
    val typeWrappersWithFactories = factoryTypes
        .map { it.enclosingElement }
        .filter { it.kind == CLASS }
        .map { MoreTypes.equivalence().wrap(it.asType()) }

    // Grab types with only @AssistedInject so we can detect missing @AssistedInject.Factory types.
    val typeWrappersWithAssisted = roundEnv.findElementsAnnotatedWith<AssistedInject>()
        .map { it.enclosingElement as TypeElement }
        .map { MoreTypes.equivalence().wrap(it.asType()) }

    (typeWrappersWithFactories + typeWrappersWithAssisted)
        .toSet()
        .map { types.asElement(it.get()) as TypeElement }
        .forEach {
          try {
            parseRequest(it).brewJava().writeTo(filer)
          } catch (e: StopProcessingException) {
            messager.printMessage(ERROR, e.message, e.originatingElement)
          } catch (e: Exception) {
            messager.printMessage(ERROR, "Uncaught error: ${e.message}")
          }
        }

    return false
  }

  private fun parseRequest(type: TypeElement): AssistedInjectRequest {
    val constructor = findAssistedConstructor(type)

    val bindingKeys = parseBindingKeys(constructor)
    val providedKeys = providedKeys(constructor, bindingKeys)
    val assistedKeys = assistedKeys(constructor, bindingKeys)

    val factoryType = findFactoryType(type)
    val factoryMethod = findFactoryMethod(factoryType, type)
    validateFactoryKeys(factoryMethod, assistedKeys.map(BindingKey::key).toSet())

    return AssistedInjectRequest(type, factoryType, factoryMethod, bindingKeys, providedKeys,
        assistedKeys)
  }

  private data class AssistedInjectRequest(
      val type: TypeElement,
      val factoryType: TypeElement,
      val factoryMethod: ExecutableElement,
      val bindingKeys: List<BindingKey>,
      val providedKeys: List<BindingKey>,
      val assistedKeys: List<BindingKey>
  )

  private fun AssistedInjectRequest.brewJava(): JavaFile {
    val typeName = TypeName.get(type.asType())
    val factoryName = ClassName.get(factoryType)
    val generatedName = ClassName.get(type).peerClass(type.simpleName.toString() + SUFFIX)
    val generatedSpec = TypeSpec.classBuilder(generatedName)
        .addModifiers(PUBLIC, FINAL)
        .addSuperinterface(factoryName)
        .addOriginatingElement(type)
        .addOriginatingElement(factoryType)
        .applyEach(providedKeys) {
          addField(it.providerType.withoutAnnotations(), it.name, PRIVATE, FINAL)
        }
        .addMethod(MethodSpec.constructorBuilder()
            .addModifiers(PUBLIC)
            .addAnnotation(INJECT)
            .applyEach(providedKeys) {
              addParameter(it.providerType, it.name)
              addStatement("this.$1N = $1N", it.name)
            }
            .build())
        .addMethod(MethodSpec.methodBuilder(factoryMethod.simpleName.toString())
            .addAnnotation(Override::class.java)
            .addModifiers(PUBLIC)
            .returns(typeName)
            .apply {
              if (typeName is ParameterizedTypeName) {
                addTypeVariables(typeName.typeArguments.filterIsInstance<TypeVariableName>())
              }
            }
            .applyEach(assistedKeys) { key ->
              addParameter(key.type, key.name)
            }
            .addCode("$[return new \$T(\n", typeName)
            .applyEachIndexed(bindingKeys) { index, key ->
              if (index > 0) addCode(",\n")
              addCode(key.bindingResolveCode())
            }
            .addCode(");$]\n")
            .build())
        .build()

    return JavaFile.builder(generatedName.packageName(), generatedSpec)
        .addFileComment("Generated by @AssistedInject. Do not modify!")
        .build()
  }

  private fun findAssistedConstructor(type: TypeElement): ExecutableElement {
    val constructors = type.enclosedElements
        .filter { it.kind == CONSTRUCTOR }
        .cast<ExecutableElement>()
    if (constructors.size == 1) {
      // Only one constructor (common case). Use it! This also allows better error messages to be
      // produced in the case where you have no @Assisted parameters on this constructor or are
      // missing @AssistedInject.
      return constructors.single()
    }

    val assistedConstructors = constructors.filter { it.hasAnnotation<AssistedInject>() }
    if (assistedConstructors.isEmpty()) {
      throw StopProcessingException(
          "Assisted injection requires an @AssistedInject-annotated constructor " +
              "with at least one @Assisted parameter.", type)
    }
    if (assistedConstructors.size > 1) {
      throw StopProcessingException("Multiple @AssistedInject-annotated constructors found.", type)
    }
    return assistedConstructors.single()
  }

  private fun parseBindingKeys(method: ExecutableElement) = method.parameters
      .map { BindingKey(it, if (it.hasAnnotation<Assisted>()) ASSISTED else PROVIDED) }

  private fun providedKeys(method: ExecutableElement, keys: List<BindingKey>): List<BindingKey> {
    val providedKeys = keys.filter { it.use == PROVIDED }
    if (providedKeys.isEmpty()) {
      throw StopProcessingException(
          "Assisted injection requires at least one non-@Assisted parameter.", method)
    }
    val duplicateKeys = providedKeys.duplicates()
    if (duplicateKeys.isNotEmpty()) {
      throw StopProcessingException(
          "Duplicate non-@Assisted parameters declared. Forget a qualifier annotation?"
              + duplicateKeys.toSet().joinToString("\n * ", prefix = "\n * "),
          method)
    }
    return providedKeys
  }

  private fun assistedKeys(method: ExecutableElement, keys: List<BindingKey>): List<BindingKey> {
    val assistedKeys = keys.filter { it.use == ASSISTED }
    if (assistedKeys.isEmpty()) {
      throw StopProcessingException(
          "Assisted injection requires at least one @Assisted parameter.", method)
    }
    val duplicateKeys = assistedKeys.duplicates()
    if (duplicateKeys.isNotEmpty()) {
      throw StopProcessingException(
          "Duplicate @Assisted parameters declared. Forget a qualifier annotation?"
              + duplicateKeys.toSet().joinToString("\n * ", prefix = "\n * "),
          method)
    }
    return assistedKeys
  }

  private fun findFactoryType(type: TypeElement): TypeElement {
    val types = type.enclosedElements
        .filterIsInstance<TypeElement>()
        .filter { it.hasAnnotation<AssistedInject.Factory>() }
    if (types.isEmpty()) {
      throw StopProcessingException("No nested @AssistedInject.Factory found.", type)
    }
    if (types.size > 1) {
      throw StopProcessingException("Multiple @AssistedInject.Factory types found.", type)
    }
    val factory = types.single()
    if (factory.kind != INTERFACE) {
      throw StopProcessingException("@AssistedInject.Factory must be an interface.", factory)
    }
    return factory
  }

  private fun findFactoryMethod(factory: TypeElement, type: TypeElement): ExecutableElement {
    val methods = factory.enclosedElements
        .filterIsInstance<ExecutableElement>() // Ignore non-method elements like constants.
        .filterNot { it.isDefault } // Ignore default methods for convenience overloads.
        .filterNot { STATIC in it.modifiers } // Ignore static helper methods.
        .filterNot { PRIVATE in it.modifiers } // Ignore private helper methods for default methods.
    if (methods.isEmpty()) {
      throw StopProcessingException("Factory interface does not define a factory method.", factory)
    }
    if (methods.size > 1) {
      throw StopProcessingException("Factory interface defines multiple factory methods.", factory)
    }
    val method = methods.single()
    //if (!types.isAssignable(type.asType(), method.returnType)) {
    //  throw StopProcessingException("Factory method returns incorrect type. "
    //      + "Must be ${type.simpleName} or one of its supertypes.", method)
    //}
    return method
  }

  private fun validateFactoryKeys(method: ExecutableElement, expectedKeys: Set<Key>) {
    val keys = method.parameters.map(::Key).toSet()
    if (keys != expectedKeys) {
      var message = "Factory method parameters do not match constructor @Assisted parameters."

      val missingKeys = expectedKeys - keys
      if (missingKeys.isNotEmpty()) {
        message += missingKeys.joinToString("\n * ", prefix = "\n\nMissing:\n * ")
      }

      val unknownKeys = keys - expectedKeys
      if (unknownKeys.isNotEmpty()) {
        message += unknownKeys.joinToString("\n * ", prefix = "\n\nUnknown:\n * ")
      }

      throw StopProcessingException(message, method)
    }
  }

  companion object {
    const val SUFFIX = "_AssistedFactory"
    private val INJECT = ClassName.get("javax.inject", "Inject")
  }
}

internal class StopProcessingException(
    message: String,
    val originatingElement: Element? = null
) : Exception(message)

{'fraction_non_alphanumeric': 0.09785662679617886, 'fraction_numerical': 0.0012041422493377218, 'mean_word_length': 3.386443661971831, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6894859', 'n_tokens_mistral': 6730, 'n_tokens_neox': 6605, 'n_words': 1606}