<commit_before>/**
 * Copyright (C) 2010-2012 LShift Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package net.lshift.diffa.kernel.config

import scala.collection.JavaConversions._
import net.lshift.diffa.schema.jooq.{DatabaseFacade => JooqDatabaseFacade}
import net.lshift.diffa.schema.tables.Members.MEMBERS
import net.lshift.diffa.schema.tables.Policies.POLICIES
import net.lshift.diffa.schema.tables.ConfigOptions.CONFIG_OPTIONS
import net.lshift.diffa.schema.tables.RepairActions.REPAIR_ACTIONS
import net.lshift.diffa.schema.tables.Escalations.ESCALATIONS
import net.lshift.diffa.schema.tables.EscalationRules.ESCALATION_RULES
import net.lshift.diffa.schema.tables.PairReports.PAIR_REPORTS
import net.lshift.diffa.schema.tables.PairViews.PAIR_VIEWS
import net.lshift.diffa.schema.tables.Endpoints.ENDPOINTS
import net.lshift.diffa.schema.tables.Pairs.PAIRS
import net.lshift.diffa.schema.tables.Spaces.SPACES
import net.lshift.diffa.schema.tables.EndpointViews.ENDPOINT_VIEWS
import net.lshift.diffa.schema.tables.Breakers.BREAKERS
import net.lshift.diffa.schema.tables.Extents.EXTENTS
import JooqConfigStoreCompanion._
import net.lshift.diffa.kernel.naming.CacheName._
import net.lshift.diffa.kernel.util.MissingObjectException
import net.lshift.diffa.kernel.lifecycle.{PairLifecycleAware, DomainLifecycleAware}
import net.lshift.diffa.kernel.util.cache.{KeyPredicate, CacheProvider}
import reflect.BeanProperty
import java.util
import collection.mutable.ListBuffer
import org.jooq.impl.Factory
import net.lshift.diffa.kernel.frontend.DomainEndpointDef
import net.lshift.diffa.kernel.frontend.DomainPairDef
import net.lshift.diffa.kernel.frontend.PairDef
import net.lshift.diffa.kernel.frontend.EndpointDef
import org.jooq.{Record, Field, Condition, Table}
import java.lang.{Long => LONG}
import system.{ConflictingConcurrentModificationException, PolicyKey}
import java.lang.{Integer => INT}
import net.lshift.diffa.kernel.util.sequence.SequenceProvider
import net.lshift.diffa.kernel.naming.SequenceName
import java.sql.SQLIntegrityConstraintViolationException
import org.jooq.exception.DataAccessException
import net.lshift.diffa.snowflake.IdProvider

class JooqDomainConfigStore(jooq:JooqDatabaseFacade,
                            cacheProvider:CacheProvider,
                            sequenceProvider:SequenceProvider,
                            idProvider:IdProvider,
                            membershipListener:DomainMembershipAware)
    extends DomainConfigStore
    with DomainLifecycleAware {

  private val pairEventSubscribers = new ListBuffer[PairLifecycleAware]
  def registerPairEventListener(p:PairLifecycleAware) = pairEventSubscribers += p

  private val cachedConfigVersions = cacheProvider.getCachedMap[Long,Int]("domain.config.versions")
  private val cachedPairs = cacheProvider.getCachedMap[Long, java.util.List[DomainPairDef]]("domain.pairs")
  private val cachedPairsByKey = cacheProvider.getCachedMap[DomainPairKey, DomainPairDef]("domain.pairs.by.key")
  private val cachedEndpoints = cacheProvider.getCachedMap[Long, java.util.List[DomainEndpointDef]]("domain.endpoints")
  private val cachedEndpointsByKey = cacheProvider.getCachedMap[DomainEndpointKey, DomainEndpointDef]("domain.endpoints.by.key")
  private val cachedPairsByEndpoint = cacheProvider.getCachedMap[DomainEndpointKey, java.util.List[DomainPairDef]]("domain.pairs.by.endpoint")
  private val cachedBreakers = cacheProvider.getCachedMap[BreakerKey, Boolean](DOMAIN_PAIR_BREAKERS)

  // Config options
  private val cachedDomainConfigOptionsMap = cacheProvider.getCachedMap[Long, java.util.Map[String,String]](DOMAIN_CONFIG_OPTIONS_MAP)
  private val cachedDomainConfigOptions = cacheProvider.getCachedMap[DomainConfigKey, String](DOMAIN_CONFIG_OPTIONS)

  // Members
  private val cachedMembers = cacheProvider.getCachedMap[Long, java.util.List[Member]](USER_DOMAIN_MEMBERS)
  private val cachedPolicies = cacheProvider.getCachedMap[SpacePolicyKey, PolicyKey](SPACE_POLICIES)

  def reset {
    cachedConfigVersions.evictAll()
    cachedPairs.evictAll()
    cachedPairsByKey.evictAll()
    cachedEndpoints.evictAll()
    cachedEndpointsByKey.evictAll()
    cachedPairsByEndpoint.evictAll()
    cachedBreakers.evictAll()

    cachedDomainConfigOptionsMap.evictAll()
    cachedDomainConfigOptions.evictAll()

    cachedMembers.evictAll()
    cachedPolicies.evictAll()
  }

  private def invalidateMembershipCache(space:Long) = {
    cachedMembers.evict(space)
  }

  private def invalidateConfigCaches(space:Long) = {
    cachedDomainConfigOptionsMap.evict(space)
    cachedDomainConfigOptions.keySubset(ConfigOptionByDomainPredicate(space)).evictAll()
  }

  private def invalidateAllCaches(space:Long) = {
    cachedConfigVersions.evict(space)
    cachedEndpoints.evict(space)
    cachedPairs.evict(space)
    cachedPairsByEndpoint.keySubset(EndpointByDomainPredicate(space)).evictAll()
    cachedPairsByKey.keySubset(PairByDomainPredicate(space)).evictAll()
    cachedEndpointsByKey.keySubset(EndpointByDomainPredicate(space)).evictAll()
    cachedBreakers.keySubset(BreakerByDomainPredicate(space)).evictAll()

    invalidateConfigCaches(space)

    invalidateMembershipCache(space)
  }

  private def invalidateEndpointCachesOnly(space:Long, endpointName: String) = {
    cachedEndpoints.evict(space)
    cachedPairsByEndpoint.keySubset(PairByDomainAndEndpointPredicate(space, endpointName)).evictAll()
    cachedEndpointsByKey.evict(DomainEndpointKey(space,endpointName))

    // TODO This is a very coarse grained invalidation of the pair caches - this could be made finer at some stage
    cachedPairs.evict(space)
    cachedPairsByKey.keySubset(PairByDomainPredicate(space)).evictAll()
  }

  private def invalidatePairCachesOnly(space:Long) = {
    cachedPairs.evict(space)
    cachedPairsByKey.keySubset(PairByDomainPredicate(space)).evictAll()
    cachedPairsByEndpoint.keySubset(EndpointByDomainPredicate(space)).evictAll()
  }

  def onDomainUpdated(space: Long) = invalidateAllCaches(space)
  def onDomainRemoved(space: Long) = invalidateAllCaches(space)

  def createOrUpdateEndpoint(space: Long, endpointDef: EndpointDef) : DomainEndpointDef = {
    jooq.execute(t => {

      val insert = t.insertInto(ENDPOINTS).
        set(ENDPOINTS.SPACE, space:LONG).
        set(ENDPOINTS.NAME, endpointDef.name).
        set(ENDPOINTS.ID, idProvider.getId:LONG).
        set(ENDPOINTS.COLLATION_TYPE, endpointDef.collation).
        set(ENDPOINTS.CONTENT_RETRIEVAL_URL, endpointDef.contentRetrievalUrl).
        set(ENDPOINTS.SCAN_URL, endpointDef.scanUrl).
        set(ENDPOINTS.VERSION_GENERATION_URL, endpointDef.versionGenerationUrl).
        set(ENDPOINTS.INBOUND_URL, endpointDef.inboundUrl)
      val update = t.update(ENDPOINTS).
        set(ENDPOINTS.COLLATION_TYPE, endpointDef.collation).
        set(ENDPOINTS.CONTENT_RETRIEVAL_URL, endpointDef.contentRetrievalUrl).
        set(ENDPOINTS.SCAN_URL, endpointDef.scanUrl).
        set(ENDPOINTS.VERSION_GENERATION_URL, endpointDef.versionGenerationUrl).
        set(ENDPOINTS.INBOUND_URL, endpointDef.inboundUrl).
        where(ENDPOINTS.SPACE.equal(space:LONG).and(ENDPOINTS.NAME.equal(endpointDef.name)))

      /* mergeInto should be used when there is merge support in MySQL.
      val merge = t.mergeInto(ENDPOINTS).
        usingDual().
        on(ENDPOINTS.SPACE.equal(space), ENDPOINTS.NAME.equal(endpointDef.name)).
        whenMatchedThenUpdate().
          set(ENDPOINTS.COLLATION_TYPE, endpointDef.collation).
          set(ENDPOINTS.CONTENT_RETRIEVAL_URL, endpointDef.contentRetrievalUrl).
          set(ENDPOINTS.SCAN_URL, endpointDef.scanUrl).
          set(ENDPOINTS.VERSION_GENERATION_URL, endpointDef.versionGenerationUrl).
          set(ENDPOINTS.INBOUND_URL, endpointDef.inboundUrl).
        whenNotMatchedThenInsert().
          set(ENDPOINTS.SPACE, space:LONG).
          set(ENDPOINTS.NAME, endpointDef.name).
          set(ENDPOINTS.ID, endpointDef.id:LONG).
          set(ENDPOINTS.COLLATION_TYPE, endpointDef.collation).
          set(ENDPOINTS.CONTENT_RETRIEVAL_URL, endpointDef.contentRetrievalUrl).
          set(ENDPOINTS.SCAN_URL, endpointDef.scanUrl).
          set(ENDPOINTS.VERSION_GENERATION_URL, endpointDef.versionGenerationUrl).
          set(ENDPOINTS.INBOUND_URL, endpointDef.inboundUrl)
       */

      // This is not a fail-safe way to create or update.  It is subject to a race condition:
      // if the same endpoint is created between when the update fails and the insert is attempted,
      // then the unique key constraint violation will trigger a DataAccessException.  In this case,
      // we just let the user know it failed.
      val rowsUpdated = update.execute()
      if (rowsUpdated > 1) {
        throw new RuntimeException("Duplicate endpoint definition found for endpoint %s".format(endpointDef.name))
      } else if (rowsUpdated < 1) {
        try {
          insert.execute()
        } catch {
          case ex: DataAccessException =>
            throw new ConflictingConcurrentModificationException(endpointDef.name)
        }
      }

      val endpointId = endpointIdByName(t, endpointDef.name, space)

      // Don't attempt to update to update any rows per se, just delete every associated
      // category and re-insert the new definitions, irrespective of
      // whether they are identical to the previous definitions

      deleteCategories(t, space, endpointDef.name)

      // Insert categories for the endpoint proper
      insertCategories(t, space, endpointId, endpointDef)

      // Update the view definitions

      t.delete(ENDPOINT_VIEWS).
          where(ENDPOINT_VIEWS.ENDPOINT.equal(endpointId)).<commit_msg>280: use mergeInto for dialects other than MySQL.<commit_after>/**
 * Copyright (C) 2010-2012 LShift Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package net.lshift.diffa.kernel.config

import scala.collection.JavaConversions._
import net.lshift.diffa.schema.jooq.{DatabaseFacade => JooqDatabaseFacade}
import net.lshift.diffa.schema.tables.Members.MEMBERS
import net.lshift.diffa.schema.tables.Policies.POLICIES
import net.lshift.diffa.schema.tables.ConfigOptions.CONFIG_OPTIONS
import net.lshift.diffa.schema.tables.RepairActions.REPAIR_ACTIONS
import net.lshift.diffa.schema.tables.Escalations.ESCALATIONS
import net.lshift.diffa.schema.tables.EscalationRules.ESCALATION_RULES
import net.lshift.diffa.schema.tables.PairReports.PAIR_REPORTS
import net.lshift.diffa.schema.tables.PairViews.PAIR_VIEWS
import net.lshift.diffa.schema.tables.Endpoints.ENDPOINTS
import net.lshift.diffa.schema.tables.Pairs.PAIRS
import net.lshift.diffa.schema.tables.Spaces.SPACES
import net.lshift.diffa.schema.tables.EndpointViews.ENDPOINT_VIEWS
import net.lshift.diffa.schema.tables.Breakers.BREAKERS
import net.lshift.diffa.schema.tables.Extents.EXTENTS
import JooqConfigStoreCompanion._
import net.lshift.diffa.kernel.naming.CacheName._
import net.lshift.diffa.kernel.util.MissingObjectException
import net.lshift.diffa.kernel.lifecycle.{PairLifecycleAware, DomainLifecycleAware}
import net.lshift.diffa.kernel.util.cache.{KeyPredicate, CacheProvider}
import reflect.BeanProperty
import java.util
import collection.mutable.ListBuffer
import org.jooq.impl.Factory
import net.lshift.diffa.kernel.frontend.DomainEndpointDef
import net.lshift.diffa.kernel.frontend.DomainPairDef
import net.lshift.diffa.kernel.frontend.PairDef
import net.lshift.diffa.kernel.frontend.EndpointDef
import org.jooq._
import java.lang.{Long => LONG}
import system.PolicyKey
import system.{ConflictingConcurrentModificationException, PolicyKey}
import java.lang.{Integer => INT}
import net.lshift.diffa.kernel.util.sequence.SequenceProvider
import net.lshift.diffa.kernel.naming.SequenceName
import java.sql.SQLIntegrityConstraintViolationException
import org.jooq.exception.DataAccessException
import net.lshift.diffa.snowflake.IdProvider
import net.lshift.diffa.kernel.frontend.DomainPairDef
import net.lshift.diffa.kernel.frontend.EndpointDef
import net.lshift.diffa.kernel.frontend.PairDef
import scala.Some
import net.lshift.diffa.kernel.config.Member
import net.lshift.diffa.kernel.config.BreakerByDomainPredicate
import net.lshift.diffa.kernel.naming.CacheName.values
import net.lshift.diffa.kernel.config.EndpointView
import net.lshift.diffa.kernel.config.DomainConfigKey
import net.lshift.diffa.kernel.config.SpacePolicyKey
import net.lshift.diffa.kernel.config.DomainPairKey
import net.lshift.diffa.kernel.config.PairRef
import net.lshift.diffa.kernel.config.PairByDomainPredicate
import net.lshift.diffa.kernel.frontend.DomainEndpointDef
import net.lshift.diffa.kernel.config.BreakerKey
import net.lshift.diffa.kernel.config.Endpoint
import net.lshift.diffa.kernel.config.ConfigOptionByDomainPredicate
import net.lshift.diffa.kernel.config.PairByDomainAndEndpointPredicate
import net.lshift.diffa.kernel.config.EndpointByDomainPredicate
import net.lshift.diffa.kernel.config.DomainEndpointKey

class JooqDomainConfigStore(jooq:JooqDatabaseFacade,
                            cacheProvider:CacheProvider,
                            sequenceProvider:SequenceProvider,
                            idProvider:IdProvider,
                            membershipListener:DomainMembershipAware)
    extends DomainConfigStore
    with DomainLifecycleAware {

  private val pairEventSubscribers = new ListBuffer[PairLifecycleAware]
  def registerPairEventListener(p:PairLifecycleAware) = pairEventSubscribers += p

  private val cachedConfigVersions = cacheProvider.getCachedMap[Long,Int]("domain.config.versions")
  private val cachedPairs = cacheProvider.getCachedMap[Long, java.util.List[DomainPairDef]]("domain.pairs")
  private val cachedPairsByKey = cacheProvider.getCachedMap[DomainPairKey, DomainPairDef]("domain.pairs.by.key")
  private val cachedEndpoints = cacheProvider.getCachedMap[Long, java.util.List[DomainEndpointDef]]("domain.endpoints")
  private val cachedEndpointsByKey = cacheProvider.getCachedMap[DomainEndpointKey, DomainEndpointDef]("domain.endpoints.by.key")
  private val cachedPairsByEndpoint = cacheProvider.getCachedMap[DomainEndpointKey, java.util.List[DomainPairDef]]("domain.pairs.by.endpoint")
  private val cachedBreakers = cacheProvider.getCachedMap[BreakerKey, Boolean](DOMAIN_PAIR_BREAKERS)

  // Config options
  private val cachedDomainConfigOptionsMap = cacheProvider.getCachedMap[Long, java.util.Map[String,String]](DOMAIN_CONFIG_OPTIONS_MAP)
  private val cachedDomainConfigOptions = cacheProvider.getCachedMap[DomainConfigKey, String](DOMAIN_CONFIG_OPTIONS)

  // Members
  private val cachedMembers = cacheProvider.getCachedMap[Long, java.util.List[Member]](USER_DOMAIN_MEMBERS)
  private val cachedPolicies = cacheProvider.getCachedMap[SpacePolicyKey, PolicyKey](SPACE_POLICIES)

  def reset {
    cachedConfigVersions.evictAll()
    cachedPairs.evictAll()
    cachedPairsByKey.evictAll()
    cachedEndpoints.evictAll()
    cachedEndpointsByKey.evictAll()
    cachedPairsByEndpoint.evictAll()
    cachedBreakers.evictAll()

    cachedDomainConfigOptionsMap.evictAll()
    cachedDomainConfigOptions.evictAll()

    cachedMembers.evictAll()
    cachedPolicies.evictAll()
  }

  private def invalidateMembershipCache(space:Long) = {
    cachedMembers.evict(space)
  }

  private def invalidateConfigCaches(space:Long) = {
    cachedDomainConfigOptionsMap.evict(space)
    cachedDomainConfigOptions.keySubset(ConfigOptionByDomainPredicate(space)).evictAll()
  }

  private def invalidateAllCaches(space:Long) = {
    cachedConfigVersions.evict(space)
    cachedEndpoints.evict(space)
    cachedPairs.evict(space)
    cachedPairsByEndpoint.keySubset(EndpointByDomainPredicate(space)).evictAll()
    cachedPairsByKey.keySubset(PairByDomainPredicate(space)).evictAll()
    cachedEndpointsByKey.keySubset(EndpointByDomainPredicate(space)).evictAll()
    cachedBreakers.keySubset(BreakerByDomainPredicate(space)).evictAll()

    invalidateConfigCaches(space)

    invalidateMembershipCache(space)
  }

  private def invalidateEndpointCachesOnly(space:Long, endpointName: String) = {
    cachedEndpoints.evict(space)
    cachedPairsByEndpoint.keySubset(PairByDomainAndEndpointPredicate(space, endpointName)).evictAll()
    cachedEndpointsByKey.evict(DomainEndpointKey(space,endpointName))

    // TODO This is a very coarse grained invalidation of the pair caches - this could be made finer at some stage
    cachedPairs.evict(space)
    cachedPairsByKey.keySubset(PairByDomainPredicate(space)).evictAll()
  }

  private def invalidatePairCachesOnly(space:Long) = {
    cachedPairs.evict(space)
    cachedPairsByKey.keySubset(PairByDomainPredicate(space)).evictAll()
    cachedPairsByEndpoint.keySubset(EndpointByDomainPredicate(space)).evictAll()
  }

  def onDomainUpdated(space: Long) = invalidateAllCaches(space)
  def onDomainRemoved(space: Long) = invalidateAllCaches(space)

  def createOrUpdateEndpoint(space: Long, endpointDef: EndpointDef) : DomainEndpointDef = {
    val id:LONG = idProvider.getId

    jooq.execute(t => {


      /* mergeInto should be used when there is merge support in MySQL. */
      if (jooq.resolvedDialect.equals(SQLDialect.MYSQL)) {
        val insert = t.insertInto(ENDPOINTS).
          set(ENDPOINTS.SPACE, space:LONG).
          set(ENDPOINTS.NAME, endpointDef.name).
          set(ENDPOINTS.ID, id).
          set(ENDPOINTS.COLLATION_TYPE, endpointDef.collation).
          set(ENDPOINTS.CONTENT_RETRIEVAL_URL, endpointDef.contentRetrievalUrl).
          set(ENDPOINTS.SCAN_URL, endpointDef.scanUrl).
          set(ENDPOINTS.VERSION_GENERATION_URL, endpointDef.versionGenerationUrl).
          set(ENDPOINTS.INBOUND_URL, endpointDef.inboundUrl)
        val update = t.update(ENDPOINTS).
          set(ENDPOINTS.COLLATION_TYPE, endpointDef.collation).
          set(ENDPOINTS.CONTENT_RETRIEVAL_URL, endpointDef.contentRetrievalUrl).
          set(ENDPOINTS.SCAN_URL, endpointDef.scanUrl).
          set(ENDPOINTS.VERSION_GENERATION_URL, endpointDef.versionGenerationUrl).
          set(ENDPOINTS.INBOUND_URL, endpointDef.inboundUrl).
          where(ENDPOINTS.SPACE.equal(space:LONG).and(ENDPOINTS.NAME.equal(endpointDef.name)))

        // This is not a fail-safe way to create or update.  It is subject to a race condition:
        // if the same endpoint is created between when the update fails and the insert is attempted,
        // then the unique key constraint violation will trigger a DataAccessException.  In this case,
        // we just let the user know it failed.
        val rowsUpdated = update.execute()
        if (rowsUpdated > 1) {
          throw new RuntimeException("Duplicate endpoint definition found for endpoint %s".format(endpointDef.name))
        } else if (rowsUpdated < 1) {
          try {
            insert.execute()
          } catch {
            case ex: DataAccessException =>
              log.warn("Potential concurrent modification inserting " + endpointDef.name, ex)
              throw new ConflictingConcurrentModificationException(endpointDef.name)
          }
        }
      } else {
      t.mergeInto(ENDPOINTS).
        usingDual().
        on(ENDPOINTS.SPACE.equal(space), ENDPOINTS.NAME.equal(endpointDef.name)).
        whenMatchedThenUpdate().
          set(ENDPOINTS.COLLATION_TYPE, endpointDef.collation).
          set(ENDPOINTS.CONTENT_RETRIEVAL_URL, endpointDef.contentRetrievalUrl).
          set(ENDPOINTS.SCAN_URL, endpointDef.scanUrl).
          set(ENDPOINTS.VERSION_GENERATION_URL, endpointDef.versionGenerationUrl).
          set(ENDPOINTS.INBOUND_URL, endpointDef.inboundUrl).
        whenNotMatchedThenInsert().
          set(ENDPOINTS.SPACE, space:LONG).
          set(ENDPOINTS.NAME, endpointDef.name).
          set(ENDPOINTS.ID, id).
          set(ENDPOINTS.COLLATION_TYPE, endpointDef.collation).
          set(ENDPOINTS.CONTENT_RETRIEVAL_URL, endpointDef.contentRetrievalUrl).
          set(ENDPOINTS.SCAN_URL, endpointDef.scanUrl).
          set(ENDPOINTS.VERSION_GENERATION_URL, endpointDef.versionGenerationUrl).
          set(ENDPOINTS.INBOUND_URL, endpointDef.inboundUrl).execute()
      }

      val endpointId = endpointIdByName(t, endpointDef.name, space)

      // Don't attempt to update to update any rows per se, just delete every associated
      // category and re-insert the new definitions, irrespective of
      // whether they are identical to the previous definitions

      deleteCategories(t, space, endpointDef.name)

      // Insert categories for the endpoint proper
      insertCategories(t, space, endpointId, endpointDef)

      // Update the view definitions

      t.delete(ENDPOINT_VIEWS).
          where(ENDPOINT_VIEWS.ENDPOINT.equal(endpointId)).
{'fraction_non_alphanumeric': 0.08844790275795386, 'fraction_numerical': 0.001427321699894102, 'mean_word_length': 4.401641382740612, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '682666', 'n_tokens_mistral': 6526, 'n_tokens_neox': 6545, 'n_words': 1149}