<commit_before>/*
 * Changes for SnappyData data platform.
 *
 * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License. See accompanying
 * LICENSE file.
 */

package io.snappydata

import scala.collection.JavaConverters._
import scala.language.implicitConversions

import com.gemstone.gemfire.CancelException
import com.gemstone.gemfire.cache.DataPolicy
import com.gemstone.gemfire.cache.execute.FunctionService
import com.gemstone.gemfire.i18n.LogWriterI18n
import com.gemstone.gemfire.internal.SystemTimer
import com.gemstone.gemfire.internal.cache.execute.InternalRegionFunctionContext
import com.gemstone.gemfire.internal.cache.{LocalRegion, PartitionedRegion}
import com.pivotal.gemfirexd.internal.engine.Misc
import com.pivotal.gemfirexd.internal.engine.distributed.{GfxdListResultCollector, GfxdMessage}
import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer
import com.pivotal.gemfirexd.internal.engine.ui.{SnappyRegionStats, SnappyRegionStatsCollectorFunction, SnappyRegionStatsCollectorResult}
import com.pivotal.gemfirexd.internal.iapi.types.RowLocation
import io.snappydata.Constant._

import org.apache.spark.sql.{SnappyContext, SnappySession}
import org.apache.spark.sql.collection.Utils
import org.apache.spark.{Logging, SparkContext}

object SnappyTableStatsProviderService extends Logging {

  @volatile private var tableSizeInfo = Map[String, SnappyRegionStats]()

  private var _snc: Option[SnappyContext] = None

  private def snc: SnappyContext = synchronized {
    _snc.getOrElse {
      val context = SnappyContext()
      _snc = Option(context)
      context
    }
  }

  @volatile private var doRun: Boolean = false
  @volatile private var running: Boolean = false

  def start(sc: SparkContext): Unit = {
    val delay = sc.getConf.getLong(Constant.SPARK_SNAPPY_PREFIX +
        "calcTableSizeInterval", DEFAULT_CALC_TABLE_SIZE_SERVICE_INTERVAL)
    doRun = true
    Misc.getGemFireCache.getCCPTimer.schedule(
      new SystemTimer.SystemTimerTask {
        private val logger: LogWriterI18n = Misc.getGemFireCache.getLoggerI18n

        override def run2(): Unit = {
          try {
            if (doRun) {
              val prevTableSizeInfo = tableSizeInfo
              running = true
              try {
                tableSizeInfo = getAggregatedTableStatsOnDemand
              } finally synchronized {
                running = false
                notifyAll()
              }
              // check if there has been a substantial change in table
              // stats, and clear the plan cache if so
              if (prevTableSizeInfo.size != tableSizeInfo.size) {
                println(s"cleaning cache 1")
                SnappySession.clearPlanCache()
              } else {
                val prevTotalRows = prevTableSizeInfo.values.map(_.getRowCount).sum
                val newTotalRows = tableSizeInfo.values.map(_.getRowCount).sum
                if (math.abs(newTotalRows - prevTotalRows) > 0.1 * prevTotalRows) {
                  println(s"cleaning cache 2")
                  SnappySession.clearPlanCache()
                }
              }
            }
          } catch {
            case _: CancelException => // ignore
            case e: Exception => if (!e.getMessage.contains(
              "com.gemstone.gemfire.cache.CacheClosedException")) {
              logger.warning(e)
            }
          }<commit_msg>fixed incorrect checkin<commit_after>/*
 * Changes for SnappyData data platform.
 *
 * Portions Copyright (c) 2016 SnappyData, Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License. See accompanying
 * LICENSE file.
 */

package io.snappydata

import scala.collection.JavaConverters._
import scala.language.implicitConversions

import com.gemstone.gemfire.CancelException
import com.gemstone.gemfire.cache.DataPolicy
import com.gemstone.gemfire.cache.execute.FunctionService
import com.gemstone.gemfire.i18n.LogWriterI18n
import com.gemstone.gemfire.internal.SystemTimer
import com.gemstone.gemfire.internal.cache.execute.InternalRegionFunctionContext
import com.gemstone.gemfire.internal.cache.{LocalRegion, PartitionedRegion}
import com.pivotal.gemfirexd.internal.engine.Misc
import com.pivotal.gemfirexd.internal.engine.distributed.{GfxdListResultCollector, GfxdMessage}
import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer
import com.pivotal.gemfirexd.internal.engine.ui.{SnappyRegionStats, SnappyRegionStatsCollectorFunction, SnappyRegionStatsCollectorResult}
import com.pivotal.gemfirexd.internal.iapi.types.RowLocation
import io.snappydata.Constant._

import org.apache.spark.sql.{SnappyContext, SnappySession}
import org.apache.spark.sql.collection.Utils
import org.apache.spark.{Logging, SparkContext}

object SnappyTableStatsProviderService extends Logging {

  @volatile private var tableSizeInfo = Map[String, SnappyRegionStats]()

  private var _snc: Option[SnappyContext] = None

  private def snc: SnappyContext = synchronized {
    _snc.getOrElse {
      val context = SnappyContext()
      _snc = Option(context)
      context
    }
  }

  @volatile private var doRun: Boolean = false
  @volatile private var running: Boolean = false

  def start(sc: SparkContext): Unit = {
    val delay = sc.getConf.getLong(Constant.SPARK_SNAPPY_PREFIX +
        "calcTableSizeInterval", DEFAULT_CALC_TABLE_SIZE_SERVICE_INTERVAL)
    doRun = true
    Misc.getGemFireCache.getCCPTimer.schedule(
      new SystemTimer.SystemTimerTask {
        private val logger: LogWriterI18n = Misc.getGemFireCache.getLoggerI18n

        override def run2(): Unit = {
          try {
            if (doRun) {
              val prevTableSizeInfo = tableSizeInfo
              running = true
              try {
                tableSizeInfo = getAggregatedTableStatsOnDemand
              } finally synchronized {
                running = false
                notifyAll()
              }
              // check if there has been a substantial change in table
              // stats, and clear the plan cache if so
              if (prevTableSizeInfo.size != tableSizeInfo.size) {
                SnappySession.clearPlanCache()
              } else {
                val prevTotalRows = prevTableSizeInfo.values.map(_.getRowCount).sum
                val newTotalRows = tableSizeInfo.values.map(_.getRowCount).sum
                if (math.abs(newTotalRows - prevTotalRows) > 0.1 * prevTotalRows) {
                  SnappySession.clearPlanCache()
                }
              }
            }
          } catch {
            case _: CancelException => // ignore
            case e: Exception => if (!e.getMessage.contains(
              "com.gemstone.gemfire.cache.CacheClosedException")) {
              logger.warning(e)
            }
          }
{'fraction_non_alphanumeric': 0.07861153649821337, 'fraction_numerical': 0.005104645227156712, 'mean_word_length': 2.885473475458602, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7488962', 'n_tokens_mistral': 2198, 'n_tokens_neox': 2126, 'n_words': 538}