<commit_before>/*
 Copyright 2013 Twitter, Inc.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

package com.twitter.summingbird.scalding.store

import com.twitter.algebird.{
  Interval, Intersection, ExclusiveUpper, InclusiveUpper }
import com.twitter.summingbird.batch.{ Batcher, BatchID }
import com.twitter.summingbird.scalding.{ WaitingState, RunningState }

import java.util.Date

/**
  * State representation used by the builder API for compatibility.
  */
object VersionedState {
  def apply(meta: HDFSMetadata, startDate: Option[Date], maxBatches: Int)
    (implicit batcher: Batcher): VersionedState =
    new VersionedState(meta, startDate, maxBatches)
}

class VersionedState(meta: HDFSMetadata, startDate: Option[Date], maxBatches: Int)
  (implicit batcher: Batcher) extends WaitingState[Date] { outer =>
  def begin: RunningState[Date] = new VersionedRunningState

  private class VersionedRunningState extends RunningState[Date] {
    /**
      * Returns a date interval spanning from the beginning of the the
      * batch stored in the most recent metadata file to the current
      * time.
      */
    def part = {
      val beginning: BatchID =
        (for {
          version <- meta.mostRecentVersion
          batchString <- version.get[String]
        } yield BatchID(batchString)
        ).orElse(startDate.map(batcher.batchOf(_)))
          .getOrElse(
          sys.error("You must supply a starting date on the job's first run!")
        )
      val end = beginning + maxBatches
      Interval.leftClosedRightOpen(<commit_msg>override old time if a startTime is supplied.<commit_after>/*
 Copyright 2013 Twitter, Inc.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

package com.twitter.summingbird.scalding.store

import com.twitter.algebird.{
  Interval, Intersection, ExclusiveUpper, InclusiveUpper }
import com.twitter.summingbird.batch.{ Batcher, BatchID }
import com.twitter.summingbird.scalding.{ WaitingState, RunningState }

import java.util.Date

/**
  * State representation used by the builder API for compatibility.
  */
object VersionedState {
  def apply(meta: HDFSMetadata, startDate: Option[Date], maxBatches: Int)
    (implicit batcher: Batcher): VersionedState =
    new VersionedState(meta, startDate, maxBatches)
}

class VersionedState(meta: HDFSMetadata, startDate: Option[Date], maxBatches: Int)
  (implicit batcher: Batcher) extends WaitingState[Date] { outer =>
  def begin: RunningState[Date] = new VersionedRunningState

  private class VersionedRunningState extends RunningState[Date] {
    /**
      * Returns a date interval spanning from the beginning of the the
      * batch stored in the most recent metadata file to the current
      * time.
      */
    def part = {
      val beginning: BatchID =
        startDate.map(batcher.batchOf(_))
          .orElse {
          for {
            version <- meta.mostRecentVersion
            batchString <- version.get[String]
          } yield BatchID(batchString)
        } getOrElse {
          sys.error("You must supply a starting date on the job's first run!")
        }
      val end = beginning + maxBatches
      Interval.leftClosedRightOpen(
{'fraction_non_alphanumeric': 0.07794722827402566, 'fraction_numerical': 0.0038731541999515854, 'mean_word_length': 3.5207877461706785, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6082775', 'n_tokens_mistral': 1152, 'n_tokens_neox': 1098, 'n_words': 413}