<commit_before>_         = require 'underscore'
Backbone  = require 'backbone'
redis     = require 'redis'
{ parse } = require("url")
{ REDIS_URL, DEFAULT_CACHE_TIME, NODE_ENV } = require('../config.js')

client = undefined

# Setup redis client
if NODE_ENV == 'development'
  client = redis.createClient()
else if NODE_ENV != 'test' and REDIS_URL
  red = parse(REDIS_URL || '');
  client = redis.createClient(red.port, red.hostname);
  client.auth(red.auth.split(':')[1]);

module.exports = (artsyUrl) ->

  # For paginated routes, this will recursively fetch until the end of the set.
  #
  # @param {Object} options Backbone sync options like `success` and `error`

  fetchUntilEnd: (options = {}) ->
    console.log client, '<<<<<<<<<<<<<',
    key = "fetch-until-end:#{@url}"
    success = =>
      page = 0
      opts = _.clone(options)
      fetchPage = =>
        @fetch _.extend opts,
          data: _.extend (opts.data ? {}), page: page += 1
          remove: false
          success: (col, res) =>
            if res.length is 0
              if client
                client.set key, @models.toJSON()
                client.expire key, DEFAULT_CACHE_TIME
              options.success?(@)
            else
              fetchPage()
          error: options.error
      fetchPage()

    if client
      client.get key, (err, json) =>
        if json
          @reset json<commit_msg>hmmmmmmm<commit_after>_         = require 'underscore'
Backbone  = require 'backbone'
redis     = require 'redis'
{ parse } = require("url")
{ REDIS_URL, DEFAULT_CACHE_TIME, NODE_ENV } = require('../').config

client = undefined

# Setup redis client
if NODE_ENV == 'development'
  client = redis.createClient()
else if NODE_ENV != 'test' and REDIS_URL
  red = parse(REDIS_URL || '');
  client = redis.createClient(red.port, red.hostname);
  client.auth(red.auth.split(':')[1]);

module.exports = (artsyUrl) ->

  # For paginated routes, this will recursively fetch until the end of the set.
  #
  # @param {Object} options Backbone sync options like `success` and `error`

  fetchUntilEnd: (options = {}) ->
    key = "fetch-until-end:#{@get('id')}"
    console.log client, '<<<<<<<<<<<<<', key
    success = =>
      page = 0
      opts = _.clone(options)
      fetchPage = =>
        @fetch _.extend opts,
          data: _.extend (opts.data ? {}), page: page += 1
          remove: false
          success: (col, res) =>
            if res.length is 0
              if client
                client.set key, @models.toJSON()
                client.expire key, DEFAULT_CACHE_TIME
              options.success?(@)
            else
              fetchPage()
          error: options.error
      fetchPage()

    if client
      client.get key, (err, json) =>
        if json
          @reset json
{'fraction_non_alphanumeric': 0.13748657357679914, 'fraction_numerical': 0.0028643036161833156, 'mean_word_length': 2.2564102564102564, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2435279', 'n_tokens_mistral': 871, 'n_tokens_neox': 856, 'n_words': 232}