<commit_before>util = require 'util'
URL = require 'url'
Backbone = require 'backbone'
_ = require 'underscore'
moment = require 'moment'
inflection = require 'inflection'
Queue = require 'queue-async'

S4 = -> (((1+Math.random())*0x10000)|0).toString(16).substring(1)

BATCH_DEFAULT_LIMIT = 1500
BATCH_DEFAULT_PARALLELISM = 1

INTERVAL_TYPES = ['milliseconds', 'seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years']


module.exports = class Utils
  @bbCallback: (callback) -> return {success: ((model, resp, options) -> callback(null, model, resp, options)), error: ((model, resp, options) -> callback(resp or new Error('Backbone call failed'), model, resp, options))}
  @wrapOptions: (options={}, callback) ->
    options = Utils.bbCallback(options) if _.isFunction(options) # node style callback
    return _.defaults(Utils.bbCallback((err, model, resp, modified_options) -> callback(err, model, resp, options)), options)

  # @private
  @guid = -> return (S4()+S4()+"-"+S4()+"-"+S4()+"-"+S4()+"-"+S4()+S4()+S4())

  # @private
  @parseUrl: (url) ->
    url_parts = URL.parse(url)
    database_parts = url_parts.pathname.split('/')
    table = database_parts.pop()
    database = database_parts[database_parts.length-1]
    url_parts.pathname = database_parts.join('/')

    result = {
      host: url_parts.hostname
      port: url_parts.port
      database_path: URL.format(url_parts)
      database: database
      table: table
      model_name: inflection.classify(inflection.singularize(table))
    }

    if url_parts.auth
      auth_parts = url_parts.auth.split(':')
      result.user = auth_parts[0]
      result.password = if auth_parts.length > 1 then auth_parts[1] else null

    return result

  ##############################
  # ModelType
  ##############################
  @configureCollectionModelType: (type, sync) ->
    modelURL = ->
      url = _.result(@collection or type::, 'url')
      unless @isNew()
        url_parts = URL.parse(url)
        url_parts.pathname = "#{url_parts.pathname}/encodeURIComponent(@id)"
        url = URL.format(url_parts)
      return url

    model_type = type::model
    if not model_type or (model_type is Backbone.Model)
      class ORMModel extends Backbone.Model
        url: modelURL
        sync: sync(ORMModel)
      return type::model = ORMModel
    else if model_type::sync is Backbone.Model::sync # override built-in backbone sync
      model_type::url = modelURL
      model_type::sync = sync(model_type)
    return model_type

  ##############################
  # Relational
  ##############################
  @findOrGenerateModelName: (model_type) ->
    return model_type.model_name if model_type.model_name
    if url = _.result(model_type.prototype, 'url')
      return model_name if model_name = Utils.parseUrl(url).model_name
    return model_type.name if model_type.name
    throw "Could not find or generate model name for #{model_type}"

  # @private
  @findOrGenerateReverseRelation: (relation) ->
    model_type = relation.model_type
    reverse_model_type = relation.reverse_model_type
    reverse_model_type.sync = model_type.createSync(reverse_model_type) unless _.isFunction(reverse_model_type.schema) # not a relational model

    if relation.as
      reverse_relation = reverse_model_type.relation(relation.as)
#      throw new Error "Reverse relation from `#{@model_type.name}` as `#{@as}` not found on model `#{@reverse_model_type.name}`" unless @reverse_relation
      if reverse_relation
        reverse_relation.foreign_key = relation.foreign_key
        reverse_relation.reverse_relation = relation
    else
      key_root = inflection.underscore(model_type.model_name)
      unless reverse_relation = reverse_model_type.relation(reverse_key = inflection.singularize(key_root)) # singular
        reverse_relation = reverse_model_type.relation(reverse_key = inflection.pluralize(key_root)) # plural

    # check for reverse since they need to store the foreign key
    if not reverse_relation and (relation.type is 'hasOne' or relation.type is 'hasMany')
      reverse_relation =  reverse_model_type.schema().generateBelongsTo(reverse_model_type, model_type)
    return reverse_relation

  # @private
  @findOrGenerateJoinTable: (relation) ->
    # already exists
    return join_table if join_table = (relation.join_table or relation.reverse_relation.join_table)
    return relation.model_type.schema().generateJoinTable(relation)

  ##############################
  # Data to Model Helpers
  ##############################

  # @private
  @dataId: (data) -> return data?.id or data

  # @private
  @dataToModel: (data, model_type) ->
    return null unless data
    return (Utils.dataToModel(item, model_type) for item in data) if _.isArray(data)
    if data instanceof Backbone.Model
      model = data
    else if _.isObject(data)
      model = new model_type(model_type::parse(data))
    else
      model = new model_type({id: data})
      model.setLoaded(false)
    return model

  @updateModel: (model, data) ->
    return if not data or (model is data) or data._orm_needs_load
    data = data.toJSON() if data instanceof Backbone.Model
    if _.isObject(data)
      model.setLoaded(true)
      model.set(data)
      schema = model.schema()
      for key of model.attributes
        continue unless _.isUndefined(data[key])
        if schema and relation = schema.relation(key)
          model.unset(key) if relation.type is 'belongsTo' and _.isUndefined(data[relation.ids_accessor]) # unset removed keys
        else
          model.unset(key)
    return model

  @updateOrNew: (data, model_type) ->
    if cache = model_type.cache
      Utils.updateModel(model, data) if model = cache.get(Utils.dataId(data))
    unless model
      model = if data instanceof Backbone.Model then data else Utils.dataToModel(data, model_type)
      cache.set(model.id, model) if model and cache
    return model

  ##############################
  # Sorting
  ##############################
  # @private
  @isSorted: (models, fields) ->
    fields = _.uniq(fields)
    for model in models
      return false if last_model and @fieldCompare(last_model, model, fields) is 1
      last_model = model
    return true

  # @private
  @fieldCompare: (model, other_model, fields) ->
    field = fields[0]
    field = field[0] if _.isArray(field) # for mongo<commit_msg>Removed unset<commit_after>util = require 'util'
URL = require 'url'
Backbone = require 'backbone'
_ = require 'underscore'
moment = require 'moment'
inflection = require 'inflection'
Queue = require 'queue-async'

S4 = -> (((1+Math.random())*0x10000)|0).toString(16).substring(1)

BATCH_DEFAULT_LIMIT = 1500
BATCH_DEFAULT_PARALLELISM = 1

INTERVAL_TYPES = ['milliseconds', 'seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years']


module.exports = class Utils
  @bbCallback: (callback) -> return {success: ((model, resp, options) -> callback(null, model, resp, options)), error: ((model, resp, options) -> callback(resp or new Error('Backbone call failed'), model, resp, options))}
  @wrapOptions: (options={}, callback) ->
    options = Utils.bbCallback(options) if _.isFunction(options) # node style callback
    return _.defaults(Utils.bbCallback((err, model, resp, modified_options) -> callback(err, model, resp, options)), options)

  # @private
  @guid = -> return (S4()+S4()+"-"+S4()+"-"+S4()+"-"+S4()+"-"+S4()+S4()+S4())

  # @private
  @parseUrl: (url) ->
    url_parts = URL.parse(url)
    database_parts = url_parts.pathname.split('/')
    table = database_parts.pop()
    database = database_parts[database_parts.length-1]
    url_parts.pathname = database_parts.join('/')

    result = {
      host: url_parts.hostname
      port: url_parts.port
      database_path: URL.format(url_parts)
      database: database
      table: table
      model_name: inflection.classify(inflection.singularize(table))
    }

    if url_parts.auth
      auth_parts = url_parts.auth.split(':')
      result.user = auth_parts[0]
      result.password = if auth_parts.length > 1 then auth_parts[1] else null

    return result

  ##############################
  # ModelType
  ##############################
  @configureCollectionModelType: (type, sync) ->
    modelURL = ->
      url = _.result(@collection or type::, 'url')
      unless @isNew()
        url_parts = URL.parse(url)
        url_parts.pathname = "#{url_parts.pathname}/encodeURIComponent(@id)"
        url = URL.format(url_parts)
      return url

    model_type = type::model
    if not model_type or (model_type is Backbone.Model)
      class ORMModel extends Backbone.Model
        url: modelURL
        sync: sync(ORMModel)
      return type::model = ORMModel
    else if model_type::sync is Backbone.Model::sync # override built-in backbone sync
      model_type::url = modelURL
      model_type::sync = sync(model_type)
    return model_type

  ##############################
  # Relational
  ##############################
  @findOrGenerateModelName: (model_type) ->
    return model_type.model_name if model_type.model_name
    if url = _.result(model_type.prototype, 'url')
      return model_name if model_name = Utils.parseUrl(url).model_name
    return model_type.name if model_type.name
    throw "Could not find or generate model name for #{model_type}"

  # @private
  @findOrGenerateReverseRelation: (relation) ->
    model_type = relation.model_type
    reverse_model_type = relation.reverse_model_type
    reverse_model_type.sync = model_type.createSync(reverse_model_type) unless _.isFunction(reverse_model_type.schema) # not a relational model

    if relation.as
      reverse_relation = reverse_model_type.relation(relation.as)
#      throw new Error "Reverse relation from `#{@model_type.name}` as `#{@as}` not found on model `#{@reverse_model_type.name}`" unless @reverse_relation
      if reverse_relation
        reverse_relation.foreign_key = relation.foreign_key
        reverse_relation.reverse_relation = relation
    else
      key_root = inflection.underscore(model_type.model_name)
      unless reverse_relation = reverse_model_type.relation(reverse_key = inflection.singularize(key_root)) # singular
        reverse_relation = reverse_model_type.relation(reverse_key = inflection.pluralize(key_root)) # plural

    # check for reverse since they need to store the foreign key
    if not reverse_relation and (relation.type is 'hasOne' or relation.type is 'hasMany')
      reverse_relation =  reverse_model_type.schema().generateBelongsTo(reverse_model_type, model_type)
    return reverse_relation

  # @private
  @findOrGenerateJoinTable: (relation) ->
    # already exists
    return join_table if join_table = (relation.join_table or relation.reverse_relation.join_table)
    return relation.model_type.schema().generateJoinTable(relation)

  ##############################
  # Data to Model Helpers
  ##############################

  # @private
  @dataId: (data) -> return data?.id or data

  # @private
  @dataToModel: (data, model_type) ->
    return null unless data
    return (Utils.dataToModel(item, model_type) for item in data) if _.isArray(data)
    if data instanceof Backbone.Model
      model = data
    else if _.isObject(data)
      model = new model_type(model_type::parse(data))
    else
      model = new model_type({id: data})
      model.setLoaded(false)
    return model

  @updateModel: (model, data) ->
    return if not data or (model is data) or data._orm_needs_load
    data = data.toJSON() if data instanceof Backbone.Model
    if _.isObject(data)
      model.setLoaded(true)
      model.set(data)
    return model

  @updateOrNew: (data, model_type) ->
    if cache = model_type.cache
      Utils.updateModel(model, data) if model = cache.get(Utils.dataId(data))
    unless model
      model = if data instanceof Backbone.Model then data else Utils.dataToModel(data, model_type)
      cache.set(model.id, model) if model and cache
    return model

  ##############################
  # Sorting
  ##############################
  # @private
  @isSorted: (models, fields) ->
    fields = _.uniq(fields)
    for model in models
      return false if last_model and @fieldCompare(last_model, model, fields) is 1
      last_model = model
    return true

  # @private
  @fieldCompare: (model, other_model, fields) ->
    field = fields[0]
    field = field[0] if _.isArray(field) # for mongo
{'fraction_non_alphanumeric': 0.14507269789983845, 'fraction_numerical': 0.005169628432956382, 'mean_word_length': 3.8175097276264593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3693095', 'n_tokens_mistral': 3791, 'n_tokens_neox': 3702, 'n_words': 924}