<commit_before>bs = require 'bs58check'
defer = require 'when'
deferObject = require 'when/keys'

HDKey = require 'hdkey'
Definition = require './definition'

HARDENED_OFFSET = 0x80000000

class Collection
  constructor: (@custodian, @model, key, @hardened=false) ->
    if !(@ instanceof Collection) then return new Collection(@custodian, @model, key, @hardened)

    key = bs.decode(key) if (typeof key is 'string')

    @hdkey = HDKey.fromExtendedKey(bs.encode(key))
    @privateKey = @hdkey.privateKey
    @publicKey = @hdkey.publicKey

    @readonly = !@privateKey?

    if @hardened then @offset = HARDENED_OFFSET
    else @offset = 0

    @_definitions = {}
    @_documents = []

  definition: (key, definition) ->
    if definition?
      Definition(@custodian, definition).save()
      .then (hash) => @_definitions[key] = bs.encode(hash)
    else if key?
      key = @_definitions[key] if (typeof key is 'string' and key.length <= 32)
      Definition(@custodian, key)
    else
      deferObject.map (@_definitions), (definition) =>
        Definition(@custodian, definition)

  insert: (definition, data, autosave=true) ->
    return false if @readonly
    document = @at()
    document.definition(@definition(definition))
    .then => document.data(data)
    .then -> document.save() if autosave
    .then => @_documents[document.hdkey.index - @offset] = document

  at: (index=@_documents.length+@offset) ->
    document = @_documents[index-@offset]
    return document if document?

    hdkey = @hdkey.deriveChild(index)
    key = hdkey.privateExtendedKey ? hdkey.publicExtendedKey
    new @model(@custodian, key)

  all: (refresh, pluck) ->
    if (typeof refresh is 'string')
      pluck = refresh
      refresh = false
    @sync(refresh).then =>
      if pluck then defer.map(@_documents, (d) -> d[pluck]?())
      else @_documents

  sync: (refresh=false) ->
    start = @offset
    start = start + @_documents.length if !refresh
    defer.iterate(
      (document) => @at(document.hdkey.index + 1)
      (document) => document.fetch().yield(false).else(true)
      (document) => @_documents[document.hdkey.index - @offset] = document
      @at(start)
    )

exports = module.exports = Collection<commit_msg>Added sync to insert on collection<commit_after>bs = require 'bs58check'
defer = require 'when'
deferObject = require 'when/keys'

HDKey = require 'hdkey'
Definition = require './definition'

HARDENED_OFFSET = 0x80000000

class Collection
  constructor: (@custodian, @model, key, @hardened=false) ->
    if !(@ instanceof Collection) then return new Collection(@custodian, @model, key, @hardened)

    key = bs.decode(key) if (typeof key is 'string')

    @hdkey = HDKey.fromExtendedKey(bs.encode(key))
    @privateKey = @hdkey.privateKey
    @publicKey = @hdkey.publicKey

    @readonly = !@privateKey?

    if @hardened then @offset = HARDENED_OFFSET
    else @offset = 0

    @_definitions = {}
    @_documents = []

  definition: (key, definition) ->
    if definition?
      Definition(@custodian, definition).save()
      .then (hash) => @_definitions[key] = bs.encode(hash)
    else if key?
      key = @_definitions[key] if (typeof key is 'string' and key.length <= 32)
      Definition(@custodian, key)
    else
      deferObject.map (@_definitions), (definition) =>
        Definition(@custodian, definition)

  insert: (definition, data, autosave=true) ->
    return false if @readonly
    @sync().then =>
      document = @at()
      document.definition(@definition(definition))
      .then => document.data(data)
      .then -> document.save() if autosave
      .then => @_documents[document.hdkey.index - @offset] = document

  at: (index=@_documents.length+@offset) ->
    document = @_documents[index-@offset]
    return document if document?

    hdkey = @hdkey.deriveChild(index)
    key = hdkey.privateExtendedKey ? hdkey.publicExtendedKey
    new @model(@custodian, key)

  all: (refresh, pluck) ->
    if (typeof refresh is 'string')
      pluck = refresh
      refresh = false
    @sync(refresh).then =>
      console.log 'all', @_documents.length
      if pluck then defer.map(@_documents, (d) -> d[pluck]?())
      else @_documents

  sync: (refresh=false) ->
    start = @offset
    start = start + @_documents.length if !refresh
    defer.iterate(
      (document) => @at(document.hdkey.index + 1)
      (document) => document.fetch().yield(false).else(true)
      (document) => @_documents[document.hdkey.index - @offset] = document
      @at(start)
    ).then => @

exports = module.exports = Collection
{'fraction_non_alphanumeric': 0.13780996269475532, 'fraction_numerical': 0.0065832784726793945, 'mean_word_length': 3.3368220742150334, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3910457', 'n_tokens_mistral': 1516, 'n_tokens_neox': 1428, 'n_words': 353}