<commit_before>semver = Npm.require 'semver'
Future = Npm.require 'fibers/future'

globals = @

# From Meteor's random/random.js
UNMISTAKABLE_CHARS = '23456789ABCDEFGHJKLMNPQRSTWXYZabcdefghijkmnopqrstuvwxyz'

INSTANCES = parseInt(process.env.PEERDB_INSTANCES ? 1)
INSTANCE = parseInt(process.env.PEERDB_INSTANCE ? 0)

throw new Error "Invalid number of instances: #{ INSTANCES }" unless 0 <= INSTANCES <= UNMISTAKABLE_CHARS.length
throw new Error "Invalid instance index: #{ INSTANCE }" unless (INSTANCES is 0 and INSTANCE is 0) or 0 <= INSTANCE < INSTANCES

# TODO: Support also other types of _id generation (like ObjectID)
# TODO: We could do also a hash of an ID and then split, this would also prevent any DOS attacks by forcing IDs of a particular form
PREFIX = UNMISTAKABLE_CHARS.split ''

if INSTANCES > 1
  range = UNMISTAKABLE_CHARS.length / INSTANCES
  PREFIX = PREFIX[Math.round(INSTANCE * range)...Math.round((INSTANCE + 1) * range)]

# Fields:
#   serial
#   migrationName
#   oldCollectionName
#   newCollectionName
#   oldVersion
#   newVersion
#   timestamp
#   migrated
# We use a lower case collection name to signal it is a system collection
globals.Document.Migrations = new Meteor.Collection 'migrations'

fieldsToProjection = (fields) ->
  projection =
    _id: 1 # In the case we want only id, that is, detect deletions
  for field in fields
    if _.isString field
      projection[field] = 1
    else
      _.extend projection, field
  projection

# TODO: Should we add retry?
observerCallback = (f) ->
  return (id, args...) ->
    try
      # We call f only if the first character of id is in PREFIX.
      # By that we allow each instance to operate only on a subset
      # of documents, allowing simple coordination while scaling.
      f id, args... if id[0] in PREFIX
    catch e
      Log.error "PeerDB exception: #{ e }: #{ util.inspect args, depth: 10 }"
      Log.debug e.stack

extractValue = (obj, path) ->
  while path.length
    obj = obj[path[0]]
    path = path[1..]
  obj

# Cannot use => here because we are not in the globals.Document._TargetedFieldsObservingField context
globals.Document._TargetedFieldsObservingField::_setupTargetObservers = (updateAll) ->
  initializing = true

  observers =
    added: observerCallback (id, fields) =>
      @updateSource id, fields if updateAll or not initializing
<commit_msg>Debug logging is not visible by default.<commit_after>semver = Npm.require 'semver'
Future = Npm.require 'fibers/future'

globals = @

# From Meteor's random/random.js
UNMISTAKABLE_CHARS = '23456789ABCDEFGHJKLMNPQRSTWXYZabcdefghijkmnopqrstuvwxyz'

INSTANCES = parseInt(process.env.PEERDB_INSTANCES ? 1)
INSTANCE = parseInt(process.env.PEERDB_INSTANCE ? 0)

throw new Error "Invalid number of instances: #{ INSTANCES }" unless 0 <= INSTANCES <= UNMISTAKABLE_CHARS.length
throw new Error "Invalid instance index: #{ INSTANCE }" unless (INSTANCES is 0 and INSTANCE is 0) or 0 <= INSTANCE < INSTANCES

# TODO: Support also other types of _id generation (like ObjectID)
# TODO: We could do also a hash of an ID and then split, this would also prevent any DOS attacks by forcing IDs of a particular form
PREFIX = UNMISTAKABLE_CHARS.split ''

if INSTANCES > 1
  range = UNMISTAKABLE_CHARS.length / INSTANCES
  PREFIX = PREFIX[Math.round(INSTANCE * range)...Math.round((INSTANCE + 1) * range)]

# Fields:
#   serial
#   migrationName
#   oldCollectionName
#   newCollectionName
#   oldVersion
#   newVersion
#   timestamp
#   migrated
# We use a lower case collection name to signal it is a system collection
globals.Document.Migrations = new Meteor.Collection 'migrations'

fieldsToProjection = (fields) ->
  projection =
    _id: 1 # In the case we want only id, that is, detect deletions
  for field in fields
    if _.isString field
      projection[field] = 1
    else
      _.extend projection, field
  projection

# TODO: Should we add retry?
observerCallback = (f) ->
  return (id, args...) ->
    try
      # We call f only if the first character of id is in PREFIX.
      # By that we allow each instance to operate only on a subset
      # of documents, allowing simple coordination while scaling.
      f id, args... if id[0] in PREFIX
    catch e
      Log.error "PeerDB exception: #{ e }: #{ util.inspect args, depth: 10 }"
      Log.error e.stack

extractValue = (obj, path) ->
  while path.length
    obj = obj[path[0]]
    path = path[1..]
  obj

# Cannot use => here because we are not in the globals.Document._TargetedFieldsObservingField context
globals.Document._TargetedFieldsObservingField::_setupTargetObservers = (updateAll) ->
  initializing = true

  observers =
    added: observerCallback (id, fields) =>
      @updateSource id, fields if updateAll or not initializing

{'fraction_non_alphanumeric': 0.08647170208289501, 'fraction_numerical': 0.009678098043341047, 'mean_word_length': 3.6791338582677167, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3459177', 'n_tokens_mistral': 1549, 'n_tokens_neox': 1480, 'n_words': 520}