<commit_before>Meteor.startup ->
  clean()
  sync()

  Settings.find().observe
    changed: sync

clean = ->
  ApplicationDefs.remove {}

toApp = (node) ->
  [ignore..., keyBase, project, appName, version] = node.key?.split('/')
  {project: project, name: appName, key: node.key?.split('/')[0..-2].join '/'}

@sync = (callback)->
  syncWithBaseKey = (baseKey, handler) ->
    getData = ->
      recursiveUrl = "#{baseKey}/#{Settings.findOne().project}?recursive=true"
      waitForChange = ->
        EtcdClient.discover "#{recursiveUrl}&wait=true", (err, nodes) ->
          console.log "etcd changes reported for #{baseKey}", nodes.length
          getData()
      EtcdClient.discover recursiveUrl, (err, nodes) ->
        console.log "getting data from etcd for #{baseKey}"
        console.log 'nodes retrieved:', nodes.length
        waitForChange()
        handler err, nodes
    getData()

  try
    syncWithBaseKey "apps", (err, nodes) ->
      throw new Error if err

      apps = if nodes then (toApp node for node in nodes) else []
      Apps.updateCollection apps

      objects = []
      if nodes
        for n in nodes
          [ignore..., keyBase, project, appName, version] = n.key.split('/')

          objects.push
            key: n.key
            project: project
            name: appName
            version: version
            def: n.value
            tags: Helper.extractTags n.value

      ApplicationDefs.updateCollection objects

    syncWithBaseKey "instances", (err, nodes) ->
      throw new Error if err

      objects = {}
      if nodes
        for n in nodes
          [ignore..., project, appName, instanceName, serviceName, propertyName] = n.key.split('/')
          key = [project, appName, instanceName].join '/'
          objects[key] = {services:{}, meta:{}} unless objects[key]
          if serviceName == 'meta_'
            if propertyName == 'parameters'
              objects[key].parameters = EJSON.parse n.value
            else
              objects[key].meta[propertyName] = n.value
          else
            objects[key].services[serviceName] = {} unless objects[key].services[serviceName]
            objects[key].services[serviceName][propertyName] = n.value

          objects[key].project = project
          objects[key].application = appName
          objects[key].name = instanceName
          objects[key].key = key

      Instances.updateCollection (value for key, value of objects)

    callback?()

  catch
    console.error "Error while trying to read #{baseKey}!"
    sync() # go back to syncing<commit_msg>Do synchronization process if there is no errors<commit_after>Meteor.startup ->
  clean()
  sync()

  Settings.find().observe
    changed: sync

clean = ->
  ApplicationDefs.remove {}

toApp = (node) ->
  [ignore..., keyBase, project, appName, version] = node.key?.split('/')
  {project: project, name: appName, key: node.key?.split('/')[0..-2].join '/'}

@sync = (callback)->
  syncWithBaseKey = (baseKey, handler) ->
    getData = ->
      recursiveUrl = "#{baseKey}/#{Settings.findOne().project}?recursive=true"
      waitForChange = ->
        EtcdClient.discover "#{recursiveUrl}&wait=true", (err, nodes) ->
          console.log "etcd changes reported for #{baseKey}", nodes.length
          getData()
      EtcdClient.discover recursiveUrl, (err, nodes) ->
        console.log "getting data from etcd for #{baseKey}"
        console.log 'nodes retrieved:', nodes.length
        waitForChange()
        handler err, nodes
    getData()

  try
    syncWithBaseKey "apps", (err, nodes) ->
      if not err
        apps = if nodes then (toApp node for node in nodes) else []
        Apps.updateCollection apps

        objects = []
        if nodes
          for n in nodes
            [ignore..., keyBase, project, appName, version] = n.key.split('/')

            objects.push
              key: n.key
              project: project
              name: appName
              version: version
              def: n.value
              tags: Helper.extractTags n.value

        ApplicationDefs.updateCollection objects

    syncWithBaseKey "instances", (err, nodes) ->
      if not err
        objects = {}
        if nodes
          for n in nodes
            [ignore..., project, appName, instanceName, serviceName, propertyName] = n.key.split('/')
            key = [project, appName, instanceName].join '/'
            objects[key] = {services:{}, meta:{}} unless objects[key]
            if serviceName == 'meta_'
              if propertyName == 'parameters'
                objects[key].parameters = EJSON.parse n.value
              else
                objects[key].meta[propertyName] = n.value
            else
              objects[key].services[serviceName] = {} unless objects[key].services[serviceName]
              objects[key].services[serviceName][propertyName] = n.value

            objects[key].project = project
            objects[key].application = appName
            objects[key].name = instanceName
            objects[key].key = key

        Instances.updateCollection (value for key, value of objects)

    callback?()

  catch
    console.error "Error while trying to read #{baseKey}!"
    sync() # go back to syncing
{'fraction_non_alphanumeric': 0.12406662837449742, 'fraction_numerical': 0.0007658433850277618, 'mean_word_length': 2.1776155717761556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7633555', 'n_tokens_mistral': 1459, 'n_tokens_neox': 1421, 'n_words': 390}