<commit_before>import { Currencies } from '/imports/api/indexDB.js';
import { MetaCurrency } from '../serverdb/metacurrency.js';
import { HTTP } from 'meteor/http'
import { log } from '../main'

SyncedCron.add({
  name: 'Update from CoinMarketCap',
  schedule: function(parser) {
    return parser.text('every 10 minutes');
  },
  job: function() {
    Meteor.call('updateMarketCap');
  }
}); 

SyncedCron.add({
  name: 'Update from Github',
  schedule: function(parser) {
    return parser.text('every 1 days');
  },
  job: function() {
    console.log("start github doogle update");
    var time = new Date().getTime() / 1000;
    var allcurrencies = Currencies.find({}, { sort: { createdAt: -1 }}).fetch();
    for (var i in allcurrencies) {
      if (allcurrencies[i].gitRepo) {
        let gitAPI = `https://api.github.com/repos/${allcurrencies[i].gitRepo.replace(/((http|https):\/\/)?github.com\//, '').replace(/\/+$/, '')}/stats/participation` 
    
        if (allcurrencies[i].gitUpdate + 604700 < time || !allcurrencies[i].gitUpdate) {
          console.log(i + " " + gitAPI + " " + allcurrencies[i]._id)
          Meteor.call('fetchGitCommits', gitAPI, allcurrencies[i]._id, time); //https://api.github.com/repos/ethereumproject/go-ethereum/stats/participation  https://github.com/ethereumproject/go-ethereum   monero-project/monero/stats/participation
        }
      }
    }
  }
});


Meteor.methods({

 fetchGitCommits (repo, id, time) {
    //var result = HTTP.call( 'GET', repo, {headers: {"User-Agent": "gazhayes-blockrazor"}} );
   HTTP.get(repo, {
   headers: {"User-Agent": "gazhayes-blockrazor"}},
   (error, result) => {
       if (!error) {
         last4weeks = result.data.all[51] + result.data.all[50] + result.data.all[49] + result.data.all[48];
         Currencies.upsert(id, {
           $set: {
             gitCommits: last4weeks,
             gitUpdate: time
           }
         }
         )

       } else {
        log.error('Error in fetchGitCommits', error)
       }
     })
},

updateMarketCap () {
var allcurrencies = Currencies.find({}, { sort: { createdAt: -1 }}).fetch();
  HTTP.call('GET', "https://api.coinmarketcap.com/v1/ticker/", {
    }, (error, result) => {
      if (!error) {

        for (var key in result.data) {
          for (var currency in allcurrencies) {//i = 0; i < Currencies.find({}, { sort: { createdAt: -1 }}).count(); i++) {
            if (result.data[key].name.toLowerCase() == allcurrencies[currency].currencyName.toLowerCase() || result.data[key].name == allcurrencies[currency].currencySymbol) {
            console.log("Updating: " + allcurrencies[currency].currencyName);<commit_msg>Problem: Errors with github API are not self-corrected (#594)<commit_after>import { Currencies, AppLogs } from '/imports/api/indexDB.js';
import { MetaCurrency } from '../serverdb/metacurrency.js';
import { HTTP } from 'meteor/http'
import { log } from '../main'

SyncedCron.add({
  name: 'Update from CoinMarketCap',
  schedule: function(parser) {
    return parser.text('every 10 minutes');
  },
  job: function() {
    Meteor.call('updateMarketCap');
  }
}); 

SyncedCron.add({
  name: 'Update from Github',
  schedule: function(parser) {
    return parser.text('every 1 days');
  },
  job: function() {
    console.log("start github doogle update");
    var time = new Date().getTime() / 1000;
    var allcurrencies = Currencies.find({}, { sort: { createdAt: -1 }}).fetch();
    for (var i in allcurrencies) {
      if (allcurrencies[i].gitRepo) {
        let gitAPI = `https://api.github.com/repos/${allcurrencies[i].gitRepo.replace(/((http|https):\/\/)?github.com\//, '').replace(/\/+$/, '')}/stats/participation` 
    
        if (allcurrencies[i].gitUpdate + 604700 < time || !allcurrencies[i].gitUpdate) {
          console.log(i + " " + gitAPI + " " + allcurrencies[i]._id)
          Meteor.call('fetchGitCommits', gitAPI, allcurrencies[i]._id, time); //https://api.github.com/repos/ethereumproject/go-ethereum/stats/participation  https://github.com/ethereumproject/go-ethereum   monero-project/monero/stats/participation
        }
      }
    }
  }
});


Meteor.methods({
  fetchGitCommits: (repo, id, time) => {
    HTTP.get(repo, {
      headers: {
        "User-Agent": "gazhayes-blockrazor"
      }
    }, (error, result) => {
      let err = false
      if (!error) {
        if (result && result.data && result.data.all) {
          // alles gut
          last4weeks = result.data.all[51] + result.data.all[50] + result.data.all[49] + result.data.all[48];
          Currencies.upsert(id, {
            $set: {
              gitCommits: last4weeks,
              gitUpdate: time
            }
          })
        } else {
          logs.error('Error when fetching git commits.', {
            id: id,
            repo: repo,
            error: 'Result not defined.'
          })

          err = true
        }
      } else {
        log.error('Error in fetchGitCommits',{
          id: id,
          repo: repo,
          error: error
        })

        err = true
      }

      if (err) { // to prevent unnecessary checks
        let count = AppLogs.find({ // get all previous
          level: 'ERROR',
          'additional.id': id,
          'additional.repo': repo
        }).count()

        if (count >= 3) { // if this happened three times, remove the git repo, apparently something is wrong
          Currencies.upsert({
            _id: id
          }, {
            $set: {
              gitRepo: ''
            }
          })
        }
      } else {
        // if the url returns results, clean out the app logs to prevent it from getting purged
        AppLogs.remove({
          level: 'ERROR',
          'additional.id': id,
          'additional.repo': repo
        })
      }
    })
  },

updateMarketCap () {
var allcurrencies = Currencies.find({}, { sort: { createdAt: -1 }}).fetch();
  HTTP.call('GET', "https://api.coinmarketcap.com/v1/ticker/", {
    }, (error, result) => {
      if (!error) {

        for (var key in result.data) {
          for (var currency in allcurrencies) {//i = 0; i < Currencies.find({}, { sort: { createdAt: -1 }}).count(); i++) {
            if (result.data[key].name.toLowerCase() == allcurrencies[currency].currencyName.toLowerCase() || result.data[key].name == allcurrencies[currency].currencySymbol) {
            console.log("Updating: " + allcurrencies[currency].currencyName);
{'fraction_non_alphanumeric': 0.1667186525265128, 'fraction_numerical': 0.009357454772301934, 'mean_word_length': 2.3877443211833067, 'pattern_counts': {'":': 3, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2620188', 'n_tokens_mistral': 2170, 'n_tokens_neox': 2042, 'n_words': 422}