<commit_before>var request = require('request-json');

var mongoose = require("mongoose");
var Anime   = require('../models/anime');

// Database
var config = require('../config/config');
mongoose.connect(config.database);

//data to get token
var data = {
  grant_type    : "client_credentials",
  client_id     :  process.env.ClientID,
  client_secret :  process.env.ClientSecret,
};
// client url
var client = request.createClient('https://anilist.co/api/');
//post to get token back
client.post('auth/access_token', data, function(err, res, body) {
  if (err) return console.log(err); //error
  var clientToken = body.access_token;
  var sendClientToken = '?access_token=' + clientToken;
  var params =
    '&year=2016' +
    '&season=winter' +
    '&type=TV' +
    '&sort=popularity-desc';

  var getUrl = 'browse/anime' + sendClientToken + params;
  //get all anime with url params
  client.get(getUrl, function(err, res, body) {
    if (err) return console.log(err); //error
    // loop request for every anime from browse response
    console.log(body.length);
    for (var i = 0; i < body.length; i++) {
      var animeId = body[i].id;
      var url = 'anime/' + animeId + sendClientToken;
      // get data for each anime by get request
      client.get(url, function(err, res, body) {
        if (err) return console.log(err); //error
        // clear description
        var re = /<br>|\(Source.+/g;
        var description = body.description.replace(re, "");
        //create a mongoose document for every returned request
        Anime.create({
          _id: body.id,
          picture: body.image_url_lge,
          title: body.title_romaji,
          rating: body.popularity,
          episodes: body.total_episodes,
          description: description,
          comments: []
        });
      })
    }
  });
});

console.log("done");<commit_msg>save to db function callback for after http request<commit_after>var request = require('request-json');

var mongoose = require("mongoose");
var Anime   = require('../models/anime');

// Database
var config = require('../config/config');
mongoose.connect(config.database);

function saveToDb(err, res, body) {
  if (err) return console.log(err); //error
  // clear description
  console.log(body);
  var re = /<br>|\(Source.+/g;
  var description = body.description.replace(re, "");
  //create a mongoose document for every returned request
  Anime.create({
    _id: body.id,
    picture: body.image_url_lge,
    title: body.title_romaji,
    rating: body.popularity,
    episodes: body.total_episodes,
    description: description,
    comments: []
  });
}


//data to get token
var data = {
  grant_type    : "client_credentials",
  client_id     :  process.env.ClientID,
  client_secret :  process.env.ClientSecret,
};
// client url
var client = request.createClient('https://anilist.co/api/');
//post to get token back
client.post('auth/access_token', data, function(err, res, body) {
  if (err) return console.log(err); //error
  var clientToken = body.access_token;
  var sendClientToken = '?access_token=' + clientToken;
  var params =
    '&year=2016' +
    '&season=winter' +
    '&type=TV' +
    '&sort=popularity-desc';

  var getUrl = 'browse/anime' + sendClientToken + params;
  //get all anime with url params
  client.get(getUrl, function(err, res, body) {
    if (err) return console.log(err); //error
    // loop request for every anime from browse response
    for (var i = 0; i < body.length; i++) {
      var animeId = body[i].id;
      var url = 'anime/' + animeId + sendClientToken;
      // get data for each anime by get request
      client.get(url, saveToDb);
    }
  });
});

console.log("done");
{'fraction_non_alphanumeric': 0.14487249050461204, 'fraction_numerical': 0.0027129679869777536, 'mean_word_length': 3.0251091703056767, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '57555', 'n_tokens_mistral': 1194, 'n_tokens_neox': 1184, 'n_words': 311}