<commit_before>var request = require('request');

var MarketPlaceIngestor = function MarketPlaceIngestor() {
  var BASE_URL = 'https://marketplace.firefox.com/api/v1/apps/search/?format=json&limit=100';
  var HOST = 'https://marketplace.firefox.com';

  var recursiveFetch = function recursiveFetch(url, apps, cb) {
    request(url, function onBody(error, response, body) {
      if (error) {
        cb(error);
        return;
      } else if (response.statusCode != 200) {
        cb({status: response.statusCode});
        return;
      }

      var marketRes = JSON.parse(body);
      var meta = marketRes.meta;
      var allApps = marketRes.objects;
      var app = null;
      for(var i = 0; i < allApps.length; i++) {
        app = allApps[i];
        if (app.device_types.indexOf('firefoxos') == -1) {
          continue;
        }

        apps.push(app);
      }

      if (meta.next) {
        recursiveFetch(HOST + meta.next, apps, cb);<commit_msg>Better query for extracting the dump<commit_after>var request = require('request');

var MarketPlaceIngestor = function MarketPlaceIngestor() {
  var BASE_URL = 'https://marketplace.firefox.com/api/v1/apps/search/?format=json&limit=100&device=firefoxos';
  var HOST = 'https://marketplace.firefox.com';

  var recursiveFetch = function recursiveFetch(url, apps, cb) {
    request(url, function onBody(error, response, body) {
      if (error) {
        cb(error);
        return;
      } else if (response.statusCode != 200) {
        cb({status: response.statusCode});
        return;
      }

      var marketRes = JSON.parse(body);
      var meta = marketRes.meta;
      var allApps = marketRes.objects;
      var app = null;
      for(var i = 0; i < allApps.length; i++) {
        app = allApps[i];
        if (app.device_types.indexOf('firefoxos') == -1) {
          continue;
        }

        apps.push(app);
      }

      if (meta.next) {
        recursiveFetch(HOST + meta.next, apps, cb);
{'fraction_non_alphanumeric': 0.13673469387755102, 'fraction_numerical': 0.009183673469387756, 'mean_word_length': 2.53971119133574, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4853868', 'n_tokens_mistral': 646, 'n_tokens_neox': 617, 'n_words': 136}