<commit_before>var request = require("request");
var fs = require('fs');
var db = require('../Schemas/config.js');
var FetchedRepo = require('../Schemas/fetchedRepos.js').FetchedRepo;
var QueryData = require('../Schemas/queryData.js').QueryData;
var pubnubPublishKey = process.env.PUBNUB_PUBLISH_KEY || api.PUBNUB_PUBLISH_KEY;
var pubnubSubscribeKey = process.env.PUBNUB_SUBSCRIBE_KEY || api.PUBNUB_SUBSCRIBE_KEY;
var pubnub = require("pubnub")({
  ssl: true, // <- enable TLS Tunneling over TCP
  publish_key: pubnubPublishKey,
  subscribe_key: pubnubSubscribeKey
});
//var api = require('./api.js');

var fullnames;

var retrieveFiles = function() {
  QueryData.find(function(err, data) {

    if (err) {
      throw err;

    } else {

      fullnames = data; //data array, retrieved from DB
      getHtml(); //
    }

  });
};

var getHtml = function() {
  var repoObj = fullnames.pop();

  if (fullnames.length > 0) {
    var apiUser = process.env.GITHUB_API_NAME || api.API_NAME;
    var apiToken = process.env.GITHUB_API_TOKEN || api.API_TOKEN;
    var req = {
      url: "https://api.github.com/search/code?q=in:file+language:html+filename:index+repo:" + repoObj.repo_name,
      headers: {
        'User-Agent': apiUser,
        'Authorization': apiToken
      }
    };

    request(req, function(error, response, body) {
      if (error) {
        console.log(error);<commit_msg>adjusting finality condition<commit_after>var request = require("request");
var fs = require('fs');
var db = require('../Schemas/config.js');
var FetchedRepo = require('../Schemas/fetchedRepos.js').FetchedRepo;
var QueryData = require('../Schemas/queryData.js').QueryData;
var pubnubPublishKey = process.env.PUBNUB_PUBLISH_KEY || api.PUBNUB_PUBLISH_KEY;
var pubnubSubscribeKey = process.env.PUBNUB_SUBSCRIBE_KEY || api.PUBNUB_SUBSCRIBE_KEY;
var pubnub = require("pubnub")({
  ssl: true, // <- enable TLS Tunneling over TCP
  publish_key: pubnubPublishKey,
  subscribe_key: pubnubSubscribeKey
});
//var api = require('./api.js');

var fullnames;

var retrieveFiles = function() {
  QueryData.find(function(err, data) {

    if (err) {
      throw err;

    } else {

      fullnames = data; //data array, retrieved from DB
      getHtml(); //
    }

  });
};

var getHtml = function() {
  var repoObj = fullnames.pop();

  if (fullnames.length >= 0) {
    var apiUser = process.env.GITHUB_API_NAME || api.API_NAME;
    var apiToken = process.env.GITHUB_API_TOKEN || api.API_TOKEN;
    var req = {
      url: "https://api.github.com/search/code?q=in:file+language:html+filename:index+repo:" + repoObj.repo_name,
      headers: {
        'User-Agent': apiUser,
        'Authorization': apiToken
      }
    };

    request(req, function(error, response, body) {
      if (error) {
        console.log(error);
{'fraction_non_alphanumeric': 0.1478167501789549, 'fraction_numerical': 0.0007158196134574087, 'mean_word_length': 3.500805152979066, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4586121', 'n_tokens_mistral': 1003, 'n_tokens_neox': 976, 'n_words': 189}