<commit_before>"use strict";

var fs = require('fs');
var readline = require('readline');

var GitHubApi = require('github');
var _ = require('lodash');
var async = require('async');

var config = require('./config');

function get_most_recent_pr(recent_cb){
  fs.readFile(config.state_file, function(err, data){
    if(err){
      fs.writeFile(config.state_file, '0', function(err){
        if(err) return recent_cb(err);
        recent_cb(null, 0);
      });
    } else {
      recent_cb(null, Number(data));
    }
  });
}

function github_process_prs(most_recent_pr_checked, pr_cb){
  var github = new GitHubApi();

  var securedrop = {
    user: config.github_user,
    repo: config.github_repo
  }

  github.authenticate({
    type: "oauth",
    token: config.github_token || process.env.GITHUB_TOKEN
  })

  // Label all PRs which meet the criteria for labelling
  function github_process_pr_page(first_page){
    return function(err, pull_requests){
      if(first_page){
        fs.writeFile(config.state_file, pull_requests[0].number, function(err){
          if(err) return pr_cb(err);
        });
      }

      var file_checks = [];
      _.each(pull_requests, function(pull_request){

        if(pull_request.number > most_recent_pr_checked){

          file_checks.push(function(fc_cb){
            github.pullRequests.getFiles(_.extend(securedrop, {
              number: pull_request.number
            }), function(err, files){
              if(err) return pr_cb(err);

              var labels = new Set();

              _.each(files, function(file){

                if(file.filename.match(/^securedrop\/journalist_templates\//) ||
                  file.filename.match(/^securedrop\/journalist.py$/)){
                  labels.add("journalist_interface");
                }

                if(file.filename.match(/^securedrop\/source_templates\//) ||
                  file.filename.match(/^securedrop\/source.py$/)){
                  labels.add("source_interface");
                }
                
              });

              fc_cb(null, {pr: pull_request.number, labels: labels});

            });
          });

        }
      });

      async.parallel(file_checks, function(err, results){
        _.each(results, function(result){<commit_msg>Adding more labels<commit_after>"use strict";

var fs = require('fs');
var readline = require('readline');

var GitHubApi = require('github');
var _ = require('lodash');
var async = require('async');

var config = require('./config');

function get_most_recent_pr(recent_cb){
  fs.readFile(config.state_file, function(err, data){
    if(err){
      fs.writeFile(config.state_file, '0', function(err){
        if(err) return recent_cb(err);
        recent_cb(null, 0);
      });
    } else {
      recent_cb(null, Number(data));
    }
  });
}

function github_process_prs(most_recent_pr_checked, pr_cb){
  var github = new GitHubApi();

  var securedrop = {
    user: config.github_user,
    repo: config.github_repo
  }

  github.authenticate({
    type: "oauth",
    token: config.github_token || process.env.GITHUB_TOKEN
  })

  // Label all PRs which meet the criteria for labelling
  function github_process_pr_page(first_page){
    return function(err, pull_requests){
      if(first_page){
        fs.writeFile(config.state_file, pull_requests[0].number, function(err){
          if(err) return pr_cb(err);
        });
      }

      var file_checks = [];
      _.each(pull_requests, function(pull_request){

        if(pull_request.number > most_recent_pr_checked){

          file_checks.push(function(fc_cb){
            github.pullRequests.getFiles(_.extend(securedrop, {
              number: pull_request.number
            }), function(err, files){
              if(err) return pr_cb(err);

              var labels = new Set();

              _.each(files, function(file){

                if(file.filename.match(/^securedrop\/journalist_templates\//) ||
                  file.filename.match(/^securedrop\/journalist.py$/)){
                  labels.add("journalist_interface");
                }

                if(file.filename.match(/^securedrop\/source_templates\//) ||
                  file.filename.match(/^securedrop\/source.py$/)){
                  labels.add("source_interface");
                }

                if(file.filename.match(/^docs\//)){
                  labels.add("docs");
                }

                if(file.filename.match(/^snap_ci\//)){
                  labels.add("snap_ci");
                }

                if(file.filename.match(/^spec_tests\//)){
                  labels.add("spec_tests");
                }

                if(file.filename.match(/^securedrop\/tests\//) ||
                  file.filename.match(/^securedrop\/source.py$/)){
                  labels.add("tests");
                }

                if(file.filename.match(/^securedrop\/tests\/functional\//)){
                  labels.add("functional_tests");
                }

              });

              fc_cb(null, {pr: pull_request.number, labels: labels});

            });
          });

        }
      });

      async.parallel(file_checks, function(err, results){
        _.each(results, function(result){
{'fraction_non_alphanumeric': 0.14080734647025062, 'fraction_numerical': 0.0011478859766596518, 'mean_word_length': 2.062683069712947, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2607308', 'n_tokens_mistral': 1606, 'n_tokens_neox': 1489, 'n_words': 238}