<commit_before>/**
 * All reading related code here. This is so that we can separate the async code from sync code
 * for testability
 */
require.paths.push(__dirname);
var fs       = require('fs'),
    callback = require('callback');

var NEW_LINE = /\n\r?/;

function collect(callback){
   findJsFiles('src', callback.waitMany(function(file) {
     console.log('reading', file, '...');
     findNgDocInJsFile(file, callback.waitMany(function(doc, line) {
       callback('@section api\n' + doc, file, line);
    }));
  }));
  findNgDocInDir('docs/content', callback.waitMany(callback));
  callback.done();
}

function findJsFiles(dir, callback){
  fs.readdir(dir, callback.waitFor(function(err, files){
    if (err) return this.error(err);
    files.forEach(function(file){
      var path = dir + '/' + file;
      fs.lstat(path, callback.waitFor(function(err, stat){
        if (err) return this.error(err);
        if (stat.isDirectory())
          findJsFiles(path, callback.waitMany(callback));
        else if (/\.js$/.test(path))
          callback(path);
      }));
    });
    callback.done();
  }));
}

function findNgDocInDir(directory, docNotify) {
  fs.readdir(directory, docNotify.waitFor(function(err, files){
    if (err) return this.error(err);
    files.forEach(function(file){
      fs.stat(directory + '/' + file, docNotify.waitFor(function(err, stats){
        if (err) return this.error(err);
        if (stats.isFile()) {
          console.log('reading', directory + '/' + file, '...');
          fs.readFile(directory + '/' + file, docNotify.waitFor(function(err, content){
            if (err) return this.error(err);
            var section = '@section ' + directory.split('/').pop() + '\n';

            //TEMPORARY FIX to strip stuff from the docs until gdocs api is fixed
            var text = content.toString();

            text = text.replace('\ufeff', '');
            text = text.replace(/\r\n/mg, '\n');
            text = text.replace(/^ /mg, '  '); //for some reason gdocs drop first space for indented lines

            // strip out all text annotation comments
            text = text.replace(/^\[a\][\S\s]*/m, '');

            // strip out all text annotations
            text = text.replace(/\[\w{1,3}\]/mg, '');

            // fix smart-quotes
            text = text.replace(/[“”]/g, '"');
            text = text.replace(/[‘’]/g, "'");
            //TEMPORARY FIX END<commit_msg>ignore non .ngdoc files<commit_after>/**
 * All reading related code here. This is so that we can separate the async code from sync code
 * for testability
 */
require.paths.push(__dirname);
var fs       = require('fs'),
    callback = require('callback');

var NEW_LINE = /\n\r?/;

function collect(callback){
   findJsFiles('src', callback.waitMany(function(file) {
     console.log('reading', file, '...');
     findNgDocInJsFile(file, callback.waitMany(function(doc, line) {
       callback('@section api\n' + doc, file, line);
    }));
  }));
  findNgDocInDir('docs/content', callback.waitMany(callback));
  callback.done();
}

function findJsFiles(dir, callback){
  fs.readdir(dir, callback.waitFor(function(err, files){
    if (err) return this.error(err);
    files.forEach(function(file){
      var path = dir + '/' + file;
      fs.lstat(path, callback.waitFor(function(err, stat){
        if (err) return this.error(err);
        if (stat.isDirectory())
          findJsFiles(path, callback.waitMany(callback));
        else if (/\.js$/.test(path))
          callback(path);
      }));
    });
    callback.done();
  }));
}

function findNgDocInDir(directory, docNotify) {
  fs.readdir(directory, docNotify.waitFor(function(err, files){
    if (err) return this.error(err);
    files.forEach(function(file){
      fs.stat(directory + '/' + file, docNotify.waitFor(function(err, stats){
        if (err) return this.error(err);
        if (stats.isFile()) {
          if (!file.match(/\.ngdoc$/)) return;
          console.log('reading', directory + '/' + file, '...');
          fs.readFile(directory + '/' + file, docNotify.waitFor(function(err, content){
            if (err) return this.error(err);
            var section = '@section ' + directory.split('/').pop() + '\n';

            //TEMPORARY FIX to strip stuff from the docs until gdocs api is fixed
            var text = content.toString();

            text = text.replace('\ufeff', '');
            text = text.replace(/\r\n/mg, '\n');
            text = text.replace(/^ /mg, '  '); //for some reason gdocs drop first space for indented lines

            // strip out all text annotation comments
            text = text.replace(/^\[a\][\S\s]*/m, '');

            // strip out all text annotations
            text = text.replace(/\[\w{1,3}\]/mg, '');

            // fix smart-quotes
            text = text.replace(/[“”]/g, '"');
            text = text.replace(/[‘’]/g, "'");
            //TEMPORARY FIX END
{'fraction_non_alphanumeric': 0.1718527557453732, 'fraction_numerical': 0.0008135041692088672, 'mean_word_length': 2.4805378627034678, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '762876', 'n_tokens_mistral': 1509, 'n_tokens_neox': 1495, 'n_words': 351}