<commit_before>var path = require("path"),
  fs = require("fs"),
  _ = require("lodash"),
  acorn = require("acorn"),
  walk = require("acorn/dist/walk"),
  settings = require("./settings");

var filterByTags = function(files, tags) {
  // Tidy up tag input. If we have a comma-delimited list, tokenize and clean it up
  if (typeof tags === "string") {
    tags = tags.trim();

    if (tags.indexOf(",") > -1) {
      tags = tags.trim().split(",").map(function (tag) {
        return tag.trim();
      });
    } else {
      tags = [tags];
    }
  }

  // If tags are empty or malformed, ignore them
  if (!_.isArray(tags) || tags.length === 0) {
    return files;
  }

  console.log("Using tag filter: ", tags);

  return files.filter(function (f) {
    // Filter by tag (or tag list):
    //
    // Parse the syntax tree of each included test and search for a property
    // definition with the name "tags" with an array expression attached to it,
    // i.e. we're looking for source code in the following form:
    //
    //    tags: [ value, value, value ]
    //
    // Match each f in files against the tag list we have in the array tags.
    //
    var foundTags = false;
    var pass = false;
    var root = acorn.parse(fs.readFileSync(path.resolve(f)));

    walk.findNodeAt(root, null, null, function (nodeType, node) {
      if (!foundTags && nodeType === "Property") {
        if (node.key && node.key.name === "tags" && node.value && node.value.type === "ArrayExpression" && node.value.elements) {
          foundTags = true;
          node.value.elements.forEach(function (tagNode) {
            if (tagNode.value && typeof tagNode.value === "string") {
              if (_.indexOf(tags, tagNode.value.trim()) > -1) {
                pass = true;
              }
            }
          });
        }
      }
    });

    return pass;
  });
};<commit_msg>Supports es6 tests, resolves #40<commit_after>var path = require("path"),
  fs = require("fs"),
  _ = require("lodash"),
  acorn = require("acorn"),
  clc = require("cli-color"),
  walk = require("acorn/dist/walk"),
  settings = require("./settings");

var filterByTags = function(files, tags) {
  // Tidy up tag input. If we have a comma-delimited list, tokenize and clean it up
  if (typeof tags === "string") {
    tags = tags.trim();

    if (tags.indexOf(",") > -1) {
      tags = tags.trim().split(",").map(function (tag) {
        return tag.trim();
      });
    } else {
      tags = [tags];
    }
  }

  // If tags are empty or malformed, ignore them
  if (!_.isArray(tags) || tags.length === 0) {
    return files;
  }

  console.log("Using tag filter: ", tags);

  return files.filter(function (f) {
    // Filter by tag (or tag list):
    //
    // Parse the syntax tree of each included test and search for a property
    // definition with the name "tags" with an array expression attached to it,
    // i.e. we're looking for source code in the following form:
    //
    //    tags: [ value, value, value ]
    //
    // Match each f in files against the tag list we have in the array tags.
    //
    var foundTags = false;
    var pass = false;
    var filename = path.resolve(f);
    var root;
    try {
      root = acorn.parse(fs.readFileSync(filename), {
        ecmaVersion: 6
      });
    } catch (err) {
      console.log(clc.redBright("Syntax error in parsing " + filename));
      throw err;
    }

    walk.findNodeAt(root, null, null, function (nodeType, node) {
      if (!foundTags && nodeType === "Property") {
        if (node.key && node.key.name === "tags" && node.value && node.value.type === "ArrayExpression" && node.value.elements) {
          foundTags = true;
          node.value.elements.forEach(function (tagNode) {
            if (tagNode.value && typeof tagNode.value === "string") {
              if (_.indexOf(tags, tagNode.value.trim()) > -1) {
                pass = true;
              }
            }
          });
        }
      }
    });

    return pass;
  });
};
{'fraction_non_alphanumeric': 0.15355711422845691, 'fraction_numerical': 0.00250501002004008, 'mean_word_length': 2.189297124600639, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '613015', 'n_tokens_mistral': 1235, 'n_tokens_neox': 1213, 'n_words': 368}