<commit_before>var tokTypes = require("babel").acorn.tokTypes;
var traverse = require("babel").traverse;
var t        = require("babel").types;

exports.toToken = function (token) {
  var type = token.type;

  if (type === tokTypes.name) {
    token.type = "Identifier";
  } else if (type === tokTypes.semi || type === tokTypes.comma || type === tokTypes.parenL || type === tokTypes.parenR || type === tokTypes.braceL || type === tokTypes.braceR || type.isAssign) {
    token.type = "Punctuator";
    if (!token.value) {
      token.value = type.type;
    }
  } else if (type.keyword) {
    token.type = "Keyword";
  } else if (type === tokTypes.num) {
    token.type = "Numeric";
    token.value = String(token.value);
  }

  return token;
};

exports.toAST = function (ast) {
  traverse(ast, astTransformVisitor);
};

function isCompatTag(tagName) {
  return tagName && /^[a-z]|\-/.test(tagName);
}

var astTransformVisitor = {
  noScope: true,
  enter: function (node, parent) {
    if (t.isSpreadProperty(node)) {
      node.type = "Property";
      node.kind = "init";
      node.computed = true;
      node.key = node.value = node.argument;
      delete node.argument;
    }

    if (t.isRestElement(node)) {
      return node.argument;
    }

    if (t.isImportBatchSpecifier(node)) {
      // ImportBatchSpecifier<name> => ImportNamespaceSpecifier<id>
      node.type = "ImportNamespaceSpecifier";
      node.id = node.name;
      delete node.name;
    }

    if (t.isAwaitExpression(node)) {
      node.type = "YieldExpression";
    }

    // classes
    
    if (t.isReferencedIdentifier(node, parent, { name: "super" })) {
      return t.inherits(t.thisExpression(), node);
    }

    if (t.isClassProperty(node)) {
      // eslint doesn't like these
      this.remove();
    }

    // JSX

    if (t.isJSXIdentifier(node)) {
      if (node.name === "this" && t.isReferenced(node, parent)) {
        return t.inherits(t.thisExpression(), node);
      } else if (!t.isJSXAttribute(parent) && !isCompatTag(node.name)) {
        node.type = "Identifier";
      } else {
        // just ignore this node as it'd be something like <div> or an attribute name
      }
    }
<commit_msg>Fix class declaration and class expressions<commit_after>var tokTypes = require("babel").acorn.tokTypes;
var traverse = require("babel").traverse;
var t        = require("babel").types;

exports.toToken = function (token) {
  var type = token.type;

  if (type === tokTypes.name) {
    token.type = "Identifier";
  } else if (type === tokTypes.semi || type === tokTypes.comma || type === tokTypes.parenL || type === tokTypes.parenR || type === tokTypes.braceL || type === tokTypes.braceR || type.isAssign) {
    token.type = "Punctuator";
    if (!token.value) {
      token.value = type.type;
    }
  } else if (type.keyword) {
    token.type = "Keyword";
  } else if (type === tokTypes.num) {
    token.type = "Numeric";
    token.value = String(token.value);
  }

  return token;
};

exports.toAST = function (ast) {
  traverse(ast, astTransformVisitor);
};

function isCompatTag(tagName) {
  return tagName && /^[a-z]|\-/.test(tagName);
}

var astTransformVisitor = {
  noScope: true,
  enter: function (node, parent) {
    if (t.isSpreadProperty(node)) {
      node.type = "Property";
      node.kind = "init";
      node.computed = true;
      node.key = node.value = node.argument;
      delete node.argument;
    }

    if (t.isRestElement(node)) {
      return node.argument;
    }

    if (t.isImportBatchSpecifier(node)) {
      // ImportBatchSpecifier<name> => ImportNamespaceSpecifier<id>
      node.type = "ImportNamespaceSpecifier";
      node.id = node.name;
      delete node.name;
    }

    if (t.isAwaitExpression(node)) {
      node.type = "YieldExpression";
    }

    // classes

    if (t.isClassDeclaration(node) || t.isClassExpression(node)) {
      node.name = node.id;
      delete node.id;
    }

    if (t.isReferencedIdentifier(node, parent, { name: "super" })) {
      return t.inherits(t.thisExpression(), node);
    }

    if (t.isClassProperty(node)) {
      // eslint doesn't like these
      this.remove();
    }

    // JSX

    if (t.isJSXIdentifier(node)) {
      if (node.name === "this" && t.isReferenced(node, parent)) {
        return t.inherits(t.thisExpression(), node);
      } else if (!t.isJSXAttribute(parent) && !isCompatTag(node.name)) {
        node.type = "Identifier";
      } else {
        // just ignore this node as it'd be something like <div> or an attribute name
      }
    }

{'fraction_non_alphanumeric': 0.15632589975712077, 'fraction_numerical': 0.0, 'mean_word_length': 2.728395061728395, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1158507', 'n_tokens_mistral': 1512, 'n_tokens_neox': 1479, 'n_words': 318}