<commit_before>/*
 * Created on Nov 12, 2004
 *
 * @author <NAME>
 */
package org.python.pydev.editor.codecompletion.revisited.modules;

import org.python.pydev.editor.codecompletion.PyCodeCompletion;
import org.python.pydev.editor.codecompletion.revisited.AbstractToken;
import org.python.pydev.editor.codecompletion.revisited.visitors.AbstractVisitor;
import org.python.pydev.parser.jython.SimpleNode;
import org.python.pydev.parser.jython.ast.Attribute;
import org.python.pydev.parser.jython.ast.ClassDef;
import org.python.pydev.parser.jython.ast.FunctionDef;
import org.python.pydev.parser.jython.ast.Import;
import org.python.pydev.parser.jython.ast.ImportFrom;
import org.python.pydev.parser.jython.ast.Name;
import org.python.pydev.parser.jython.ast.NameTok;
import org.python.pydev.parser.jython.ast.keywordType;
import org.python.pydev.parser.visitors.NodeUtils;

/**
 * @author <NAME>
 */
public class SourceToken extends AbstractToken{

    /**
     * 
     */
    private static final long serialVersionUID = 1L;
    private SimpleNode ast;

    /**
     * @param node
     */
    public SourceToken(SimpleNode node, String rep, String args, String doc, String parentPackage) {
        super(rep, doc, args, parentPackage, getType(node));
        this.ast = node;
    }

    /**
     * @param node
     */
    public SourceToken(SimpleNode node, String rep, String args, String doc, String parentPackage, int type) {
        super(rep, doc, args, parentPackage, type);
        this.ast = node;
    }

    /**
     * @param node<commit_msg>*** empty log message ***<commit_after>/*
 * Created on Nov 12, 2004
 *
 * @author <NAME>
 */
package org.python.pydev.editor.codecompletion.revisited.modules;

import org.python.pydev.editor.codecompletion.PyCodeCompletion;
import org.python.pydev.editor.codecompletion.revisited.AbstractToken;
import org.python.pydev.editor.codecompletion.revisited.visitors.AbstractVisitor;
import org.python.pydev.parser.jython.SimpleNode;
import org.python.pydev.parser.jython.ast.Attribute;
import org.python.pydev.parser.jython.ast.ClassDef;
import org.python.pydev.parser.jython.ast.FunctionDef;
import org.python.pydev.parser.jython.ast.Import;
import org.python.pydev.parser.jython.ast.ImportFrom;
import org.python.pydev.parser.jython.ast.Name;
import org.python.pydev.parser.jython.ast.NameTok;
import org.python.pydev.parser.jython.ast.keywordType;
import org.python.pydev.parser.visitors.NodeUtils;

/**
 * @author <NAME>
 */
public class SourceToken extends AbstractToken{

    private static final long serialVersionUID = 1L;
    
    /**
     * The AST that generated this SourceToken
     */
    private SimpleNode ast;

    /**
     * @param node
     */
    public SourceToken(SimpleNode node, String rep, String args, String doc, String parentPackage) {
        super(rep, doc, args, parentPackage, getType(node));
        this.ast = node;
    }

    /**
     * @param node
     */
    public SourceToken(SimpleNode node, String rep, String args, String doc, String parentPackage, int type) {
        super(rep, doc, args, parentPackage, type);
        this.ast = node;
    }

    /**
     * @param node
{'fraction_non_alphanumeric': 0.12025316455696203, 'fraction_numerical': 0.004430379746835443, 'mean_word_length': 3.6145985401459853, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '684837', 'n_tokens_mistral': 1032, 'n_tokens_neox': 999, 'n_words': 201}