<commit_before>/**
 * 
 */
package com.googlecode.goclipse.editors;

import org.eclipse.jface.text.TextAttribute;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.IWordDetector;
import org.eclipse.jface.text.rules.Token;
import org.eclipse.jface.text.rules.WordRule;
import org.eclipse.swt.SWT;

/**
 * @author eko
 *
 * This is a wordrule that automatically has all of the Go keywords
 */
public class GoSourceWordScanner extends WordRule {

	public GoSourceWordScanner() {
		super(new WordDetector(), new Token(new TextAttribute(IGoSourceColorConstants.DEFAULT)));

		addKeyWords();
	}

	private void addKeyWords()
	{
		IToken keyword = new Token(new TextAttribute(IGoSourceColorConstants.KEYWORD, null, SWT.BOLD, null));
		IToken type = new Token(new TextAttribute(IGoSourceColorConstants.TYPE, null, SWT.BOLD, null));

		// Key words
		//   break        default      func         interface    select
		//   case         defer        go           map          struct
		//   chan         else         goto         package      switch
		//   const        fallthrough  if           range        type
		//   continue     for          import       return       var
		addWord("break",       keyword);
		addWord("case",        keyword);
		addWord("chan",        keyword);
		addWord("const",       keyword);
		addWord("continue",    keyword);
		addWord("default",     keyword);
		addWord("defer",       keyword);
		addWord("else",        keyword);
		addWord("fallthrough", keyword);
		addWord("for",         keyword);
		addWord("func",        keyword);
		addWord("go",          keyword);
		addWord("goto",        keyword);
		addWord("if",          keyword);
		addWord("import",      keyword);
		addWord("interface",   keyword);
		addWord("map",         keyword);
		addWord("package",     keyword);
		addWord("range",       keyword);
		addWord("return",      keyword);
		addWord("select",      keyword);
		addWord("struct",      keyword);
		addWord("switch",      keyword);
		addWord("type",        keyword);
		addWord("var",         keyword);
		addWord("iota",        keyword);

		// Types
		//   uint8    int8
		//   uint16   int16   
		//   uint32   int32   float32   complex64
		//   uint64   int64   float64   complex128  byte
		//   uint     int     float     complex     uintptr   string
		addWord("byte",       type);  
		addWord("uint",       type); 
		addWord("uint8",      type); 
		addWord("uint16",     type); 
		addWord("uint32",     type); 
		addWord("uint64",     type);
		addWord("uintptr",    type);
		addWord("int",        type);
		addWord("int8",       type);
		addWord("int16",      type);
		addWord("int32",      type);
		addWord("int64",      type);
		addWord("float",      type);
		addWord("float32",    type);
		addWord("float64",    type);<commit_msg>Added "nil" as a known reserved word<commit_after>/**
 * 
 */
package com.googlecode.goclipse.editors;

import org.eclipse.jface.text.TextAttribute;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.IWordDetector;
import org.eclipse.jface.text.rules.Token;
import org.eclipse.jface.text.rules.WordRule;
import org.eclipse.swt.SWT;

/**
 * @author eko
 *
 * This is a wordrule that automatically has all of the Go keywords
 */
public class GoSourceWordScanner extends WordRule {

	public GoSourceWordScanner() {
		super(new WordDetector(), new Token(new TextAttribute(IGoSourceColorConstants.DEFAULT)));

		addKeyWords();
	}

	private void addKeyWords()
	{
		IToken keyword = new Token(new TextAttribute(IGoSourceColorConstants.KEYWORD, null, SWT.BOLD, null));
		IToken type = new Token(new TextAttribute(IGoSourceColorConstants.TYPE, null, SWT.BOLD, null));

		// Key words
		//   break        default      func         interface    select
		//   case         defer        go           map          struct
		//   chan         else         goto         package      switch
		//   const        fallthrough  if           range        type
		//   continue     for          import       return       var
		addWord("break",       keyword);
		addWord("case",        keyword);
		addWord("chan",        keyword);
		addWord("const",       keyword);
		addWord("continue",    keyword);
		addWord("default",     keyword);
		addWord("defer",       keyword);
		addWord("else",        keyword);
		addWord("fallthrough", keyword);
		addWord("for",         keyword);
		addWord("func",        keyword);
		addWord("go",          keyword);
		addWord("goto",        keyword);
		addWord("if",          keyword);
		addWord("import",      keyword);
		addWord("interface",   keyword);
		addWord("map",         keyword);
		addWord("package",     keyword);
		addWord("range",       keyword);
		addWord("return",      keyword);
		addWord("select",      keyword);
		addWord("struct",      keyword);
		addWord("switch",      keyword);
		addWord("type",        keyword);
		addWord("var",         keyword);
		addWord("iota",        keyword);
		addWord("nil",         keyword);

		// Types
		//   uint8    int8
		//   uint16   int16   
		//   uint32   int32   float32   complex64
		//   uint64   int64   float64   complex128  byte
		//   uint     int     float     complex     uintptr   string
		addWord("byte",       type);  
		addWord("uint",       type); 
		addWord("uint8",      type); 
		addWord("uint16",     type); 
		addWord("uint32",     type); 
		addWord("uint64",     type);
		addWord("uintptr",    type);
		addWord("int",        type);
		addWord("int8",       type);
		addWord("int16",      type);
		addWord("int32",      type);
		addWord("int64",      type);
		addWord("float",      type);
		addWord("float32",    type);
		addWord("float64",    type);
{'fraction_non_alphanumeric': 0.13175556340515718, 'fraction_numerical': 0.014482515012363122, 'mean_word_length': 2.3568464730290457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5396845', 'n_tokens_mistral': 1886, 'n_tokens_neox': 1700, 'n_words': 387}