<commit_before>// ----------------------------------------------------------------------
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
// --
// Copyright 2016-2018 <NAME> <https://andidittrich.de>
// ----------------------------------------------------------------------

// Generic Rules/Regex
import _language_common_rules from '../engine/generic-rules';
import {generic} from './generic';

import _token from '../engine/token';
import _microTokenizer from '../engine/micro-tokenizer';

// C# Language
// Author: [<NAME>]
// --
export class csharp extends generic {

    // language aliases
    static alias(){
        return ['c#'];
    }

    setupLanguage() {

        // template strings. Stage-2 Analyzing
        function parseInterpolatedStrings(token){

            // run the MicroTokenizer to identify the template tags
            return _microTokenizer(token, /\{.*?}/g, function(match){
                return [_token(match[0], 's3')];
            });
        }

        this.rules = [

            // strings, chars
            {
                regex: /\$("(?:[^"\\]|\\.)*")/g,
                type: 's0',
                filter: parseInterpolatedStrings
            },

            _language_common_rules.dqStrings,
            _language_common_rules.char,

            // boolean, null
            _language_common_rules.null,
            _language_common_rules.boolean,

            // properties
            _language_common_rules.prop,

            // types
            {
                regex: /\b(bool|byte|char|decimal|double|float|int|long|sbyte|short|uint|ulong|ushort|void)\b/g,
                type: 'k5'
            },

            // control keywords
            {
                regex: /\b(while|try|throw|switch|if|goto|foreach|for|finally|else|do|continue|catch|case|break)\b/g,
                type: 'k1'
            },

            // package directives
            {
                regex: /^((?:using|namespace)\s+)(\w[\w._]+[;{\n])/gm,
                type: ['k0', 'k10']
            },

            // type initialization
            {
                regex: /\b(enum|struct|var)\b/g,
                type: 'k2'
            },

            // qualifier/modifier
            {
                regex: /\b(const|in|out)\b/g,
                type: 'k8'
            },

            // keywords
            {
                regex: /\b(using|volatile|virtual|using|unsafe|unchecked|static|stackalloc|sealed|return|ref|readonly|public|protected|private|params|override|operator|object|namespace|lock|is|internal|interface|implicit|fixed|extern|explicit|event|delegate|default|class|checked|base|as|abstract)\b/g,
                type: 'k0'
            },

            // special inheritance
            {
                regex: /\b(this)\b/g,
                type: 'k9'
            },

            // operator
            {
                regex: /\b(new|sizeof|typeof)\b/g,
                type: 'k3'
            },<commit_msg>Update C# to include contextual keywords<commit_after>// ----------------------------------------------------------------------
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
// --
// Copyright 2016-2018 <NAME> <https://andidittrich.de>
// ----------------------------------------------------------------------

// Generic Rules/Regex
import _language_common_rules from '../engine/generic-rules';
import {generic} from './generic';

import _token from '../engine/token';
import _microTokenizer from '../engine/micro-tokenizer';

// C# Language
// Author: [<NAME>]
// --
export class csharp extends generic {

    // language aliases
    static alias(){
        return ['c#'];
    }

    setupLanguage() {

        // template strings. Stage-2 Analyzing
        function parseInterpolatedStrings(token){

            // run the MicroTokenizer to identify the template tags
            return _microTokenizer(token, /\{.*?}/g, function(match){
                return [_token(match[0], 's3')];
            });
        }

        this.rules = [

            // strings, chars
            {
                regex: /\$("(?:[^"\\]|\\.)*")/g,
                type: 's0',
                filter: parseInterpolatedStrings
            },

            _language_common_rules.dqStrings,
            _language_common_rules.char,

            // boolean, null
            _language_common_rules.null,
            _language_common_rules.boolean,

            // properties
            _language_common_rules.prop,

            // types
            {
                regex: /\b(bool|byte|char|decimal|double|float|int|long|sbyte|short|uint|ulong|ushort|void)\b/g,
                type: 'k5'
            },

            // control keywords
            {
                regex: /\b(while|try|throw|switch|if|goto|foreach|for|finally|else|do|continue|catch|case|break)\b/g,
                type: 'k1'
            },

            // package directives
            {
                regex: /^((?:using|namespace)\s+)(\w[\w._]+[;{\n])/gm,
                type: ['k0', 'k10']
            },

            // type initialization
            {
                regex: /\b(enum|struct|var)\b/g,
                type: 'k2'
            },

            // qualifier/modifier
            {
                regex: /\b(const|in|out)\b/g,
                type: 'k8'
            },

            // keywords
            {
                regex: /\b(using|volatile|virtual|using|unsafe|unchecked|static|stackalloc|sealed|return|ref|readonly|public|protected|private|params|override|operator|object|namespace|lock|is|internal|interface|implicit|fixed|extern|explicit|event|delegate|default|class|checked|base|as|abstract)\b/g,
                type: 'k0'
            },

            // contextual keywords
            {
                regex: /\b(add|alias|ascending|async|await|by|descending|dynamic|equals|from|get|global|group|into|join|let|nameof|on|orderby|partial|remove|select|set|unmanaged|value|var|when|where|yield)\b/g,
                type: 'k0'
            },

            // special inheritance
            {
                regex: /\b(this)\b/g,
                type: 'k9'
            },

            // operator
            {
                regex: /\b(new|sizeof|typeof)\b/g,
                type: 'k3'
            },
{'fraction_non_alphanumeric': 0.17651538227231192, 'fraction_numerical': 0.008071885470606152, 'mean_word_length': 1.388, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3349017', 'n_tokens_mistral': 1903, 'n_tokens_neox': 1736, 'n_words': 323}