<commit_before># Copyright 2016 The ci_edit Authors. All rights reserved.
# Use of this source code is governed by an Apache-style license that can be
# found in the LICENSE file.

import app.log
import curses
import re
import sys
import time
import traceback

importStartTime = time.time()

commentColorIndex = 2
defaultColorIndex = 0
foundColorIndex = 32
keywordsColorIndex = 21
selectedColor = 64 # Active find is a selection.
specialsColorIndex = 224
stringColorIndex = 5

def joinReList(reList):
  return r"("+r")|(".join(reList)+r")"

def joinReWordList(reList):
  return r"(\b"+r"\b)|(\b".join(reList)+r"\b)"

__common_keywords = [
  'break', 'continue', 'do', 'else',
  'for', 'if', 'return', 'while',
]

__c_keywords = __common_keywords + [
  'case', 'const',
  'sizeof', 'static', 'struct', 'switch',
  'typedef',
]

__c_primitive_types = [
  'char', 'double', 'float', 'int', 'long', 'signed', 'short', 'unsigned',
  'int8_t', 'int16_t', 'int32_t', 'int64_t',
  'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t',
  'void', 'wchar_t',
]

# Trivia: all English contractions except 'sup, 'tis and 'twas will
# match this regex (with re.I):  [adegIlnotuwy]'[acdmlsrtv]
# The prefix part of that is used in the expression below to identify
# English contractions.
__english_contraction = \
    r"(\"(\\\"|[^\"])*?\")|(?<![adegIlnotuwy])('(\\\'|[^'])*?')"

__special_string_escapes = [
  r'\\\\', r'\\b', r'\\f', r'\\n', r'\\r', r'\\t', r'\\v', r'\\0[0-7]{0,3}',
]

__common_numbers = [
  r'[-+]?[0-9]*\.[0-9]+(?:[eE][+-][0-9]+)?[fF]?(?!\w)',
  r'[-+]?[0-9]+(?:\.[0-9]*(?:[eE][+-][0-9]+)?)?[fF]?(?!\w)',
  r'[-+]?[0-9]+(?:[uUlL][lL]?[lL]?)?(?!\w)',
  r'0[xX][^A-Fa-f0-9]+(?:[uUlL][lL]?[lL]?)?(?!\w)',
]

numbersRe = re.compile(joinReList(__common_numbers))

def numberTest(str, expectRegs):
  sre = numbersRe.search(str)
  if sre:
    app.log.startup('%16s %16s %-16s %s ' % (
        str, expectRegs, sre.regs[0], sre.groups()))
  else:
    app.log.startup('%16s %16s %-16s ' % (str, expectRegs, sre))


numberTest('.', None)
numberTest('2', (0, 1))
numberTest(' 2 ', (1, 2))
numberTest('242.2', (0, 5))
numberTest('.2', (0, 2))
numberTest('2.', (0, 2))
numberTest('.2a', None)
numberTest('2.a', (0, 1))
numberTest('+0.2e-15', (0, 8))
numberTest('02factor', None)
numberTest('02f', (0, 3))

# These prefs are not fully working.
prefs = {
  'colors': {
    'default': defaultColorIndex,
    'text': defaultColorIndex,
    'keywords': keywordsColorIndex,
    'specials': specialsColorIndex,
    'c': defaultColorIndex,
    'cpp_block_comment': commentColorIndex,
    'cpp_line_comment': commentColorIndex,
    'c_preprocessor': 1,
    'c_raw_string1': stringColorIndex,
    'c_raw_string2': stringColorIndex,
    'c_string1': stringColorIndex,
    'c_string2': stringColorIndex,
    'doc_block_comment': commentColorIndex,
    'html_block_comment': commentColorIndex,
    'pound_comment': commentColorIndex,
    'py_raw_string1': stringColorIndex,
    'py_raw_string2': stringColorIndex,
    'py_string1': stringColorIndex,
    'py_string2': stringColorIndex,
    'quoted_string2': stringColorIndex,
  },
  'filetype': {
    'bash': {
      'ext': ['.bash', '.sh'],
      'grammar': 'bash',
    },
    'binary': {
      'ext': [
        '.exe', '.gz', '.gzip', '.jar', '.jpg', '.jpeg', '.o', '.obj',
        '.png', '.pyc', '.pyo', '.tgz', '.tiff', '.zip',
      ],
      'grammar': 'binary',
    },
    'c': {
      'ext': ['.c', '.h'],
      'grammar': 'c',
    },
    'cpp': {
      'ext': ['.cc', '.cpp', '.cxx', '.c++', '.hpp', '.hxx', '.h++', '.inc'],
      'grammar': 'cpp',
    },
    'css': {
      'ext': ['.css', '_css.html'],
      'grammar': 'css',
    },
    'html': {
      'ext': ['.htm', '.html'],
      'grammar': 'html',
    },
    'java': {
      'ext': ['.java',],
      'grammar': 'java',
    },
    'js': {
      'ext': ['.json', '.js'],
      'grammar': 'js',
    },
    'python': {
      'ext': ['.py'],
      'grammar': 'py',
    },
  },
  'grammar': {
    # A grammar is
    # 'grammar_name': {
    #   'begin': None or regex,
    #   'continued': None or string,
    #       Prefixed used when continuing to another line,
    #   'end': None or regex,
    #   'escaped': None or regex,
    #   'indent': None or string,
    #   'keywords': None or list of string,
    #   'single_line': Boolean, Whether entire grammar must be on a single line,
    #   'type': text or binary. default: text.
    #   'contains': other grammars that may be contained within this grammar.
    # }
    'bash': {
      'indent': '  ',
      'keywords': [
        'break', 'case', 'continue', 'do', 'done', 'exit', 'fi', 'if', 'for',
        'return', 'switch', 'then', 'while',
      ],
      'contains': ['c_string1', 'c_string2', 'pound_comment'],
    },
    'binary': {
      'type': 'binary',
    },
    'c': {
      'indent': '  ',
      'keywords': __c_keywords,
      'types': __c_primitive_types,
      'contains': ['cpp_block_comment', 'cpp_line_comment', 'c_preprocessor',
        'c_string1', 'c_string2'],
    },
    'cpp': {
      'indent': '  ',
      'keywords': __c_keywords + [
        'auto', 'catch', 'class', 'constexpr',
        'namespace', 'nullptr',
        'private', 'protected', 'public',
        'template', 'this', 'throw', 'typename',
      ],
      'namespaces': [
        '::', 'std::',
      ],
      'types': __c_primitive_types,
      'contains': ['cpp_block_comment', 'cpp_line_comment', 'c_preprocessor',
        'c_string1', 'c_string2'],
    },
    'cpp_block_comment': {
      'begin': r'/\*',
      'continued': ' * ',
      'end': r'\*/',
      'indent': '  ',
      'keywords': [],
      'special': [<commit_msg>add keywords<commit_after># Copyright 2016 The ci_edit Authors. All rights reserved.
# Use of this source code is governed by an Apache-style license that can be
# found in the LICENSE file.

import app.log
import curses
import re
import sys
import time
import traceback

importStartTime = time.time()

commentColorIndex = 2
defaultColorIndex = 0
foundColorIndex = 32
keywordsColorIndex = 21
selectedColor = 64 # Active find is a selection.
specialsColorIndex = 224
stringColorIndex = 5

def joinReList(reList):
  return r"("+r")|(".join(reList)+r")"

def joinReWordList(reList):
  return r"(\b"+r"\b)|(\b".join(reList)+r"\b)"

__common_keywords = [
  'break', 'continue', 'do', 'else',
  'for', 'if', 'return', 'while',
]

__c_keywords = __common_keywords + [
  'case', 'const', 'default',
  'sizeof', 'static', 'struct', 'switch',
  'typedef',
]

__c_primitive_types = [
  'bool', 'char', 'double', 'float', 'int', 'long', 'signed', 'short',
  'unsigned',
  'int8_t', 'int16_t', 'int32_t', 'int64_t',
  'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t',
  'void', 'wchar_t',
]

# Trivia: all English contractions except 'sup, 'tis and 'twas will
# match this regex (with re.I):  [adegIlnotuwy]'[acdmlsrtv]
# The prefix part of that is used in the expression below to identify
# English contractions.
__english_contraction = \
    r"(\"(\\\"|[^\"])*?\")|(?<![adegIlnotuwy])('(\\\'|[^'])*?')"

__special_string_escapes = [
  r'\\\\', r'\\b', r'\\f', r'\\n', r'\\r', r'\\t', r'\\v', r'\\0[0-7]{0,3}',
]

__common_numbers = [
  r'[-+]?[0-9]*\.[0-9]+(?:[eE][+-][0-9]+)?[fF]?(?!\w)',
  r'[-+]?[0-9]+(?:\.[0-9]*(?:[eE][+-][0-9]+)?)?[fF]?(?!\w)',
  r'[-+]?[0-9]+(?:[uUlL][lL]?[lL]?)?(?!\w)',
  r'0[xX][^A-Fa-f0-9]+(?:[uUlL][lL]?[lL]?)?(?!\w)',
]

numbersRe = re.compile(joinReList(__common_numbers))

def numberTest(str, expectRegs):
  sre = numbersRe.search(str)
  if sre:
    app.log.startup('%16s %16s %-16s %s ' % (
        str, expectRegs, sre.regs[0], sre.groups()))
  else:
    app.log.startup('%16s %16s %-16s ' % (str, expectRegs, sre))


numberTest('.', None)
numberTest('2', (0, 1))
numberTest(' 2 ', (1, 2))
numberTest('242.2', (0, 5))
numberTest('.2', (0, 2))
numberTest('2.', (0, 2))
numberTest('.2a', None)
numberTest('2.a', (0, 1))
numberTest('+0.2e-15', (0, 8))
numberTest('02factor', None)
numberTest('02f', (0, 3))

# These prefs are not fully working.
prefs = {
  'colors': {
    'default': defaultColorIndex,
    'text': defaultColorIndex,
    'keywords': keywordsColorIndex,
    'specials': specialsColorIndex,
    'c': defaultColorIndex,
    'cpp_block_comment': commentColorIndex,
    'cpp_line_comment': commentColorIndex,
    'c_preprocessor': 1,
    'c_raw_string1': stringColorIndex,
    'c_raw_string2': stringColorIndex,
    'c_string1': stringColorIndex,
    'c_string2': stringColorIndex,
    'doc_block_comment': commentColorIndex,
    'html_block_comment': commentColorIndex,
    'pound_comment': commentColorIndex,
    'py_raw_string1': stringColorIndex,
    'py_raw_string2': stringColorIndex,
    'py_string1': stringColorIndex,
    'py_string2': stringColorIndex,
    'quoted_string2': stringColorIndex,
  },
  'filetype': {
    'bash': {
      'ext': ['.bash', '.sh'],
      'grammar': 'bash',
    },
    'binary': {
      'ext': [
        '.exe', '.gz', '.gzip', '.jar', '.jpg', '.jpeg', '.o', '.obj',
        '.png', '.pyc', '.pyo', '.tgz', '.tiff', '.zip',
      ],
      'grammar': 'binary',
    },
    'c': {
      'ext': ['.c', '.h'],
      'grammar': 'c',
    },
    'cpp': {
      'ext': ['.cc', '.cpp', '.cxx', '.c++', '.hpp', '.hxx', '.h++', '.inc'],
      'grammar': 'cpp',
    },
    'css': {
      'ext': ['.css', '_css.html'],
      'grammar': 'css',
    },
    'html': {
      'ext': ['.htm', '.html'],
      'grammar': 'html',
    },
    'java': {
      'ext': ['.java',],
      'grammar': 'java',
    },
    'js': {
      'ext': ['.json', '.js'],
      'grammar': 'js',
    },
    'python': {
      'ext': ['.py'],
      'grammar': 'py',
    },
  },
  'grammar': {
    # A grammar is
    # 'grammar_name': {
    #   'begin': None or regex,
    #   'continued': None or string,
    #       Prefixed used when continuing to another line,
    #   'end': None or regex,
    #   'escaped': None or regex,
    #   'indent': None or string,
    #   'keywords': None or list of string,
    #   'single_line': Boolean, Whether entire grammar must be on a single line,
    #   'type': text or binary. default: text.
    #   'contains': other grammars that may be contained within this grammar.
    # }
    'bash': {
      'indent': '  ',
      'keywords': [
        'break', 'case', 'continue', 'do', 'done', 'exit', 'fi', 'if', 'for',
        'return', 'switch', 'then', 'while',
      ],
      'contains': ['c_string1', 'c_string2', 'pound_comment'],
    },
    'binary': {
      'type': 'binary',
    },
    'c': {
      'indent': '  ',
      'keywords': __c_keywords,
      'types': __c_primitive_types,
      'contains': ['cpp_block_comment', 'cpp_line_comment', 'c_preprocessor',
        'c_string1', 'c_string2'],
    },
    'cpp': {
      'indent': '  ',
      'keywords': __c_keywords + [
        'auto', 'catch', 'class', 'constexpr', 'false',
        'namespace', 'nullptr',
        'private', 'protected', 'public',
        'template', 'this', 'throw', 'true', 'typename',
      ],
      'namespaces': [
        '::', 'std::',
      ],
      'types': __c_primitive_types,
      'contains': ['cpp_block_comment', 'cpp_line_comment', 'c_preprocessor',
        'c_string1', 'c_string2'],
    },
    'cpp_block_comment': {
      'begin': r'/\*',
      'continued': ' * ',
      'end': r'\*/',
      'indent': '  ',
      'keywords': [],
      'special': [
{'fraction_non_alphanumeric': 0.23078954317401637, 'fraction_numerical': 0.020244696769650558, 'mean_word_length': 3.07532281205165, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '446639', 'n_tokens_mistral': 4472, 'n_tokens_neox': 4310, 'n_words': 931}