<commit_before>/// <reference path="./globals.d.ts" />

// This doesn't import all of CodeMirror, instead it only imports
// a small subset. This hack is brought to you by webpack and you
// can read all about it in webpack.common.js.
import CodeMirror from 'codemirror'

// This is a hack, some modes (looking at you markdown) uses
// CodeMirror.innerMode which isn't defined in the stripped down
// runmode. Luckily it's a simple, dependency free method so we'll
// just import it and stick it on the global CodeMirror object.
import { innerMode } from 'codemirror/src/modes'
const cm = CodeMirror as any
cm.innerMode = cm.innerMode || innerMode

import 'codemirror/mode/javascript/javascript'
import 'codemirror/mode/jsx/jsx'
import 'codemirror/mode/sass/sass'
import 'codemirror/mode/htmlmixed/htmlmixed'
import 'codemirror/mode/markdown/markdown'
import 'codemirror/mode/yaml/yaml'
import 'codemirror/mode/xml/xml'
import 'codemirror/mode/clike/clike'
import 'codemirror/mode/shell/shell'

interface IToken {
  length: number
  text: string
  token: string
}

type Tokens = {
  [line: number]: { [startIndex: number]: IToken }
}

const extensionMIMEMap = new Map<string, string>()

extensionMIMEMap.set('.ts', 'text/typescript')
extensionMIMEMap.set('.tsx', 'text/jsx')
extensionMIMEMap.set('.js', 'text/javascript')
extensionMIMEMap.set('.json', 'application/json')
extensionMIMEMap.set('.html', 'text/html')
extensionMIMEMap.set('.htm', 'text/html')
extensionMIMEMap.set('.markdown', 'text/x-markdown')
extensionMIMEMap.set('.md', 'text/x-markdown')
extensionMIMEMap.set('.css', 'text/css')
extensionMIMEMap.set('.scss', 'text/x-scss')
extensionMIMEMap.set('.less', 'text/x-less')
extensionMIMEMap.set('.yaml', 'text/yaml')
extensionMIMEMap.set('.yml', 'text/yaml')

extensionMIMEMap.set('.objc', 'text/x-objectivec')
extensionMIMEMap.set('.scala', 'text/x-scala')
extensionMIMEMap.set('.sc', 'text/x-scala')
extensionMIMEMap.set('.cs', 'text/x-csharp')
extensionMIMEMap.set('.java', 'text/x-java')
extensionMIMEMap.set('.c', 'text/x-c')
extensionMIMEMap.set('.h', 'text/x-c')
extensionMIMEMap.set('.cpp', 'text/x-c++src')

extensionMIMEMap.set('.xml', 'text/xml')
extensionMIMEMap.set('.xaml', 'text/xml')
extensionMIMEMap.set('.csproj', 'text/xml')

extensionMIMEMap.set('.sh', 'text/x-sh')
onmessage = (ev: MessageEvent) => {
  const startTime = performance ? performance.now() : null

  const tabSize: number = ev.data.tabSize
  const extension: string = ev.data.extension
  const contents: string = ev.data.contents
  const requestedLines: Array<number> | undefined = ev.data.lines

  const mimeType = extensionMIMEMap.get(extension)

  if (!mimeType) {
    console.debug(
      `Could not map '${extension}' to supported mime type for highlighting`
    )
    postMessage({})
    return
  }

  const mode: CodeMirror.Mode<{}> = CodeMirror.getMode({ tabSize }, mimeType)<commit_msg>Add go, perl, php, and python highlighting<commit_after>/// <reference path="./globals.d.ts" />

// This doesn't import all of CodeMirror, instead it only imports
// a small subset. This hack is brought to you by webpack and you
// can read all about it in webpack.common.js.
import CodeMirror from 'codemirror'

// This is a hack, some modes (looking at you markdown) uses
// CodeMirror.innerMode which isn't defined in the stripped down
// runmode. Luckily it's a simple, dependency free method so we'll
// just import it and stick it on the global CodeMirror object.
import { innerMode } from 'codemirror/src/modes'
const cm = CodeMirror as any
cm.innerMode = cm.innerMode || innerMode

import 'codemirror/mode/javascript/javascript'
import 'codemirror/mode/jsx/jsx'
import 'codemirror/mode/sass/sass'
import 'codemirror/mode/htmlmixed/htmlmixed'
import 'codemirror/mode/markdown/markdown'
import 'codemirror/mode/yaml/yaml'
import 'codemirror/mode/xml/xml'
import 'codemirror/mode/clike/clike'
import 'codemirror/mode/shell/shell'
import 'codemirror/mode/go/go'
import 'codemirror/mode/perl/perl'
import 'codemirror/mode/php/php'
import 'codemirror/mode/python/python'

interface IToken {
  length: number
  text: string
  token: string
}

type Tokens = {
  [line: number]: { [startIndex: number]: IToken }
}

const extensionMIMEMap = new Map<string, string>()

extensionMIMEMap.set('.ts', 'text/typescript')
extensionMIMEMap.set('.tsx', 'text/jsx')
extensionMIMEMap.set('.js', 'text/javascript')
extensionMIMEMap.set('.json', 'application/json')
extensionMIMEMap.set('.html', 'text/html')
extensionMIMEMap.set('.htm', 'text/html')
extensionMIMEMap.set('.markdown', 'text/x-markdown')
extensionMIMEMap.set('.md', 'text/x-markdown')
extensionMIMEMap.set('.css', 'text/css')
extensionMIMEMap.set('.scss', 'text/x-scss')
extensionMIMEMap.set('.less', 'text/x-less')
extensionMIMEMap.set('.yaml', 'text/yaml')
extensionMIMEMap.set('.yml', 'text/yaml')

extensionMIMEMap.set('.objc', 'text/x-objectivec')
extensionMIMEMap.set('.scala', 'text/x-scala')
extensionMIMEMap.set('.sc', 'text/x-scala')
extensionMIMEMap.set('.cs', 'text/x-csharp')
extensionMIMEMap.set('.java', 'text/x-java')
extensionMIMEMap.set('.c', 'text/x-c')
extensionMIMEMap.set('.h', 'text/x-c')
extensionMIMEMap.set('.cpp', 'text/x-c++src')

extensionMIMEMap.set('.xml', 'text/xml')
extensionMIMEMap.set('.xaml', 'text/xml')
extensionMIMEMap.set('.csproj', 'text/xml')

extensionMIMEMap.set('.sh', 'text/x-sh')

extensionMIMEMap.set('.go', 'text/x-go')

extensionMIMEMap.set('.pl', 'text/x-perl')

extensionMIMEMap.set('.php', 'text/x-php')

extensionMIMEMap.set('.py', 'text/x-python')

onmessage = (ev: MessageEvent) => {
  const startTime = performance ? performance.now() : null

  const tabSize: number = ev.data.tabSize
  const extension: string = ev.data.extension
  const contents: string = ev.data.contents
  const requestedLines: Array<number> | undefined = ev.data.lines

  const mimeType = extensionMIMEMap.get(extension)

  if (!mimeType) {
    console.debug(
      `Could not map '${extension}' to supported mime type for highlighting`
    )
    postMessage({})
    return
  }

  const mode: CodeMirror.Mode<{}> = CodeMirror.getMode({ tabSize }, mimeType)
{'fraction_non_alphanumeric': 0.16166394779771615, 'fraction_numerical': 0.0, 'mean_word_length': 4.964007782101167, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4376323', 'n_tokens_mistral': 2155, 'n_tokens_neox': 2140, 'n_words': 473}