<commit_before>const BY_LANGUAGE = {
  xml: "<!-- %s -->",
  hash: "# %s",
  bat: "REM %s",
  percent: "% %s",
  handlebars: "{{!-- %s --}}",
  doubleDash: "-- %s",
  block: "/* %s */",
  doubleLine: "-- %s",
  pug: "//- %s",
  common: "// %s"
};

const BY_PATTERN = {
  block: ["css"],
  hash: [
    "python",
    "shellscript",
    "dockerfile",
    "ruby",
    "coffeescript",
    "dockerfile",
    "elixir",
    "julia",
    "makefile",
    "perl",
    "yaml"
  ],
  xml: ["xml", "xsl", "html", "markdown"],
  percent: ["tex", "prolog"],
  pug: ["pug", "jade"],
  doubleDash: ["lua", "sql", "haskell", "cabal"],
  doubleLine: ["sql", "ada", "haskell", "lua", "pig"],
  common: [
    "javascript",
    "javascriptreact",
    "typescript",
    "java",
    "c",
    "cpp",
    "d"
  ]
};

function computePatternByLanguage(): { [s: string]: string } {
  let result: { [s: string]: string } = {};
  for (let [pattern, languages] of Object.entries(BY_PATTERN)) {
    languages.forEach(language => {
      result[language] = BY_LANGUAGE[pattern];
    });
  }
  return result;
}

const patterns = computePatternByLanguage();

export default patterns;<commit_msg>Add TODO to comment patterns config to add all language ids<commit_after>// TODO: add all languages from https://code.visualstudio.com/docs/languages/identifiers

const BY_LANGUAGE = {
  xml: "<!-- %s -->",
  hash: "# %s",
  bat: "REM %s",
  percent: "% %s",
  handlebars: "{{!-- %s --}}",
  doubleDash: "-- %s",
  block: "/* %s */",
  doubleLine: "-- %s",
  pug: "//- %s",
  common: "// %s"
};

const BY_PATTERN = {
  block: ["css"],
  hash: [
    "python",
    "shellscript",
    "dockerfile",
    "ruby",
    "coffeescript",
    "dockerfile",
    "elixir",
    "julia",
    "makefile",
    "perl",
    "yaml"
  ],
  xml: ["xml", "xsl", "html", "markdown"],
  percent: ["tex", "prolog"],
  pug: ["pug", "jade"],
  doubleDash: ["lua", "sql", "haskell", "cabal"],
  doubleLine: ["sql", "ada", "haskell", "lua", "pig"],
  common: [
    "javascript",
    "javascriptreact",
    "typescript",
    "java",
    "c",
    "c#",
    "cpp",
    "d"
  ]
};

function computePatternByLanguage(): { [s: string]: string } {
  let result: { [s: string]: string } = {};
  for (let [pattern, languages] of Object.entries(BY_PATTERN)) {
    languages.forEach(language => {
      result[language] = BY_LANGUAGE[pattern];
    });
  }
  return result;
}

const patterns = computePatternByLanguage();

export default patterns;
{'fraction_non_alphanumeric': 0.22849025974025974, 'fraction_numerical': 0.0, 'mean_word_length': 3.008130081300813, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4923201', 'n_tokens_mistral': 948, 'n_tokens_neox': 925, 'n_words': 209}