<commit_before>import { unique } from './util/array';
import { WorkspaceConfiguration, workspace, extensions, Extension } from 'vscode';

interface ColorizeConfig {
  languages: string[];
  isHideCurrentLineDecorations: boolean;
  colorizedVariables: string[];
  colorizedColors: string[];
  filesToExcludes: string[];
  filesToIncludes: string[];
  inferedFilesToInclude: string[];
  searchVariables: boolean;
}

function getColorizeConfig(): ColorizeConfig {
  const configuration: WorkspaceConfiguration = workspace.getConfiguration('colorize');

  // remove duplicates (if duplicates)
  const colorizedVariables = Array.from(new Set(configuration.get('colorized_variables', []))); // [...new Set(array)] // works too
  const colorizedColors = Array.from(new Set(configuration.get('colorized_colors', []))); // [...new Set(array)] // works too

  const languages = configuration.get('languages', []);

  const inferedFilesToInclude = inferFilesToInclude(languages).map(extension => `**/*${extension}`);
  const filesToIncludes = Array.from(new Set(configuration.get('include', [])));
  const filesToExcludes = Array.from(new Set(configuration.get('exclude', [])));

  const searchVariables = configuration.get('enable_search_variables', false);

  return {
    languages,
    isHideCurrentLineDecorations: configuration.get('hide_current_line_decorations'),
    colorizedColors,
    colorizedVariables,
    filesToIncludes,
    filesToExcludes,
    inferedFilesToInclude,
    searchVariables
  };
}


function inferFilesToInclude(languagesConfig: string[]): string[] {
  const filesExtensions = extensions.all.reduce((acc, extension: Extension<unknown>) => {
    if (extension.packageJSON?.contributes?.languages) {
      extension.packageJSON.contributes.languages.forEach(language => {
        if (languagesConfig.indexOf(language.id) !== -1) {
          acc = [
            ...acc,
            ...language.extensions
          ];
        }
      });<commit_msg>fix: check if extension as files extensions contributions<commit_after>import { unique } from './util/array';
import { WorkspaceConfiguration, workspace, extensions, Extension } from 'vscode';

interface ColorizeConfig {
  languages: string[];
  isHideCurrentLineDecorations: boolean;
  colorizedVariables: string[];
  colorizedColors: string[];
  filesToExcludes: string[];
  filesToIncludes: string[];
  inferedFilesToInclude: string[];
  searchVariables: boolean;
}

function getColorizeConfig(): ColorizeConfig {
  const configuration: WorkspaceConfiguration = workspace.getConfiguration('colorize');

  // remove duplicates (if duplicates)
  const colorizedVariables = Array.from(new Set(configuration.get('colorized_variables', []))); // [...new Set(array)] // works too
  const colorizedColors = Array.from(new Set(configuration.get('colorized_colors', []))); // [...new Set(array)] // works too

  const languages = configuration.get('languages', []);

  const inferedFilesToInclude = inferFilesToInclude(languages).map(extension => `**/*${extension}`);
  const filesToIncludes = Array.from(new Set(configuration.get('include', [])));
  const filesToExcludes = Array.from(new Set(configuration.get('exclude', [])));

  const searchVariables = configuration.get('enable_search_variables', false);

  return {
    languages,
    isHideCurrentLineDecorations: configuration.get('hide_current_line_decorations'),
    colorizedColors,
    colorizedVariables,
    filesToIncludes,
    filesToExcludes,
    inferedFilesToInclude,
    searchVariables
  };
}


function inferFilesToInclude(languagesConfig: string[]): string[] {
  const filesExtensions = extensions.all.reduce((acc, extension: Extension<unknown>) => {
    if (extension.packageJSON?.contributes?.languages) {
      extension.packageJSON.contributes.languages.forEach(language => {
        if (languagesConfig.indexOf(language.id) !== -1 && language.extensions) {
          acc = [
            ...acc,
            ...language.extensions
          ];
        }
      });
{'fraction_non_alphanumeric': 0.1334666333416646, 'fraction_numerical': 0.0004998750312421895, 'mean_word_length': 4.566063977746871, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5908073', 'n_tokens_mistral': 1089, 'n_tokens_neox': 1082, 'n_words': 214}