<commit_before>/* type.js: Types of files.
 * Copyright © 2011 <NAME>, <NAME>. All rights reserved.
 * The following code is covered by the GPLv2 license. */

var nodepath = require('path'),
    nodefs = require('fs'),
    driver = require('./driver'),
    camp = require('../camp/camp');

// Types are internally represented by integers.
//
// We go from _mime type_ world to _int_ world and back
// with `fromName` and `nameFromType`.

var fromName = {},     // Links the mime type to a unique integer.
    nbTypes = 0,   // Number of different types.
    fallbacks = [],    // Array of parent types (all being integers).
    nameFromType = [];  // Array of mime types (Strings).

function addType (mimeType, parents) {
  fallbacks.push(parents || []);
  nameFromType.push(mimeType);
  fromName[mimeType] = nbTypes;
  nbTypes++;
}


// The following algorithm is not cyclic-aware.
// Please do not create cycles in the type system.
function isCompatible (type, ancestor) {
  if (type === ancestor)  return true;
  if (!fallbacks[type]) {
    console.error('TYPE:isCompatible: type %s (%s) does not have fallbacks',
        nameFromType[type], type);
    return false;
  }
  for (var i = 0;  i < fallbacks[type].length;  i++) {
    if (isCompatible(fallbacks[type][i], ancestor))  return true;
  }
  return false;
}

// Defining current types (and their compatibility table).

addType('dir');            // contains a JSON string of files.
addType('binary');
addType('text/plain', [fromName['binary']]);
addType('text/html', [fromName['text/plain']]);
addType('text/xml', [fromName['text/plain']]);
addType('text/css', [fromName['text/plain']]);
addType('text/dtd', [fromName['text/plain']]);

addType('text/javascript', [fromName['text/plain']]);
addType('application/json', [fromName['text/plain']]);
addType('text/csv', [fromName['text/plain']]);
addType('text/x-csrc', [fromName['text/plain']]);
addType('text/x-c++src', [fromName['text/plain']]);
addType('text/x-csharp', [fromName['text/plain']]);
addType('text/x-java', [fromName['text/plain']]);
addType('text/x-go', [fromName['text/plain']]);
addType('text/x-groovy', [fromName['text/plain']]);
addType('text/x-clojure', [fromName['text/plain']]);
addType('text/x-coffeescript', [fromName['text/plain']]);
addType('text/x-diff', [fromName['text/plain']]);
addType('text/x-ecl', [fromName['text/plain']]);
addType('text/x-haskell', [fromName['text/plain']]);
addType('text/less', [fromName['text/plain']]);
addType('text/x-lua', [fromName['text/plain']]);
addType('text/x-markdown', [fromName['text/plain']]);
addType('text/x-mysql', [fromName['text/plain']]);
addType('text/n-triples', [fromName['text/plain']]);
addType('text/x-pascal', [fromName['text/plain']]);
addType('text/x-perl', [fromName['text/plain']]);
addType('text/x-php', [fromName['text/plain']]);
addType('text/x-plsql', [fromName['text/plain']]);
addType('text/x-properties', [fromName['text/plain']]);
addType('text/x-python', [fromName['text/plain']]);
addType('text/x-rpm-spec', [fromName['text/plain']]);
addType('text/x-rpm-changes', [fromName['text/plain']]);
addType('text/x-rsrc', [fromName['text/plain']]);
addType('text/x-rst', [fromName['text/plain']]);
addType('text/x-ruby', [fromName['text/plain']]);
addType('text/x-rustsrc', [fromName['text/plain']]);
addType('text/x-scheme', [fromName['text/plain']]);
addType('text/x-scheme', [fromName['text/plain']]);
addType('text/x-stsrc', [fromName['text/plain']]);
addType('text/x-sparql-query', [fromName['text/plain']]);
addType('text/x-stex', [fromName['text/plain']]);
addType('text/velocity', [fromName['text/plain']]);
addType('text/x-verilog', [fromName['text/plain']]);
addType('text/x-yaml', [fromName['text/plain']]);

addType('application/ps', [fromName['application/pdf'], fromName['binary']]);<commit_msg>[Mime Types] Removed duplicate text/x-scheme<commit_after>/* type.js: Types of files.
 * Copyright © 2011 <NAME>, <NAME>. All rights reserved.
 * The following code is covered by the GPLv2 license. */

var nodepath = require('path'),
    nodefs = require('fs'),
    driver = require('./driver'),
    camp = require('../camp/camp');

// Types are internally represented by integers.
//
// We go from _mime type_ world to _int_ world and back
// with `fromName` and `nameFromType`.

var fromName = {},     // Links the mime type to a unique integer.
    nbTypes = 0,   // Number of different types.
    fallbacks = [],    // Array of parent types (all being integers).
    nameFromType = [];  // Array of mime types (Strings).

function addType (mimeType, parents) {
  fallbacks.push(parents || []);
  nameFromType.push(mimeType);
  fromName[mimeType] = nbTypes;
  nbTypes++;
}


// The following algorithm is not cyclic-aware.
// Please do not create cycles in the type system.
function isCompatible (type, ancestor) {
  if (type === ancestor)  return true;
  if (!fallbacks[type]) {
    console.error('TYPE:isCompatible: type %s (%s) does not have fallbacks',
        nameFromType[type], type);
    return false;
  }
  for (var i = 0;  i < fallbacks[type].length;  i++) {
    if (isCompatible(fallbacks[type][i], ancestor))  return true;
  }
  return false;
}

// Defining current types (and their compatibility table).

addType('dir');            // contains a JSON string of files.
addType('binary');
addType('text/plain', [fromName['binary']]);
addType('text/html', [fromName['text/plain']]);
addType('text/xml', [fromName['text/plain']]);
addType('text/css', [fromName['text/plain']]);
addType('text/dtd', [fromName['text/plain']]);

addType('text/javascript', [fromName['text/plain']]);
addType('application/json', [fromName['text/plain']]);
addType('text/csv', [fromName['text/plain']]);
addType('text/x-csrc', [fromName['text/plain']]);
addType('text/x-c++src', [fromName['text/plain']]);
addType('text/x-csharp', [fromName['text/plain']]);
addType('text/x-java', [fromName['text/plain']]);
addType('text/x-go', [fromName['text/plain']]);
addType('text/x-groovy', [fromName['text/plain']]);
addType('text/x-clojure', [fromName['text/plain']]);
addType('text/x-coffeescript', [fromName['text/plain']]);
addType('text/x-diff', [fromName['text/plain']]);
addType('text/x-ecl', [fromName['text/plain']]);
addType('text/x-haskell', [fromName['text/plain']]);
addType('text/less', [fromName['text/plain']]);
addType('text/x-lua', [fromName['text/plain']]);
addType('text/x-markdown', [fromName['text/plain']]);
addType('text/x-mysql', [fromName['text/plain']]);
addType('text/n-triples', [fromName['text/plain']]);
addType('text/x-pascal', [fromName['text/plain']]);
addType('text/x-perl', [fromName['text/plain']]);
addType('text/x-php', [fromName['text/plain']]);
addType('text/x-plsql', [fromName['text/plain']]);
addType('text/x-properties', [fromName['text/plain']]);
addType('text/x-python', [fromName['text/plain']]);
addType('text/x-rpm-spec', [fromName['text/plain']]);
addType('text/x-rpm-changes', [fromName['text/plain']]);
addType('text/x-rsrc', [fromName['text/plain']]);
addType('text/x-rst', [fromName['text/plain']]);
addType('text/x-ruby', [fromName['text/plain']]);
addType('text/x-rustsrc', [fromName['text/plain']]);
addType('text/x-scheme', [fromName['text/plain']]);
addType('text/x-stsrc', [fromName['text/plain']]);
addType('text/x-sparql-query', [fromName['text/plain']]);
addType('text/x-stex', [fromName['text/plain']]);
addType('text/velocity', [fromName['text/plain']]);
addType('text/x-verilog', [fromName['text/plain']]);
addType('text/x-yaml', [fromName['text/plain']]);

addType('application/ps', [fromName['application/pdf'], fromName['binary']]);
{'fraction_non_alphanumeric': 0.22947146434690918, 'fraction_numerical': 0.0018452616317385, 'mean_word_length': 5.679577464788732, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3334585', 'n_tokens_mistral': 2666, 'n_tokens_neox': 2598, 'n_words': 499}