<commit_before>
 /*
  Copyright 2016 - 2017 by <NAME>

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
 */

(function(factory) {
  "use strict";

  var dependencies = [];

  if (typeof define === 'function' && define.amd) {
    dependencies.push("module");
    define(dependencies, factory);
  }
  else if (typeof exports === 'object') {
    module.exports = factory.apply(undefined, dependencies.map(function(d) {return require(d);}));
  }
}(function(amdModule) {
  "use strict";

  /**
   * eol is always <code>\n</code> in modern browsers. On node, it depends on the platform, and is offered by
   * the os-module.
   */
  var eol = "\n";
  if (typeof exports === "object") {
    eol = require("os").EOL;
  }

  var path;
  if (typeof exports === "object") {
    path = require("path");
  }

  var dirSeparator = "/";
  var thisDirectory = ".";
  var parentDirectory = "..";

  function browserModuleLocation(amdModule) {
    var location = window.location.href;
    location = location.split(dirSeparator);
    if (0 <= location[location.length - 1].indexOf(".")) { // last entry is a file
      location.pop();
    }
    location = location.concat(amdModule.uri.split(dirSeparator));
    var dotLocation = location.indexOf(thisDirectory); // remove "this directory" path elements
    while (0 <= dotLocation) {
      location.splice(dotLocation, 1);
      dotLocation = location.indexOf(thisDirectory);
    }
    dotLocation = location.indexOf(parentDirectory); // remove "parent directory" path elements
    while (0 <= dotLocation) {
      location.splice(dotLocation - 1, 2);
      dotLocation = location.indexOf(parentDirectory);
    }
    return location.join(dirSeparator);
  }

  var fileName = (typeof module === "object") ? module.filename : browserModuleLocation(amdModule);

  function typeOf(obj) {
    if (obj === null) { // workaround for some weird implementations
      return "null";
    }
    var result = Object.prototype.toString.call(obj).match(/\s([a-zA-Z]+)/)[1].toLowerCase();
    // on some browsers, the main window returns as "global" (WebKit) or "window" (FF), but this is an object too
    if (result === "global" || result === "window") {
      result = "object";
    }
    return result; // return String
  }

  /**
   * <p>Returns the directory name of a path, similar to the Unix dirname command.
   * For example:</p>
   * <pre>
   * path.dirname('/foo/bar/baz/asdf/quux');
   * // Returns: '/foo/bar/baz/asdf'
   * </pre>
   * <p>A <code>TypeError</code> is thrown if path is not a string.</p>
   * <p>This method is a wrapper around node's <code>path.dirname</code>, which is not available on the browser
   * directly.
   */
  var dirname = path
    ? path.dirname
    : function(path) {
        if (typeOf(path) !== "string") {
          throw new TypeError("path is not a string");
        }
        var result = path.split(dirSeparator);
        result.pop();
        return result.join(dirSeparator);
      };<commit_msg>Note a MUDO and a TODO<commit_after>
 /*
  Copyright 2016 - 2017 by <NAME>

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
 */

(function(factory) {
  "use strict";

  var dependencies = [];

  if (typeof define === 'function' && define.amd) {
    dependencies.push("module");
    define(dependencies, factory);
  }
  else if (typeof exports === 'object') {
    module.exports = factory.apply(undefined, dependencies.map(function(d) {return require(d);}));
  }
}(function(amdModule) {
  "use strict";

  /**
   * eol is always <code>\n</code> in modern browsers. On node, it depends on the platform, and is offered by
   * the os-module.
   */
  // TODO is there no way to do this without the os-dependency?
  var eol = "\n";
  if (typeof exports === "object") {
    eol = require("os").EOL;
  }

  var path;
  if (typeof exports === "object") {
    path = require("path");
  }

  var dirSeparator = "/";
  var thisDirectory = ".";
  var parentDirectory = "..";

  function browserModuleLocation(amdModule) {
    var location = window.location.href;
    location = location.split(dirSeparator);
    if (0 <= location[location.length - 1].indexOf(".")) { // last entry is a file
      location.pop();
    }
    location = location.concat(amdModule.uri.split(dirSeparator));
    var dotLocation = location.indexOf(thisDirectory); // remove "this directory" path elements
    while (0 <= dotLocation) {
      location.splice(dotLocation, 1);
      dotLocation = location.indexOf(thisDirectory);
    }
    dotLocation = location.indexOf(parentDirectory); // remove "parent directory" path elements
    while (0 <= dotLocation) {
      location.splice(dotLocation - 1, 2);
      dotLocation = location.indexOf(parentDirectory);
    }
    return location.join(dirSeparator);
  }

  var fileName = (typeof module === "object") ? module.filename : browserModuleLocation(amdModule);

  function typeOf(obj) {
    if (obj === null) { // workaround for some weird implementations
      return "null";
    }
    var result = Object.prototype.toString.call(obj).match(/\s([a-zA-Z]+)/)[1].toLowerCase();
    // on some browsers, the main window returns as "global" (WebKit) or "window" (FF), but this is an object too
    if (result === "global" || result === "window") {
      result = "object";
    }
    return result; // return String
  }

  /* MUDO we still need a unit test for this function; better then to just do away with ugly path dependency
     once and for all */
  /**
   * <p>Returns the directory name of a path, similar to the Unix dirname command.
   * For example:</p>
   * <pre>
   * path.dirname('/foo/bar/baz/asdf/quux');
   * // Returns: '/foo/bar/baz/asdf'
   * </pre>
   * <p>A <code>TypeError</code> is thrown if path is not a string.</p>
   * <p>This method is a wrapper around node's <code>path.dirname</code>, which is not available on the browser
   * directly.
   */
  var dirname = path
    ? path.dirname
    : function(path) {
        if (typeOf(path) !== "string") {
          throw new TypeError("path is not a string");
        }
        var result = path.split(dirSeparator);
        result.pop();
        return result.join(dirSeparator);
      };
{'fraction_non_alphanumeric': 0.13270608060665637, 'fraction_numerical': 0.005617188597107148, 'mean_word_length': 3.074370709382151, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '74081', 'n_tokens_mistral': 2107, 'n_tokens_neox': 2057, 'n_words': 686}