<commit_before>/**
 * Hyphenation, using Frank Liang's algorithm.
 * Inspired by <NAME>'s:
 *   -> http://nedbatchelder.com/code/modules/hyphenate.py
 *
 * Usage: $ /path/to/hyphenate.js "WORDS I WANT TO HYPHENATE!"
 *  > WORDS I WANT TO HY-PHEN-ATE!
 */

var _ = require("underscore")._;

var MSRGX = /\s{2,}/g,
    DRGX = /\d/,
    CRGX = /[a-z]/,
    ACRGX = /[.a-z]/;

var Hyphenator = function(patterns, exceptions) {
  this.tree = {};
  this.exceptions = {};

  this._insert_exception = function(ex) {
    var a = [0];
    var x = ex.split(CRGX).forEach(function(h) {
      a.push(h === "-" ? 1 : 0);
    });
    this.exceptions[ex.replace("-", "")] = a;
  };

  this._insert_pattern = function(pat) {
    var chars = pat.replace(DRGX, ""),
        points = pat.split(ACRGX).map(function(h) { return parseInt(h) || 0 }),
        c = "";
    var t = this.tree;

    for (var i = 0; i < chars.length; i++) {
      c = chars.charAt(i);

      if (!(c in t)) {
        t[c] = {};
      }
      t = t[c];
    };
    t[null] = points;
  };

  /**
   * Hyphenates a word.
   *
   * @param string word
   *   A word to hyphenate.
   *
   * @return string
   *   The word, hyphenated.
   */
  this.hyphenate_word = function(word) {
    if (word.length <= 4) {
      return [word];
    }

    var points = [],
        lword = word.toLowerCase();

    if (lword in this.exceptions) {
      points = this.exceptions[lword];
    }
    else {
      var work = "." + lword + ".";

      // Initialize array with 0s.
      points = Array.apply(null, Array(work.length + 1)).map(function() {
        return 0;
      });

      for (var i = 0; i < work.length; i++) {
        var t = this.tree,
            sub_work = work.substring(i);

        for (var n = 0; n < sub_work.length; n++) {
          c = sub_work[n];

          if (c in t) {
            t = t[c];
            if (null in t) {
              var p = t[null];
              for (var j = 0; j < p.length; j++) {
                points[i + j] = Math.max(points[i + j], p[j]);
              }
            }
          }
          else {
            break;
          }
        }
      }
      points[1] = points[2] = points[points.length - 2] = points[points.length - 3] = 0
    }

    var pieces = [""];
    var zipped = _.zip(word, points.slice(2, points.length - 1));
    zipped.forEach(function(z) {
      var c = z[0],
          p = z[1];
      pieces[pieces.length - 1] += c;
      if (p % 2) {
        pieces.push("");
      }
    });
    return pieces;
  };

  patterns.replace(MSRGX, " ").split(" ").forEach(this._insert_pattern.bind(this));
  exceptions.replace(MSRGX, " ").split(" ").forEach(this._insert_exception.bind(this));
};


/**
 * Knuth and Liang's original hyphenation patterns from classic TeX.
 * In the public domain.<commit_msg>Code cleanup.<commit_after>/**
 * Hyphenation, using Frank Liang's algorithm.
 * Inspired by <NAME>atchelder's:
 *   -> http://nedbatchelder.com/code/modules/hyphenate.py
 *
 * Usage: $ /path/to/hyphenate.js "WORDS I WANT TO HYPHENATE!"
 *  > WORDS I WANT TO HY-PHEN-ATE!
 */

var _ = require("underscore")._;

var MSRGX = /\s{2,}/g,
    DRGX = /\d/,
    CRGX = /[a-z]/,
    ACRGX = /[.a-z]/;

var Hyphenator = function(patterns, exceptions) {
  this.tree = {};
  this.exceptions = {};

  this._insert_exception = function(ex) {
    var a = [0];
    var x = ex.split(CRGX).forEach(function(h) {
      a.push(h === "-" ? 1 : 0);
    });
    this.exceptions[ex.replace("-", "")] = a;
  };

  this._insert_pattern = function(pat) {
    var chars = pat.replace(DRGX, ""),
        points = pat.split(ACRGX).map(function(h) { return parseInt(h) || 0 }),
        c = "",
        t = this.tree;

    for (var i = 0; i < chars.length; i++) {
      c = chars.charAt(i);

      if (!(c in t)) {
        t[c] = {};
      }
      t = t[c];
    };
    t[null] = points;
  };

  /**
   * Hyphenates a word.
   *
   * @param string word
   *   A word to hyphenate.
   *
   * @return string
   *   The word, hyphenated.
   */
  this.hyphenate_word = function(word) {
    if (word.length <= 4) {
      return [word];
    }

    var points = [],
        lword = word.toLowerCase();

    if (lword in this.exceptions) {
      points = this.exceptions[lword];
    }
    else {
      var work = "." + lword + ".";

      // Initialize array with 0s.
      points = Array.apply(null, Array(work.length + 1)).map(function() {
        return 0;
      });

      for (var i = 0; i < work.length; i++) {
        var t = this.tree,
            sub_work = work.substring(i);

        for (var n = 0; n < sub_work.length; n++) {
          c = sub_work[n];
          if (!(c in t)) break;

          t = t[c];
          if (!(null in t)) continue;

          var p = t[null];
          for (var j = 0; j < p.length; j++) {
            points[i + j] = Math.max(points[i + j], p[j]);
          }
        }
      }
      points[1] = points[2] = points[points.length - 2] = points[points.length - 3] = 0
    }

    var pieces = [""],
        zipped = _.zip(word, points.slice(2, points.length - 1));

    zipped.forEach(function(z) {
      var c = z[0],
          p = z[1];

      pieces[pieces.length - 1] += c;
      if (p % 2) {
        pieces.push("");
      }
    });

    return pieces;
  };

  patterns.replace(MSRGX, " ").split(" ").forEach(this._insert_pattern.bind(this));
  exceptions.replace(MSRGX, " ").split(" ").forEach(this._insert_exception.bind(this));
};


/**
 * Knuth and Liang's original hyphenation patterns from classic TeX.
 * In the public domain.
{'fraction_non_alphanumeric': 0.18129288551823763, 'fraction_numerical': 0.00866738894907909, 'mean_word_length': 1.8684619368203004, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1295866', 'n_tokens_mistral': 2098, 'n_tokens_neox': 2001, 'n_words': 452}