<commit_before>var diff = require("diff");

module.exports = {
  diff: function(lhs, rhs) {
    var result = []

    var getString = function(str) {
      return (str == null || str == undefined) ? "" : str;
    }
    var addDelta = function(delta) {
      delete delta["count"];
      result.push(delta);
    }

    var lhs_ = getString(lhs);
    var rhs_ = getString(rhs);

    var lineDiff = diff.diffLines(lhs_, rhs_);
    var index = 0;
    while (index < lineDiff.length) {
      var delta1 = lineDiff[index];
      if (index < lineDiff.length - 1) {
        var delta2 = lineDiff[index + 1];

        if (delta1.removed && delta2.added) {
          if (delta1.count == delta2.count) {
            // Might use word diff when number of lines of original and revised texts are same.
            var wordDiff = diff.diffWords(delta1.value, delta2.value);

            var useLineDiffFlag = false;

            for (var i = 0; i < wordDiff.length; i++) {
              var delta = wordDiff[i];
              if ((delta.added || delta.removed) && ~delta.value.indexOf('\n')) {
                useLineDiffFlag = true;
                break;
              }
            }

            if (useLineDiffFlag) {
              // Use line diff
              addDelta(delta1);
              addDelta(delta2);
            } else {
              // Use word diff
              for (var i = 0; i < wordDiff.length; i++) {
                addDelta(wordDiff[i]);
              }
            }

            index += 2;
          } else {
            addDelta(delta1);
            addDelta(delta2);
            index += 2;
          }
        } else {
          addDelta(delta1);

          index += 1;
        }
      } else {
        addDelta(delta1);

        index += 1;
      }
    }

    return result;
  },
  addLineNumbers: function(diff_) {
    // Convert diff data structure to display
    var diff = [];
    var lineNumberOfLhs = 0;
    var lineNumberOfRhs = 0;
    var isInLhs = false;
    var isInRhs = false;
    var delta = null;
    for (delta_ of diff_) {
      var lines = delta_.value.split('\n');
      for (var i = 0; i < lines.length; i++) {
        var line = lines[i]

        delta = delta || {};
        delta.values = delta.values || [];


        if (!delta_.removed) {
          // delta.value is in the rhs
          isInRhs = true;
        }
        if (!delta_.added) {
          // delta.value is in the lhs
          isInLhs = true;
        }

        if (delta_.removed) {
          delta.values.push({
            removed: true,
            value: line
          });
        } else if (delta_.added){
          delta.values.push({
            added: true,
            value: line
          });
        } else {
          delta.values.push({
            value: line
          })
        }
        if ((i != lines.length - 1) || delta_.value[delta_.value.length - 1] == '\n') {
          // End a line
          if (isInRhs) {
            // delta.value is in the rhs
            delta.lineNumberOfRhs = lineNumberOfRhs;
            lineNumberOfRhs += 1;
          }
          if (isInLhs) {
            // delta.value is in the lhs
            delta.lineNumberOfLhs = lineNumberOfLhs;
            lineNumberOfLhs += 1;
          }
          diff.push(delta);

          // Initialize the local variables
          isInRhs = false;
          isInLhs = false;
          delta = null;
        }
      }
    }
    if (delta != null) {
      // Deal with a corner case (when last character of text is not \n)
      if (isInRhs) {
        // delta.value is in the rhs
        delta.lineNumberOfRhs = lineNumberOfRhs;
        lineNumberOfRhs += 1;
      }
      if (isInLhs) {
        // delta.value is in the lhs
        delta.lineNumberOfLhs = lineNumberOfLhs;
        lineNumberOfLhs += 1;
      }
      diff.push(delta);
    }

    return diff;
  }
}
<commit_msg>Fix the implementation of addLineNumbers<commit_after>var diff = require("diff");

module.exports = {
  diff: function(lhs, rhs) {
    var result = []

    var getString = function(str) {
      return (str == null || str == undefined) ? "" : str;
    }
    var addDelta = function(delta) {
      delete delta["count"];
      result.push(delta);
    }

    var lhs_ = getString(lhs);
    var rhs_ = getString(rhs);

    var lineDiff = diff.diffLines(lhs_, rhs_);
    var index = 0;
    while (index < lineDiff.length) {
      var delta1 = lineDiff[index];
      if (index < lineDiff.length - 1) {
        var delta2 = lineDiff[index + 1];

        if (delta1.removed && delta2.added) {
          if (delta1.count == delta2.count) {
            // Might use word diff when number of lines of original and revised texts are same.
            var wordDiff = diff.diffWords(delta1.value, delta2.value);

            var useLineDiffFlag = false;

            for (var i = 0; i < wordDiff.length; i++) {
              var delta = wordDiff[i];
              if ((delta.added || delta.removed) && ~delta.value.indexOf('\n')) {
                useLineDiffFlag = true;
                break;
              }
            }

            if (useLineDiffFlag) {
              // Use line diff
              addDelta(delta1);
              addDelta(delta2);
            } else {
              // Use word diff
              for (var i = 0; i < wordDiff.length; i++) {
                addDelta(wordDiff[i]);
              }
            }

            index += 2;
          } else {
            addDelta(delta1);
            addDelta(delta2);
            index += 2;
          }
        } else {
          addDelta(delta1);

          index += 1;
        }
      } else {
        addDelta(delta1);

        index += 1;
      }
    }

    return result;
  },
  addLineNumbers: function(diff_) {
    // Convert diff data structure to display
    var diff = [];
    var lineNumberOfLhs = 0;
    var lineNumberOfRhs = 0;
    var isInLhs = false;
    var isInRhs = false;
    var delta = null;

    var i = 0;
    while (i < diff_.length) {
      if (
        (i != diff_.length - 1) &&
        diff_[i].removed && diff_[i + 1].added &&
        diff_[i].value.endsWith("\n") && diff_[i + 1].value.endsWith("\n")
      ) {
        // Replace lines
        var removedDelta = diff_[i];
        var addedDelta = diff_[i + 1];

        // Add removedDelta to diff
        var rlines = removedDelta.value.split("\n");
        for (var j = 0; j < rlines.length - 1; j++) {
          var line = rlines[j];
          diff.push({
            lineNumberOfLhs: lineNumberOfLhs,
            values: [{
              removed: true,
              value: line
            }]
          });
          lineNumberOfLhs += 1;
        }
        // Add addedDelta to diff
        var alines = addedDelta.value.split("\n");
        for (var j = 0; j < alines.length - 1; j++) {
          var line = alines[j];
          diff.push({
            lineNumberOfRhs: lineNumberOfRhs,
            values: [{
              added: true,
              value: line
            }]
          });
          lineNumberOfRhs += 1;
        }

        // Add last \n to diff
        diff.push({
          lineNumberOfRhs: lineNumberOfRhs,
          lineNumberOfLhs: lineNumberOfLhs,
          values: [{value: ""}]
        });

        i += 2;
      } else {
        var delta_ = diff_[i];
        var lines = delta_.value.split('\n');

        for (var j = 0; j < lines.length; j++) {
          delta = delta || {};
          delta.values = delta.values || [];

          var line = lines[j];

          if (!delta_.removed) {
            // delta.value is in the rhs
            isInRhs = true;
          }
          if (!delta_.added) {
            // delta.value is in the lhs
            isInLhs = true;
          }

          if (line.length != 0) {
            if (delta_.removed) {
              delta.values.push({
                removed: true,
                value: line
              });
            } else if (delta_.added){
              delta.values.push({
                added: true,
                value: line
              });
            } else {
              delta.values.push({
                value: line
              })
            }
          }

          if (j != lines.length - 1) {
            // End a line
            if (isInRhs) {
              // delta.value is in the rhs
              delta.lineNumberOfRhs = lineNumberOfRhs;
              lineNumberOfRhs += 1;
            }
            if (isInLhs) {
              // delta.value is in the lhs
              delta.lineNumberOfLhs = lineNumberOfLhs;
              lineNumberOfLhs += 1;
            }
            if (delta.values.length == 0) {
              if (isInRhs && isInLhs) {
                delta.values.push({value: ""});
              } else if (isInRhs) {
                delta.values.push({
                  added: true,
                  value: ""
                });
              } else if (isInLhs) {
                delta.values.push({
                  removed: true,
                  value: ""
                });
              }
            }
            diff.push(delta);

            // Initialize the local variables
            isInRhs = false;
            isInLhs = false;
            delta = null;
          }
        }
        i += 1;
      }
    }

    if (delta != null) {
      // Deal with a corner case (when last character of text is not \n)
      if (isInRhs) {
        // delta.value is in the rhs
        delta.lineNumberOfRhs = lineNumberOfRhs;
        lineNumberOfRhs += 1;
      }
      if (isInLhs) {
        // delta.value is in the lhs
        delta.lineNumberOfLhs = lineNumberOfLhs;
        lineNumberOfLhs += 1;
      }

      if (delta.values.length == 0) {
        if (isInRhs && isInLhs) {
          delta.values.push({value: ""});
        } else if (isInRhs) {
          delta.values.push({
            added: true,
            value: ""
          });
        } else if (isInLhs) {
          delta.values.push({
            removed: true,
            value: ""
          });
        }
      }

      diff.push(delta);
    }

    return diff;
  }
}

{'fraction_non_alphanumeric': 0.11782746339533043, 'fraction_numerical': 0.00781559161060546, 'mean_word_length': 1.2369993361363134, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1112446', 'n_tokens_mistral': 3040, 'n_tokens_neox': 2937, 'n_words': 741}