<commit_before>var _ = require('lodash');
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
var extname = require('path').extname;
var lodash = require('lodash');
require('es6-set/implement'); // required for text-buffer
require('iconv-lite').extendNodeEncodings();
var TextBuffer = require('text-buffer');
var Point = require('text-buffer/lib/point');
var Range = require('text-buffer/lib/range');

var util = require('slap-util');
var word = require('./word');
var editorWidgetOpts = require('./opts');
var highlightClient = require('./highlight/client');

var BaseWidget = require('base-widget');

function Editor (opts) {
  var self = this;

  if (!(self instanceof Editor)) return new Editor(opts);

  BaseWidget.call(self, _.merge({
    focusable: true,
    multiLine: true
  }, editorWidgetOpts.editor, opts));

  self.gutter = new BaseWidget(_.merge({
    parent: self,
    tags: true,
    wrap: false,
    style: {},
    top: 0,
    left: 0,
    bottom: 0
  }, self.options.gutter));

  self.buffer = new BaseWidget(_.merge({
    parent: self,
    tags: true,
    wrap: false,
    style: {},
    top: 0,
    left: self.options.multiLine && !self.gutter.options.hidden ? self.gutter.width : 0,
    right: 0,
    bottom: 0
  }, self.options.buffer));

  self.textBuf = new TextBuffer({encoding: self.options.defaultEncoding});
  self.textBuf.loadSync();

  self.selection = self.textBuf.markPosition(new Point(0, 0), {type: 'selection', invalidate: 'never'});
  self.scroll = new Point(0, 0);
  self.data.updatePreferredX = true;

  self.language(false);
  self.toggleInsertMode();
  self._initHighlighting();

  var _updateContent = self._updateContent.bind(self);
  self._updateContent = lodash.throttle(_updateContent, self.options.perf.updateContentThrottle, false);
  self._updateContent();
}
Editor.prototype.__proto__ = BaseWidget.prototype;

Editor.prototype.open = Promise.method(function (givenPath) {
  // Handles nonexistent paths
  var self = this;
  return self.ready // ensures textBuf path-changed is triggered in _initHandlers
    .then(function () { return Editor.getOpenParams(givenPath); })
    .tap(function (params) { return self.textBuf.setPath(params.path); })
    .tap(function (params) { if (params.exists) return self.textBuf.load(); })
    .tap(function (params) { self.selection.setHeadPosition(params.position); })
    .return(self);
});
Editor.prototype.save = Promise.method(function (path) {
  var self = this;
  return Promise.resolve()
    .then(function () { return path ? self.textBuf.saveAs(util.resolvePath(path)) : self.textBuf.save(); })
    .then(function () { return self.textBuf.getPath(); })
});

// looks for path like /home/dan/file.c:3:8 but matches every string
Editor.openRegExp = new RegExp('^'
  + '(.*?)'          // path:   match[1] (like /home/dan/file.c)
  + '(?:\\:(\\d+))?' // row:    match[2] (like 3, optional)
  + '(?:\\:(\\d+))?' // column: match[3] (like 8, optional)
  + '$');
Editor.getOpenParams = Promise.method(function (givenPath) {
  givenPath = util.resolvePath(givenPath);
  var baseParams = {
    path: givenPath,
    exists: false,
    position: new Point(0, 0)
  };
  var match = givenPath.match(Editor.openRegExp); // always matches
  return [
    // if a path like file.c:3:8 is passed, see if `file.c:3:8` exists first,
    // then try `file.c:3` line 8, then `file.c` line 3 column 8
    baseParams,
    _.merge({}, baseParams, {
      path: match[1]+':'+match[2],
      position: {row: Editor.parseCoordinate(match[3])}
    }),
    _.merge({}, baseParams, {
      path: match[1],
      position: {row: Editor.parseCoordinate(match[2]), column: Editor.parseCoordinate(match[3])}
    })
  ].reduce(function (promise, params) {
    return promise.then(function (resultParams) {
      if ((resultParams || {}).exists) return resultParams;
      return Editor.exists(params.path).then(function (exists) {
        params.exists = exists;
        return params;
      });
    });
  }, Promise.resolve());
});

Editor.parseCoordinate = function (n) { return (parseInt(n, 10) - 1) || 0; };
Editor.exists = Promise.method(function (givenPath) {
  return fs.openAsync(givenPath, 'r')
    .then(function (fd) { return fs.closeAsync(fd); })
    .return(true)
    .catch(function (err) {
      if (err.code !== 'ENOENT') throw err;
      return false;
    });
});

Editor.prototype.insertMode = util.getterSetter('insertMode', null, Boolean);
Editor.prototype.toggleInsertMode = function () { return this.insertMode(!this.insertMode()); };

Editor.prototype.language = util.getterSetter('language', null, null);
Editor.prototype.readOnly = util.getterSetter('readOnly', null, Boolean);
Editor.prototype.lineWithEndingForRow = function (row) {
  var self = this;
  return self.textBuf.lineForRow(row) + self.textBuf.lineEndingForRow(row);
};
Editor.prototype.delete = function (range) {
  var self = this;
  if (!range) {
    self.textBuf.setTextInRange(self.selection.getRange(), '');
  } else {
    self.textBuf.delete(range);
  }
  return self;
};
Editor.prototype._getTabString = function () {
  var self = this;
  return self.buffer.options.useSpaces
    ? _.repeat(' ', self.buffer.options.tabSize)
    : '\t';
};
Editor.prototype.indent = function (range, dedent) {
  var self = this;

  var tabString = self._getTabString();
  var indentRegex = new RegExp('^(\t| {0,'+self.buffer.options.tabSize+'})', 'g');
  var startDiff = 0, endDiff = 0;
  var linesRange = range.copy();
  linesRange.start.column = 0;
  linesRange.end.column = Infinity;
  self.textBuf.setTextInRange(linesRange, util.text.splitLines(self.textBuf.getTextInRange(linesRange))
    .map(function (line, i) {
      var result = !dedent
        ? tabString + line
        : line.replace(indentRegex, '');
      if (i === 0)                       startDiff = result.length - line.length;
      if (i === range.getRowCount() - 1) endDiff   = result.length - line.length;
      return result;
    })
    .join(''));
  range = range.copy();
  range.start.column += startDiff;
  range.end.column += endDiff;
  self.selection.setRange(range);
  return self;
};

Editor._tabRegExp = /\t/g;
Editor.prototype.visiblePos = function (pos) {
  var self = this;
  if (pos instanceof Range) return new Range(self.visiblePos(pos.start), self.visiblePos(pos.end));
  pos = Point.fromObject(pos, true);
  pos.column = self.lineWithEndingForRow(pos.row)
    .slice(0, Math.max(pos.column, 0))
    .replace(Editor._tabRegExp, _.repeat('\t', self.buffer.options.tabSize))
    .length;
  return pos;
};
Editor.prototype.realPos = function (pos) {
  var self = this;
  if (pos instanceof Range) return new Range(self.realPos(pos.start), self.realPos(pos.end));
  pos = Point.fromObject(pos, true);
  pos.column = this.lineWithEndingForRow(Math.min(pos.row, self.textBuf.getLastRow()))
    .replace(Editor._tabRegExp, _.repeat('\t', this.buffer.options.tabSize))
    .slice(0, Math.max(pos.column, 0))
    .replace(new RegExp('\\t{1,'+this.buffer.options.tabSize+'}', 'g'), '\t')
    .length;
  return self.textBuf.clipPosition(pos);
};

Editor.prototype.moveCursorVertical = function (count, paragraphs) {
  var self = this;
  var selection = self.selection;
  var cursor = selection.getHeadPosition().copy();
  if (count < 0 && cursor.row === 0) {
    selection.setHeadPosition(new Point(0, 0));
  } else if (count > 0 && cursor.row === self.textBuf.getLastRow()) {
    selection.setHeadPosition(new Point(Infinity, Infinity));
  } else {
    if (paragraphs) {
      paragraphs = Math.abs(count);
      var direction = count ? paragraphs / count : 0;
      while (paragraphs--) {
        while (true) {
          cursor.row += direction;

          if (!(0 <= cursor.row && cursor.row < self.textBuf.getLastRow())) break;
          if (/^\s*$/g.test(self.textBuf.lineForRow(cursor.row))) break;
        }<commit_msg>fix(movement): fix pageup/down to boundary<commit_after>var _ = require('lodash');
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
var extname = require('path').extname;
var lodash = require('lodash');
require('es6-set/implement'); // required for text-buffer
require('iconv-lite').extendNodeEncodings();
var TextBuffer = require('text-buffer');
var Point = require('text-buffer/lib/point');
var Range = require('text-buffer/lib/range');

var util = require('slap-util');
var word = require('./word');
var editorWidgetOpts = require('./opts');
var highlightClient = require('./highlight/client');

var BaseWidget = require('base-widget');

function Editor (opts) {
  var self = this;

  if (!(self instanceof Editor)) return new Editor(opts);

  BaseWidget.call(self, _.merge({
    focusable: true,
    multiLine: true
  }, editorWidgetOpts.editor, opts));

  self.gutter = new BaseWidget(_.merge({
    parent: self,
    tags: true,
    wrap: false,
    style: {},
    top: 0,
    left: 0,
    bottom: 0
  }, self.options.gutter));

  self.buffer = new BaseWidget(_.merge({
    parent: self,
    tags: true,
    wrap: false,
    style: {},
    top: 0,
    left: self.options.multiLine && !self.gutter.options.hidden ? self.gutter.width : 0,
    right: 0,
    bottom: 0
  }, self.options.buffer));

  self.textBuf = new TextBuffer({encoding: self.options.defaultEncoding});
  self.textBuf.loadSync();

  self.selection = self.textBuf.markPosition(new Point(0, 0), {type: 'selection', invalidate: 'never'});
  self.scroll = new Point(0, 0);
  self.data.updatePreferredX = true;

  self.language(false);
  self.toggleInsertMode();
  self._initHighlighting();

  var _updateContent = self._updateContent.bind(self);
  self._updateContent = lodash.throttle(_updateContent, self.options.perf.updateContentThrottle, false);
  self._updateContent();
}
Editor.prototype.__proto__ = BaseWidget.prototype;

Editor.prototype.open = Promise.method(function (givenPath) {
  // Handles nonexistent paths
  var self = this;
  return self.ready // ensures textBuf path-changed is triggered in _initHandlers
    .then(function () { return Editor.getOpenParams(givenPath); })
    .tap(function (params) { return self.textBuf.setPath(params.path); })
    .tap(function (params) { if (params.exists) return self.textBuf.load(); })
    .tap(function (params) { self.selection.setHeadPosition(params.position); })
    .return(self);
});
Editor.prototype.save = Promise.method(function (path) {
  var self = this;
  return Promise.resolve()
    .then(function () { return path ? self.textBuf.saveAs(util.resolvePath(path)) : self.textBuf.save(); })
    .then(function () { return self.textBuf.getPath(); })
});

// looks for path like /home/dan/file.c:3:8 but matches every string
Editor.openRegExp = new RegExp('^'
  + '(.*?)'          // path:   match[1] (like /home/dan/file.c)
  + '(?:\\:(\\d+))?' // row:    match[2] (like 3, optional)
  + '(?:\\:(\\d+))?' // column: match[3] (like 8, optional)
  + '$');
Editor.getOpenParams = Promise.method(function (givenPath) {
  givenPath = util.resolvePath(givenPath);
  var baseParams = {
    path: givenPath,
    exists: false,
    position: new Point(0, 0)
  };
  var match = givenPath.match(Editor.openRegExp); // always matches
  return [
    // if a path like file.c:3:8 is passed, see if `file.c:3:8` exists first,
    // then try `file.c:3` line 8, then `file.c` line 3 column 8
    baseParams,
    _.merge({}, baseParams, {
      path: match[1]+':'+match[2],
      position: {row: Editor.parseCoordinate(match[3])}
    }),
    _.merge({}, baseParams, {
      path: match[1],
      position: {row: Editor.parseCoordinate(match[2]), column: Editor.parseCoordinate(match[3])}
    })
  ].reduce(function (promise, params) {
    return promise.then(function (resultParams) {
      if ((resultParams || {}).exists) return resultParams;
      return Editor.exists(params.path).then(function (exists) {
        params.exists = exists;
        return params;
      });
    });
  }, Promise.resolve());
});

Editor.parseCoordinate = function (n) { return (parseInt(n, 10) - 1) || 0; };
Editor.exists = Promise.method(function (givenPath) {
  return fs.openAsync(givenPath, 'r')
    .then(function (fd) { return fs.closeAsync(fd); })
    .return(true)
    .catch(function (err) {
      if (err.code !== 'ENOENT') throw err;
      return false;
    });
});

Editor.prototype.insertMode = util.getterSetter('insertMode', null, Boolean);
Editor.prototype.toggleInsertMode = function () { return this.insertMode(!this.insertMode()); };

Editor.prototype.language = util.getterSetter('language', null, null);
Editor.prototype.readOnly = util.getterSetter('readOnly', null, Boolean);
Editor.prototype.lineWithEndingForRow = function (row) {
  var self = this;
  return self.textBuf.lineForRow(row) + self.textBuf.lineEndingForRow(row);
};
Editor.prototype.delete = function (range) {
  var self = this;
  if (!range) {
    self.textBuf.setTextInRange(self.selection.getRange(), '');
  } else {
    self.textBuf.delete(range);
  }
  return self;
};
Editor.prototype._getTabString = function () {
  var self = this;
  return self.buffer.options.useSpaces
    ? _.repeat(' ', self.buffer.options.tabSize)
    : '\t';
};
Editor.prototype.indent = function (range, dedent) {
  var self = this;

  var tabString = self._getTabString();
  var indentRegex = new RegExp('^(\t| {0,'+self.buffer.options.tabSize+'})', 'g');
  var startDiff = 0, endDiff = 0;
  var linesRange = range.copy();
  linesRange.start.column = 0;
  linesRange.end.column = Infinity;
  self.textBuf.setTextInRange(linesRange, util.text.splitLines(self.textBuf.getTextInRange(linesRange))
    .map(function (line, i) {
      var result = !dedent
        ? tabString + line
        : line.replace(indentRegex, '');
      if (i === 0)                       startDiff = result.length - line.length;
      if (i === range.getRowCount() - 1) endDiff   = result.length - line.length;
      return result;
    })
    .join(''));
  range = range.copy();
  range.start.column += startDiff;
  range.end.column += endDiff;
  self.selection.setRange(range);
  return self;
};

Editor._tabRegExp = /\t/g;
Editor.prototype.visiblePos = function (pos) {
  var self = this;
  if (pos instanceof Range) return new Range(self.visiblePos(pos.start), self.visiblePos(pos.end));
  pos = Point.fromObject(pos, true);
  pos.column = self.lineWithEndingForRow(pos.row)
    .slice(0, Math.max(pos.column, 0))
    .replace(Editor._tabRegExp, _.repeat('\t', self.buffer.options.tabSize))
    .length;
  return pos;
};
Editor.prototype.realPos = function (pos) {
  var self = this;
  if (pos instanceof Range) return new Range(self.realPos(pos.start), self.realPos(pos.end));
  pos = Point.fromObject(pos, true);
  pos.column = this.lineWithEndingForRow(self.textBuf.clipPosition(pos).row)
    .replace(Editor._tabRegExp, _.repeat('\t', this.buffer.options.tabSize))
    .slice(0, Math.max(pos.column, 0))
    .replace(new RegExp('\\t{1,'+this.buffer.options.tabSize+'}', 'g'), '\t')
    .length;
  return self.textBuf.clipPosition(pos);
};

Editor.prototype.moveCursorVertical = function (count, paragraphs) {
  var self = this;
  var selection = self.selection;
  var cursor = selection.getHeadPosition().copy();
  if (count < 0 && cursor.row === 0) {
    selection.setHeadPosition(new Point(0, 0));
  } else if (count > 0 && cursor.row === self.textBuf.getLastRow()) {
    selection.setHeadPosition(new Point(Infinity, Infinity));
  } else {
    if (paragraphs) {
      paragraphs = Math.abs(count);
      var direction = count ? paragraphs / count : 0;
      while (paragraphs--) {
        while (true) {
          cursor.row += direction;

          if (!(0 <= cursor.row && cursor.row < self.textBuf.getLastRow())) break;
          if (/^\s*$/g.test(self.textBuf.lineForRow(cursor.row))) break;
        }
{'fraction_non_alphanumeric': 0.16608513607815772, 'fraction_numerical': 0.00723212586436592, 'mean_word_length': 3.6419316843345113, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3338690', 'n_tokens_mistral': 5193, 'n_tokens_neox': 5079, 'n_words': 1109}