<commit_before>import Delta from 'rich-text/lib/delta';
import Parchment from 'parchment';
import Quill from '../core/quill';
import logger from '../core/logger';
import Module from '../core/module';
import { BlockEmbed } from '../blots/block';

import { AlignStyle } from '../formats/align';
import { BackgroundStyle } from '../formats/background';
import { ColorStyle } from '../formats/color';
import { DirectionStyle } from '../formats/direction';
import { FontStyle } from '../formats/font';
import { SizeStyle } from '../formats/size';

let debug = logger('quill:clipboard');


const STYLE_ATTRIBUTORS = [
  AlignStyle,
  BackgroundStyle,
  ColorStyle,
  DirectionStyle,
  FontStyle,
  SizeStyle
].reduce(function(memo, attr) {
  memo[attr.keyName] = attr;
  return memo;
}, {});


class Clipboard extends Module {
  constructor(quill, options) {
    if (options.matchers !== Clipboard.DEFAULTS.matchers) {
      options.matchers = Clipboard.DEFAULTS.matchers.concat(options.matchers);
    }
    super(quill, options);
    this.quill.root.addEventListener('paste', this.onPaste.bind(this));
    this.container = this.quill.addContainer('ql-clipboard');
    this.container.setAttribute('contenteditable', true);
    this.container.setAttribute('tabindex', -1);
    this.matchers = [];
    this.options.matchers.forEach((pair) => {
      this.addMatcher(...pair);
    });
  }

  addMatcher(selector, matcher) {
    this.matchers.push([selector, matcher]);
  }

  clean() {
    let treeWalker = document.createTreeWalker(
      this.container,
      NodeFilter.SHOW_COMMENT,
      { acceptNode: function(node) { return NodeFilter.FILTER_ACCEPT; } },
      false
    );
    let comments = [];
    while(treeWalker.nextNode()) {
      comments.push(treeWalker.currentNode);
    }
    comments.forEach(function(node) {
      if (node != null && node.parentNode != null) {
        node.parentNode.removeChild(node);
      }
    });
    this.container.normalize();
  }

  convert(html) {
    const DOM_KEY = '__ql-matcher';
    if (typeof html === 'string') {
      this.container.innerHTML = html;
    }
    this.clean();
    let textMatchers = [], elementMatchers = [];
    this.matchers.forEach((pair) => {
      let [selector, matcher] = pair;
      switch (selector) {
        case Node.TEXT_NODE:
          textMatchers.push(matcher);
          break;
        case Node.ELEMENT_NODE:
          elementMatchers.push(matcher);
          break;
        default:
          [].forEach.call(this.container.querySelectorAll(selector), (node) => {
            // TODO use weakmap
            node[DOM_KEY] = node[DOM_KEY] || [];
            node[DOM_KEY].push(matcher);
          });
          break;
      }
    });
    let traverse = (node) => {  // Post-order
      if (node.nodeType === node.TEXT_NODE) {
        return textMatchers.reduce(function(delta, matcher) {
          return matcher(node, delta);
        }, new Delta());
      } else if (node.nodeType === node.ELEMENT_NODE) {
        return [].reduce.call(node.childNodes || [], (delta, childNode) => {
          let childrenDelta = traverse(childNode);
          if (childNode.nodeType === node.ELEMENT_NODE) {
            childrenDelta = elementMatchers.reduce(function(childrenDelta, matcher) {
              return matcher(childNode, childrenDelta);
            }, childrenDelta);
            childrenDelta = (childNode[DOM_KEY] || []).reduce(function(childrenDelta, matcher) {
              return matcher(childNode, childrenDelta);
            }, childrenDelta);
          }
          return delta.concat(childrenDelta);
        }, new Delta());
      }
    };
    let delta = traverse(this.container);
    // Remove trailing newline
    if (deltaEndsWith(delta, '\n') && delta.ops[delta.ops.length - 1].attributes == null) {
      delta = delta.compose(new Delta().retain(delta.length() - 1).delete(1));
    }
    debug.log('convert', this.container.innerHTML, delta);
    this.container.innerHTML = '';
    return delta;
  }
<commit_msg>dont need to preprocess comments, just ignore<commit_after>import Delta from 'rich-text/lib/delta';
import Parchment from 'parchment';
import Quill from '../core/quill';
import logger from '../core/logger';
import Module from '../core/module';
import { BlockEmbed } from '../blots/block';

import { AlignStyle } from '../formats/align';
import { BackgroundStyle } from '../formats/background';
import { ColorStyle } from '../formats/color';
import { DirectionStyle } from '../formats/direction';
import { FontStyle } from '../formats/font';
import { SizeStyle } from '../formats/size';

let debug = logger('quill:clipboard');


const STYLE_ATTRIBUTORS = [
  AlignStyle,
  BackgroundStyle,
  ColorStyle,
  DirectionStyle,
  FontStyle,
  SizeStyle
].reduce(function(memo, attr) {
  memo[attr.keyName] = attr;
  return memo;
}, {});


class Clipboard extends Module {
  constructor(quill, options) {
    if (options.matchers !== Clipboard.DEFAULTS.matchers) {
      options.matchers = Clipboard.DEFAULTS.matchers.concat(options.matchers);
    }
    super(quill, options);
    this.quill.root.addEventListener('paste', this.onPaste.bind(this));
    this.container = this.quill.addContainer('ql-clipboard');
    this.container.setAttribute('contenteditable', true);
    this.container.setAttribute('tabindex', -1);
    this.matchers = [];
    this.options.matchers.forEach((pair) => {
      this.addMatcher(...pair);
    });
  }

  addMatcher(selector, matcher) {
    this.matchers.push([selector, matcher]);
  }

  convert(html) {
    const DOM_KEY = '__ql-matcher';
    if (typeof html === 'string') {
      this.container.innerHTML = html;
    }
    let textMatchers = [], elementMatchers = [];
    this.matchers.forEach((pair) => {
      let [selector, matcher] = pair;
      switch (selector) {
        case Node.TEXT_NODE:
          textMatchers.push(matcher);
          break;
        case Node.ELEMENT_NODE:
          elementMatchers.push(matcher);
          break;
        default:
          [].forEach.call(this.container.querySelectorAll(selector), (node) => {
            // TODO use weakmap
            node[DOM_KEY] = node[DOM_KEY] || [];
            node[DOM_KEY].push(matcher);
          });
          break;
      }
    });
    let traverse = (node) => {  // Post-order
      if (node.nodeType === node.TEXT_NODE) {
        return textMatchers.reduce(function(delta, matcher) {
          return matcher(node, delta);
        }, new Delta());
      } else if (node.nodeType === node.ELEMENT_NODE) {
        return [].reduce.call(node.childNodes || [], (delta, childNode) => {
          let childrenDelta = traverse(childNode);
          if (childNode.nodeType === node.ELEMENT_NODE) {
            childrenDelta = elementMatchers.reduce(function(childrenDelta, matcher) {
              return matcher(childNode, childrenDelta);
            }, childrenDelta);
            childrenDelta = (childNode[DOM_KEY] || []).reduce(function(childrenDelta, matcher) {
              return matcher(childNode, childrenDelta);
            }, childrenDelta);
          }
          return delta.concat(childrenDelta);
        }, new Delta());
      } else {
        return new Delta();
      }
    };
    let delta = traverse(this.container);
    // Remove trailing newline
    if (deltaEndsWith(delta, '\n') && delta.ops[delta.ops.length - 1].attributes == null) {
      delta = delta.compose(new Delta().retain(delta.length() - 1).delete(1));
    }
    debug.log('convert', this.container.innerHTML, delta);
    this.container.innerHTML = '';
    return delta;
  }

{'fraction_non_alphanumeric': 0.14829500396510706, 'fraction_numerical': 0.00105736188210415, 'mean_word_length': 2.762804574838389, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6846542', 'n_tokens_mistral': 2253, 'n_tokens_neox': 2235, 'n_words': 436}