<commit_before>'use strict';

var last = require('lodash/last');
var forEach = require('lodash/forEach');
var clone = require('lodash/clone');
var extend = require('lodash/extend');
var oo = require('../util/oo');
// var uuid = require('../util/uuid');
var createCountingIdGenerator = require('../util/createCountingIdGenerator');
var ArrayIterator = require('../util/ArrayIterator');

/**
  A generic base implementation for XML/HTML importers.

  @class
  @param {Object} config
 */
function DOMImporter(config) {
  if (!config.converters) {
    throw new Error('config.converters is mandatory');
  }
  this.config = extend({ idAttribute: 'id' }, config);
  this.schema = config.schema;
  this.state = null;

  this._defaultBlockConverter = null;
  this._allConverters = [];
  this._blockConverters = [];
  this._propertyAnnotationConverters = [];

  var schema = this.schema;
  var defaultTextType = schema.getDefaultTextType();

  config.converters.forEach(function(Converter) {
    var converter;
    if (typeof Converter === 'function') {
      // console.log('installing converter', Converter);
      converter = new Converter();
    } else {
      converter = Converter;
    }
    if (!converter.type) {
      console.error('Converter must provide the type of the associated node.', converter);
      return;
    }
    if (!converter.matchElement && !converter.tagName) {
      console.error('Converter must provide a matchElement function or a tagName property.', converter);
      return;
    }
    if (!converter.matchElement) {
      converter.matchElement = this._defaultElementMatcher.bind(converter);
    }
    var NodeClass = schema.getNodeClass(converter.type);
    if (!NodeClass) {
      console.error('No node type defined for converter', converter.type);
      return;
    }
    if (!this._defaultBlockConverter && defaultTextType === converter.type) {
      this._defaultBlockConverter = converter;
    }

    this._allConverters.push(converter);
    // Defaults to _blockConverters
    if (NodeClass.static.isPropertyAnnotation) {
      this._propertyAnnotationConverters.push(converter);
    } else {
      this._blockConverters.push(converter);
    }

  }.bind(this));

  this.state = new DOMImporter.State();
}

DOMImporter.Prototype = function DOMImporterPrototype() {

  this.reset = function() {
    this.state.reset();
  };

  this.createDocument = function() {
    this.state.doc = this._createDocument(this.config.schema);
    return this.state.doc;
  };

  // Note: this is e.g. shared by ClipboardImporter which has a different
  // implementation of this.createDocument()
  this._createDocument = function(schema) {
    // create an empty document and initialize the container if not present
    var doc = new this.config.DocumentClass(schema);
    return doc;
  };

  // should be called at the end to finish conversion
  // For instance, the creation of annotation is deferred
  // to make sure that the nodes they are attached to are created first
  // TODO: we might want to rethink this in future
  // as it makes this a bit more complicated
  this.generateDocument = function() {
    var doc = this.state.doc;
    if (!doc) {
      doc = this.createDocument();
    }
    // creating all nodes
    this.state.nodes.forEach(function(node) {
      // delete if the node exists already
      if (doc.get(node.id)) {
        doc.delete(node.id);
      }
      doc.create(node);
    });
    // creating annotations afterwards so that the targeted nodes exist for sure
    this.state.inlineNodes.forEach(function(node) {
      if (doc.get(node.id)) {
        doc.delete(node.id);
      }
      doc.create(node);
    });
    return doc;
  };

  /**
    Converts and shows all children of a given element.

    @param {ui/DOMElement[]} elements All elements that should be converted into the container.
    @param {String} containerId The id of the target container node.
    @returns {Object} the preliminary container node
   */
  this.convertContainer = function(elements, containerId) {
    var state = this.state;
    state.container = [];
    state.containerId = containerId;
    var iterator = new ArrayIterator(elements);
    while(iterator.hasNext()) {
      var el = iterator.next();
      var blockTypeConverter = this._getConverterForElement(el, 'block');
      var node;
      if (blockTypeConverter) {
        node = this._nodeData(el, blockTypeConverter.type);
        state.pushContext(el.tagName, blockTypeConverter);
        node = blockTypeConverter.import(el, node, this) || node;
        state.popContext();
        this._createAndShow(node);
      } else {
        if (el.isCommentNode()) {
          // skip HTML comment nodes on block level
        } else if (el.isTextNode()) {
          var text = el.textContent;
          if (/^\s*$/.exec(text)) continue;
          // If we find text nodes on the block level we wrap
          // it into a paragraph element (or what is configured as default block level element)
          iterator.back();
          this._wrapInlineElementsIntoBlockElement(iterator);
        } else if (el.isElementNode()) {
          // NOTE: hard to tell if unsupported nodes on this level
          // should be treated as inline or not.
          // ATM: we apply a catch-all to handle cases where inline content
          // is found on top level
          iterator.back();
          this._wrapInlineElementsIntoBlockElement(iterator);
        }
      }
    }
    var container = {
      type: 'container',
      id: containerId,
      nodes: this.state.container.slice(0)
    };
    this.createNode(container);
    return container;
  };

  /**
    Converts a single HTML element and creates a node in the current document.

    @param {ui/DOMElement} el the HTML element
    @returns {object} the created node as JSON
   */
  this.convertElement = function(el) {
    var doc = this.state.doc;
    var nodeData = this._convertElement(el);
    var node;
    if (doc) {
      node = doc.create(nodeData);
    } else {
      var NodeClass = this.schema.getNodeClass(nodeData.type);
      node = new NodeClass(doc, nodeData);
    }
    return node;
  };

  this._convertElement = function(el, mode) {
    var node;
    var converter = this._getConverterForElement(el, mode);
    if (converter) {
      node = this._nodeData(el, converter.type);
      var NodeClass = this.schema.getNodeClass(node.type);
      this.state.pushContext(el.tagName, converter);
      // Note: special treatment for property annotations and inline nodes
      // i.e. if someone calls `importer.convertElement(annoEl)`
      // usually, annotations are imported in the course of `importer.annotatedText(..)`
      // The peculiarity here is that in such a case, it is not
      // not clear, which property the annotation is attached to
      if (NodeClass.static.isInline) {
        this._convertInlineNode(el, node, converter);
      }
      else if (NodeClass.static.isPropertyAnnotation) {
        this._convertPropertyAnnotation(el, node);
      } else {
        node = converter.import(el, node, this) || node;
      }
      this.state.popContext();
      this.createNode(node);
    } else {
      throw new Error('No converter found for '+el.tagName);
    }
    return node;
  };

  this._convertPropertyAnnotation = function(el, node) {
    // if there is no context, this is called stand-alone
    // i.e., user tries to convert an annotation element
    // directly, not part of a block element, such as a paragraph
    node.path = [node.id, '_content'];
    node._content = this.annotatedText(el, node.path);
    node.startOffset = 0;
    node.endOffset = node._content.length;
  };

  this._convertInlineNode = function(el, node, converter) {
    node.path = [node.id, 'content'];
    node._content = '$';
    node.startOffset = 0;
    node.endOffset = 1;
    return converter.import(el, node, this);
  };

  this.createNode = function(node) {
    if (this.state.ids[node.id]) {
      throw new Error('Node with id alread exists:' + node.id);
    }
    this.state.ids[node.id] = true;
    this.state.nodes.push(node);<commit_msg>Minor refactorings in DOMImporter.<commit_after>'use strict';

var last = require('lodash/last');
var forEach = require('lodash/forEach');
var clone = require('lodash/clone');
var extend = require('lodash/extend');
var oo = require('../util/oo');
// var uuid = require('../util/uuid');
var createCountingIdGenerator = require('../util/createCountingIdGenerator');
var ArrayIterator = require('../util/ArrayIterator');

/**
  A generic base implementation for XML/HTML importers.

  @class
  @param {Object} config
 */
function DOMImporter(config) {
  if (!config.converters) {
    throw new Error('config.converters is mandatory');
  }
  this.config = extend({ idAttribute: 'id' }, config);
  this.schema = config.schema;
  this.state = null;

  this._defaultBlockConverter = null;
  this._allConverters = [];
  this._blockConverters = [];
  this._propertyAnnotationConverters = [];

  var schema = this.schema;
  var defaultTextType = schema.getDefaultTextType();

  config.converters.forEach(function(Converter) {
    var converter;
    if (typeof Converter === 'function') {
      // console.log('installing converter', Converter);
      converter = new Converter();
    } else {
      converter = Converter;
    }
    if (!converter.type) {
      console.error('Converter must provide the type of the associated node.', converter);
      return;
    }
    if (!converter.matchElement && !converter.tagName) {
      console.error('Converter must provide a matchElement function or a tagName property.', converter);
      return;
    }
    if (!converter.matchElement) {
      converter.matchElement = this._defaultElementMatcher.bind(converter);
    }
    var NodeClass = schema.getNodeClass(converter.type);
    if (!NodeClass) {
      console.error('No node type defined for converter', converter.type);
      return;
    }
    if (!this._defaultBlockConverter && defaultTextType === converter.type) {
      this._defaultBlockConverter = converter;
    }

    this._allConverters.push(converter);
    // Defaults to _blockConverters
    if (NodeClass.static.isPropertyAnnotation) {
      this._propertyAnnotationConverters.push(converter);
    } else {
      this._blockConverters.push(converter);
    }

  }.bind(this));

  this.state = new DOMImporter.State();
}

DOMImporter.Prototype = function DOMImporterPrototype() {

  this.reset = function() {
    this.state.reset();
  };

  this.createDocument = function() {
    this.state.doc = this._createDocument(this.config.schema);
    return this.state.doc;
  };

  // Note: this is e.g. shared by ClipboardImporter which has a different
  // implementation of this.createDocument()
  this._createDocument = function(schema) {
    // create an empty document and initialize the container if not present
    var doc = new this.config.DocumentClass(schema);
    return doc;
  };

  // should be called at the end to finish conversion
  // For instance, the creation of annotation is deferred
  // to make sure that the nodes they are attached to are created first
  // TODO: we might want to rethink this in future
  // as it makes this a bit more complicated
  this.generateDocument = function() {
    if (!this.state.doc) {
      this.state.doc = this.createDocument();
    }
    var doc = this.state.doc;
    this._createNodes();
    return doc;
  };

  this._createNodes = function() {
    var state = this.state;
    var doc = state.doc;
    // creating all nodes
    state.nodes.forEach(function(node) {
      // delete if the node exists already
      if (doc.get(node.id)) {
        doc.delete(node.id);
      }
      doc.create(node);
    });
    this._createInlineNodes();
  };

  this._createInlineNodes = function() {
    var state = this.state;
    var doc = state.doc;
    // creating annotations afterwards so that the targeted nodes exist for sure
    state.inlineNodes.forEach(function(node) {
      if (doc.get(node.id)) {
        doc.delete(node.id);
      }
      doc.create(node);
    });
  };

  /**
    Converts and shows all children of a given element.

    @param {ui/DOMElement[]} elements All elements that should be converted into the container.
    @param {String} containerId The id of the target container node.
    @returns {Object} the preliminary container node
   */
  this.convertContainer = function(elements, containerId) {
    var state = this.state;
    state.container = [];
    state.containerId = containerId;
    var iterator = new ArrayIterator(elements);
    while(iterator.hasNext()) {
      var el = iterator.next();
      var blockTypeConverter = this._getConverterForElement(el, 'block');
      var node;
      if (blockTypeConverter) {
        node = this._nodeData(el, blockTypeConverter.type);
        state.pushContext(el.tagName, blockTypeConverter);
        node = blockTypeConverter.import(el, node, this) || node;
        state.popContext();
        this._createAndShow(node);
      } else {
        if (el.isCommentNode()) {
          // skip HTML comment nodes on block level
        } else if (el.isTextNode()) {
          var text = el.textContent;
          if (/^\s*$/.exec(text)) continue;
          // If we find text nodes on the block level we wrap
          // it into a paragraph element (or what is configured as default block level element)
          iterator.back();
          this._wrapInlineElementsIntoBlockElement(iterator);
        } else if (el.isElementNode()) {
          // NOTE: hard to tell if unsupported nodes on this level
          // should be treated as inline or not.
          // ATM: we apply a catch-all to handle cases where inline content
          // is found on top level
          iterator.back();
          this._wrapInlineElementsIntoBlockElement(iterator);
        }
      }
    }
    var container = {
      type: 'container',
      id: containerId,
      nodes: this.state.container.slice(0)
    };
    this.createNode(container);
    return container;
  };

  /**
    Converts a single HTML element and creates a node in the current document.

    @param {ui/DOMElement} el the HTML element
    @returns {object} the created node as JSON
   */
  this.convertElement = function(el) {
    var doc = this.state.doc;
    var nodeData = this._convertElement(el);
    var node;
    if (doc) {
      node = doc.create(nodeData);
    } else {
      var NodeClass = this.schema.getNodeClass(nodeData.type);
      node = new NodeClass(doc, nodeData);
    }
    return node;
  };

  this._convertElement = function(el, mode) {
    var node;
    var converter = this._getConverterForElement(el, mode);
    if (converter) {
      node = this._nodeData(el, converter.type);
      var NodeClass = this.schema.getNodeClass(node.type);
      this.state.pushContext(el.tagName, converter);
      // Note: special treatment for property annotations and inline nodes
      // i.e. if someone calls `importer.convertElement(annoEl)`
      // usually, annotations are imported in the course of `importer.annotatedText(..)`
      // The peculiarity here is that in such a case, it is not
      // not clear, which property the annotation is attached to
      if (NodeClass.static.isInline) {
        this._convertInlineNode(el, node, converter);
      }
      else if (NodeClass.static.isPropertyAnnotation) {
        this._convertPropertyAnnotation(el, node);
      } else {
        node = converter.import(el, node, this) || node;
      }
      this.state.popContext();
      this.createNode(node);
    } else {
      throw new Error('No converter found for '+el.tagName);
    }
    return node;
  };

  this._convertPropertyAnnotation = function(el, node) {
    // if there is no context, this is called stand-alone
    // i.e., user tries to convert an annotation element
    // directly, not part of a block element, such as a paragraph
    node.path = [node.id, '_content'];
    node._content = this.annotatedText(el, node.path);
    node.startOffset = 0;
    node.endOffset = node._content.length;
  };

  this._convertInlineNode = function(el, node, converter) {
    node.path = [node.id, 'content'];
    node._content = '$';
    node.startOffset = 0;
    node.endOffset = 1;
    node = converter.import(el, node, this);
    return node;
  };

  this.createNode = function(node) {
    if (this.state.ids[node.id]) {
      throw new Error('Node with id alread exists:' + node.id);
    }
    this.state.ids[node.id] = true;
    this.state.nodes.push(node);
{'fraction_non_alphanumeric': 0.1163030303030303, 'fraction_numerical': 0.00048484848484848484, 'mean_word_length': 2.812615526802218, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1736965', 'n_tokens_mistral': 4771, 'n_tokens_neox': 4678, 'n_words': 1344}