<commit_before>import JsDom from 'jsdom';
import CollapseWhitespace from 'collapse-whitespace';
import Util from './util.js';
import MarkdownConverters from './markdown.js';
import GfmConverters from './gfm.js';

/**
 * Types of node.
 * @type {Object}
 * @see https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeType
 */
const NodeTypes = {
  ELEMENT_NODE: 1,
  TEXT_NODE: 3
};

/**
 * RegExp.
 * @type {Object}
 */
const RegExps = {
  Alphabet: /A/,
  Space: /^\s*$/i,
  Leading: /^[ \r\n\t]/,
  Trailing: /[ \r\n\t]$/
};

/**
 * Convert the WordPress's post to Markdown.
 * Design and implementation was in reference to the npm to-markdown.
 *
 * @see https://github.com/domchristie/to-markdown
 */
export default class Converter {
  /**
   * Check that conversion is possible.
   *
   * @param {Node} node                      DOM node.
   * @param {String|Array.<String>|Function} filter Filter.
   *
   * @return {Boolean} "true" if the conversion is possible.
   */
  static canConvert( node, filter ) {
    if( typeof filter === 'string' ) {
      return filter === node.nodeName.toLowerCase();
    }

    if( Array.isArray( filter ) ) {
      return filter.indexOf( node.nodeName.toLowerCase() ) !== -1;
    } else if( typeof filter === 'function' ) {
      return filter( node );
    }

    throw new TypeError( '"filter" needs to be a string, array, or function' );
  }

  /**
   * Collapse the whitespace from ELEMENT_NODE node.
   * TEXT_NODE will keep the original indentation and whitespace.
   *
   * @param {Array.<Node>} nodes DOM nodes.
   */
  static collapseWhitespace( nodes ) {
    nodes.forEach( ( node ) => {
      if( node.nodeType === NodeTypes.ELEMENT_NODE ) {
        CollapseWhitespace( node, Util.isBlockElement );
      }
    } );
  }

  /**
   * Convert the WordPress's post to Markdown.
   *
   * @param {String} post    WordPress's post text.
   * @param {Object} options Options.
   *
   * @return {String} Markdown text.
   */
  static convert( post, options = {} ) {
    return new Promise( ( resolve, reject ) => {
      if( typeof post !== 'string' ) {
        return reject( new TypeError( '"post" is not a string.' ) );
      }

      let converters = MarkdownConverters.slice( 0 );
      if( options.gmf ) {
        converters = GfmConverters.concat( converters );
      }

      if( options.converters ) {
        converters = options.converters.concat( converters );
      }

      const body  = JsDom.jsdom( post.replace( /(\d+)\. /g, '$1\\. ' ) ).body;
      const nodes = Converter.flattenNodes( body );
      Converter.collapseWhitespace( nodes );

      // Process through nodes in reverse ( so deepest child elements are first ).
      for( let i = nodes.length - 1; 0 <= i; --i ) {
        Converter.process( nodes[ i ], converters );
      }

      let result = Converter.getContent( body );
      result = result.replace( /^[\t\r\n]+|[\t\r\n\s]+$/g, '' )
                     .replace( /\n\s+\n/g, '\n\n' )
                     .replace( /\n{3,}/g, '\n\n' );

      return resolve( result );
    } );
  }

  /**
   * Flanking the whitespaces.
   *
   * @param {Node} node DOM node.
   *
   * @return {Object} whitespaces.
   */
  static flankingWhitespace( node ) {
    let leading  = '';
    let trailing = '';

    if( !( Util.isBlockElement( node ) ) ) {
      const hasLeading  = RegExps.Leading.test( node.innerHTML );
      const hasTrailing = RegExps.Trailing.test( node.innerHTML );
      if( hasLeading && !( Converter.isFlankedByWhitespace( 'left', node ) ) ) {
        leading = ' ';
      }

      if( hasTrailing && !( Converter.isFlankedByWhitespace( 'right', node ) ) ) {
        trailing = ' ';
      }
    }

    return { leading: leading, trailing: trailing };
  }

  /**
   * Flattens the tree structure of nodes.
   *
   * @param {Node} node DOM node.
   *
   * @return {Array.<Node>} Nodes.
   */
  static flattenNodes( node ) {
    const inqueue  = [ node ];
    const outqueue = [];

    while( 0 < inqueue.length ) {
      const elm = inqueue.shift();
      outqueue.push( elm );

      for( let i = 0, max = elm.childNodes.length; i < max; ++i ) {
        const child = elm.childNodes[ i ];
        if( child.nodeType === NodeTypes.ELEMENT_NODE ) {
          inqueue.push( child );
        }
      }
    }

    outqueue.shift(); // Remove root
    return outqueue;
  }

  /**
   * Get a child contents text.
   *
   * @param {Node} node DOM node.
   *
   * @return {Text} Text.
   */
  static getContent( node ) {
    let text = '';
    for( let i = 0, max = node.childNodes.length; i < max; ++i ) {
      const elm = node.childNodes[ i ];
      if( elm.nodeType === NodeTypes.ELEMENT_NODE ) {
        text += node.childNodes[ i ]._replacement;
      } else if ( elm.nodeType === NodeTypes.TEXT_NODE ) {
        text += elm.data;
      }
    }

    return text;
  }

  /**
   * Check a flanked by whitespace.
   *
   * @param {String} side
   * @param {Node}   node Node.
   *
   * @return {Boolean} Flanked if "true".
   */
  static isFlankedByWhitespace( side, node ) {
    let sibling = null;
    let regexp  = null;
    if( side === 'left' ) {
      sibling = node.previousSibling;
      regexp  = / $/;
    } else {<commit_msg>#21 #18 Add a convert shortcode<commit_after>import JsDom from 'jsdom';
import CollapseWhitespace from 'collapse-whitespace';
import Util from './util.js';
import Shortcode from './shortcode.js';
import MarkdownConverters from './markdown.js';
import GfmConverters from './gfm.js';

/**
 * Types of node.
 * @type {Object}
 * @see https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeType
 */
const NodeTypes = {
  ELEMENT_NODE: 1,
  TEXT_NODE: 3
};

/**
 * RegExp.
 * @type {Object}
 */
const RegExps = {
  Alphabet: /A/,
  Space: /^\s*$/i,
  Leading: /^[ \r\n\t]/,
  Trailing: /[ \r\n\t]$/
};

/**
 * Convert the WordPress's post to Markdown.
 * Design and implementation was in reference to the npm to-markdown.
 *
 * @see https://github.com/domchristie/to-markdown
 */
export default class Converter {
  /**
   * Check that conversion is possible.
   *
   * @param {Node} node                      DOM node.
   * @param {String|Array.<String>|Function} filter Filter.
   *
   * @return {Boolean} "true" if the conversion is possible.
   */
  static canConvert( node, filter ) {
    if( typeof filter === 'string' ) {
      return filter === node.nodeName.toLowerCase();
    }

    if( Array.isArray( filter ) ) {
      return filter.indexOf( node.nodeName.toLowerCase() ) !== -1;
    } else if( typeof filter === 'function' ) {
      return filter( node );
    }

    throw new TypeError( '"filter" needs to be a string, array, or function' );
  }

  /**
   * Collapse the whitespace from ELEMENT_NODE node.
   * TEXT_NODE will keep the original indentation and whitespace.
   *
   * @param {Array.<Node>} nodes DOM nodes.
   */
  static collapseWhitespace( nodes ) {
    nodes.forEach( ( node ) => {
      if( node.nodeType === NodeTypes.ELEMENT_NODE ) {
        CollapseWhitespace( node, Util.isBlockElement );
      }
    } );
  }

  /**
   * Convert the WordPress's post to Markdown.
   *
   * @param {String} post    WordPress's post text.
   * @param {Object} options Options.
   *
   * @return {String} Markdown text.
   */
  static convert( post, options = {} ) {
    return new Promise( ( resolve, reject ) => {
      if( typeof post !== 'string' ) {
        return reject( new TypeError( '"post" is not a string.' ) );
      }

      let converters = MarkdownConverters.slice( 0 );
      if( options.gfm ) {
        converters = GfmConverters.concat( converters );
      }

      if( options.converters ) {
        converters = options.converters.concat( converters );
      }

      const body  = JsDom.jsdom( post.replace( /(\d+)\. /g, '$1\\. ' ) ).body;
      const nodes = Converter.flattenNodes( body );
      Converter.collapseWhitespace( nodes );

      // Process through nodes in reverse ( so deepest child elements are first ).
      for( let i = nodes.length - 1; 0 <= i; --i ) {
        Converter.process( nodes[ i ], converters );
      }

      let result = Converter.getContent( body );
      result = result.replace( /^[\t\r\n]+|[\t\r\n\s]+$/g, '' )
                     .replace( /\n\s+\n/g, '\n\n' )
                     .replace( /\n{3,}/g, '\n\n' );

      return resolve( result );
    } );
  }

  /**
   * Flanking the whitespaces.
   *
   * @param {Node} node DOM node.
   *
   * @return {Object} whitespaces.
   */
  static flankingWhitespace( node ) {
    let leading  = '';
    let trailing = '';

    if( !( Util.isBlockElement( node ) ) ) {
      const hasLeading  = RegExps.Leading.test( node.innerHTML );
      const hasTrailing = RegExps.Trailing.test( node.innerHTML );
      if( hasLeading && !( Converter.isFlankedByWhitespace( 'left', node ) ) ) {
        leading = ' ';
      }

      if( hasTrailing && !( Converter.isFlankedByWhitespace( 'right', node ) ) ) {
        trailing = ' ';
      }
    }

    return { leading: leading, trailing: trailing };
  }

  /**
   * Flattens the tree structure of nodes.
   *
   * @param {Node} node DOM node.
   *
   * @return {Array.<Node>} Nodes.
   */
  static flattenNodes( node ) {
    const inqueue  = [ node ];
    const outqueue = [];

    while( 0 < inqueue.length ) {
      const elm = inqueue.shift();
      outqueue.push( elm );

      for( let i = 0, max = elm.childNodes.length; i < max; ++i ) {
        const child = elm.childNodes[ i ];
        if( child.nodeType === NodeTypes.ELEMENT_NODE ) {
          inqueue.push( child );
        }
      }
    }

    outqueue.shift(); // Remove root
    return outqueue;
  }

  /**
   * Get a child contents text.
   *
   * @param {Node} node DOM node.
   *
   * @return {Text} Text.
   */
  static getContent( node ) {
    let text = '';
    for( let i = 0, max = node.childNodes.length; i < max; ++i ) {
      const elm = node.childNodes[ i ];
      if( elm.nodeType === NodeTypes.ELEMENT_NODE ) {
        text += node.childNodes[ i ]._replacement;
      } else if ( elm.nodeType === NodeTypes.TEXT_NODE ) {
        text += elm.data;
      }
    }

    return text;
  }

  static initText( text ) {
    const result = Shortcode.convert( text );

    // Escape number list
    return result.replace( /(\d+)\. /g, '$1\\. ' );
  }

  /**
   * Check a flanked by whitespace.
   *
   * @param {String} side
   * @param {Node}   node Node.
   *
   * @return {Boolean} Flanked if "true".
   */
  static isFlankedByWhitespace( side, node ) {
    let sibling = null;
    let regexp  = null;
    if( side === 'left' ) {
      sibling = node.previousSibling;
      regexp  = / $/;
    } else {
{'fraction_non_alphanumeric': 0.15192488262910797, 'fraction_numerical': 0.0025352112676056337, 'mean_word_length': 2.1813022700119475, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1030666', 'n_tokens_mistral': 3601, 'n_tokens_neox': 3520, 'n_words': 916}