<commit_before>/*
 * DocumentValidator
 * Copyright (c) 2013-, <NAME>, All rights reserved.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 3.0 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library.
 */

package org.unigram.docvalidator.parser.markdown;

import org.parboiled.common.StringUtils;
import org.pegdown.Printer;
import org.pegdown.ast.AbbreviationNode;
import org.pegdown.ast.AutoLinkNode;
import org.pegdown.ast.BlockQuoteNode;
import org.pegdown.ast.BulletListNode;
import org.pegdown.ast.CodeNode;
import org.pegdown.ast.DefinitionListNode;
import org.pegdown.ast.DefinitionNode;
import org.pegdown.ast.DefinitionTermNode;
import org.pegdown.ast.ExpImageNode;
import org.pegdown.ast.ExpLinkNode;
import org.pegdown.ast.HeaderNode;
import org.pegdown.ast.HtmlBlockNode;
import org.pegdown.ast.InlineHtmlNode;
import org.pegdown.ast.ListItemNode;
import org.pegdown.ast.MailLinkNode;
import org.pegdown.ast.Node;
import org.pegdown.ast.OrderedListNode;
import org.pegdown.ast.ParaNode;
import org.pegdown.ast.QuotedNode;
import org.pegdown.ast.RefImageNode;
import org.pegdown.ast.RefLinkNode;
import org.pegdown.ast.ReferenceNode;
import org.pegdown.ast.RootNode;
import org.pegdown.ast.SimpleNode;
import org.pegdown.ast.SpecialTextNode;
import org.pegdown.ast.StrikeNode;
import org.pegdown.ast.StrongEmphSuperNode;
import org.pegdown.ast.SuperNode;
import org.pegdown.ast.TableBodyNode;
import org.pegdown.ast.TableCaptionNode;
import org.pegdown.ast.TableCellNode;
import org.pegdown.ast.TableColumnNode;
import org.pegdown.ast.TableHeaderNode;
import org.pegdown.ast.TableNode;
import org.pegdown.ast.TableRowNode;
import org.pegdown.ast.TextNode;
import org.pegdown.ast.VerbatimNode;
import org.pegdown.ast.Visitor;
import org.pegdown.ast.WikiLinkNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.unigram.docvalidator.parser.ParseUtils;
import org.unigram.docvalidator.store.FileContent;
import org.unigram.docvalidator.store.Paragraph;
import org.unigram.docvalidator.store.Section;
import org.unigram.docvalidator.store.Sentence;
import org.unigram.docvalidator.util.DocumentValidatorException;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.parboiled.common.Preconditions.checkArgNotNull;

/**
 * Using Pegdown Parser. <br/>
 *
 * @see https://github.com/sirthias/pegdown
 */
public class ToFileContentSerializer implements Visitor {

  private static final Logger LOG =
      LoggerFactory.getLogger(ToFileContentSerializer.class);

  private FileContent fileContent = null;

  private final Map<String, ReferenceNode> references =
      new HashMap<String, ReferenceNode>();

  private final Map<String, String> abbreviations =
      new HashMap<String, String>();

  private int itemDepth = 0;

  private Section currentSection = null;

  protected void visitChildren(SuperNode node) {
    for (Node child : node.getChildren()) {
      child.accept(this);
    }
  }

  private List<Integer> lineList = null;

  // TODO multi period character not supported
  private String period;

  private List<CandidateSentence> candidateSentences =
      new ArrayList<CandidateSentence>();

  /**
   * Constructor.
   *
   * @param content          FileContent
   * @param listOfLineNumber the list of line number
   * @param parserPeriod     end character of sentence
   */
  public ToFileContentSerializer(FileContent content,
                                 List<Integer> listOfLineNumber,
                                 String parserPeriod) {
    this.fileContent = content;
    this.lineList = listOfLineNumber;
    this.period = parserPeriod;
    currentSection = fileContent.getLastSection();
  }

  /**
   * traverse markdown tree that parsed Pegdown
   * @param astRoot Pegdown RootNode
   *                (markdown tree that is parsed pegdown parser)
   * @return file content that re-parse Pegdown RootNode.
   */
  public FileContent toFileContent(RootNode astRoot) throws DocumentValidatorException {
    try {
      checkArgNotNull(astRoot, "astRoot");
      astRoot.accept(this);
    } catch (Throwable e) {
      LOG.error("Fail to traverse RootNode.");
      throw new DocumentValidatorException("Fail to traverse RootNode.", e);
    }
    return fileContent;
  }

  private void fixSentence() {
    // 1. remain sentence append currentSection
    //TODO need line number
    List<Sentence> sentences = createSentenceList();
    for (Sentence sentence : sentences) {
      currentSection.appendSentence(sentence);
    }
  }

  /**
   * Buffer list of to candidate sentence.
   */
  private final class CandidateSentence {
    private int lineNum;<commit_msg>add exception comment<commit_after>/*
 * DocumentValidator
 * Copyright (c) 2013-, <NAME>, All rights reserved.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 3.0 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library.
 */

package org.unigram.docvalidator.parser.markdown;

import org.parboiled.common.StringUtils;
import org.pegdown.Printer;
import org.pegdown.ast.AbbreviationNode;
import org.pegdown.ast.AutoLinkNode;
import org.pegdown.ast.BlockQuoteNode;
import org.pegdown.ast.BulletListNode;
import org.pegdown.ast.CodeNode;
import org.pegdown.ast.DefinitionListNode;
import org.pegdown.ast.DefinitionNode;
import org.pegdown.ast.DefinitionTermNode;
import org.pegdown.ast.ExpImageNode;
import org.pegdown.ast.ExpLinkNode;
import org.pegdown.ast.HeaderNode;
import org.pegdown.ast.HtmlBlockNode;
import org.pegdown.ast.InlineHtmlNode;
import org.pegdown.ast.ListItemNode;
import org.pegdown.ast.MailLinkNode;
import org.pegdown.ast.Node;
import org.pegdown.ast.OrderedListNode;
import org.pegdown.ast.ParaNode;
import org.pegdown.ast.QuotedNode;
import org.pegdown.ast.RefImageNode;
import org.pegdown.ast.RefLinkNode;
import org.pegdown.ast.ReferenceNode;
import org.pegdown.ast.RootNode;
import org.pegdown.ast.SimpleNode;
import org.pegdown.ast.SpecialTextNode;
import org.pegdown.ast.StrikeNode;
import org.pegdown.ast.StrongEmphSuperNode;
import org.pegdown.ast.SuperNode;
import org.pegdown.ast.TableBodyNode;
import org.pegdown.ast.TableCaptionNode;
import org.pegdown.ast.TableCellNode;
import org.pegdown.ast.TableColumnNode;
import org.pegdown.ast.TableHeaderNode;
import org.pegdown.ast.TableNode;
import org.pegdown.ast.TableRowNode;
import org.pegdown.ast.TextNode;
import org.pegdown.ast.VerbatimNode;
import org.pegdown.ast.Visitor;
import org.pegdown.ast.WikiLinkNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.unigram.docvalidator.parser.ParseUtils;
import org.unigram.docvalidator.store.FileContent;
import org.unigram.docvalidator.store.Paragraph;
import org.unigram.docvalidator.store.Section;
import org.unigram.docvalidator.store.Sentence;
import org.unigram.docvalidator.util.DocumentValidatorException;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.parboiled.common.Preconditions.checkArgNotNull;

/**
 * Using Pegdown Parser. <br/>
 *
 * @see https://github.com/sirthias/pegdown
 */
public class ToFileContentSerializer implements Visitor {

  private static final Logger LOG =
      LoggerFactory.getLogger(ToFileContentSerializer.class);

  private FileContent fileContent = null;

  private final Map<String, ReferenceNode> references =
      new HashMap<String, ReferenceNode>();

  private final Map<String, String> abbreviations =
      new HashMap<String, String>();

  private int itemDepth = 0;

  private Section currentSection = null;

  protected void visitChildren(SuperNode node) {
    for (Node child : node.getChildren()) {
      child.accept(this);
    }
  }

  private List<Integer> lineList = null;

  // TODO multi period character not supported
  private String period;

  private List<CandidateSentence> candidateSentences =
      new ArrayList<CandidateSentence>();

  /**
   * Constructor.
   *
   * @param content          FileContent
   * @param listOfLineNumber the list of line number
   * @param parserPeriod     end character of sentence
   */
  public ToFileContentSerializer(FileContent content,
                                 List<Integer> listOfLineNumber,
                                 String parserPeriod) {
    this.fileContent = content;
    this.lineList = listOfLineNumber;
    this.period = parserPeriod;
    currentSection = fileContent.getLastSection();
  }

  /**
   * traverse markdown tree that parsed Pegdown.
   * @param astRoot Pegdown RootNode
   *                (markdown tree that is parsed pegdown parser)
   * @return file content that re-parse Pegdown RootNode.
   * @exception org.unigram.docvalidator.util.DocumentValidatorException
   * Fail to traverse markdown tree
   *
   */
  public FileContent toFileContent(RootNode astRoot)
      throws DocumentValidatorException {
    try {
      checkArgNotNull(astRoot, "astRoot");
      astRoot.accept(this);
    } catch (Throwable e) {
      LOG.error("Fail to traverse RootNode.");
      throw new DocumentValidatorException("Fail to traverse RootNode.", e);
    }
    return fileContent;
  }

  private void fixSentence() {
    // 1. remain sentence append currentSection
    //TODO need line number
    List<Sentence> sentences = createSentenceList();
    for (Sentence sentence : sentences) {
      currentSection.appendSentence(sentence);
    }
  }

  /**
   * Buffer list of to candidate sentence.
   */
  private final class CandidateSentence {
    private int lineNum;
{'fraction_non_alphanumeric': 0.08920276803488482, 'fraction_numerical': 0.0018959143046734287, 'mean_word_length': 4.03099666189795, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4159969', 'n_tokens_mistral': 3096, 'n_tokens_neox': 3043, 'n_words': 818}