<commit_before>/* $Id$ */
package net.sf.mmm.search.parser.impl;

import java.io.InputStream;
import java.util.Properties;

import org.pdfbox.pdfparser.PDFParser;
import org.pdfbox.pdmodel.PDDocument;
import org.pdfbox.pdmodel.PDDocumentInformation;
import org.pdfbox.util.PDFTextStripper;

import net.sf.mmm.search.parser.base.AbstractContentParser;

/**
 * This is the implementation of the
 * {@link net.sf.mmm.search.parser.api.ContentParser} interface for PDF
 * documents (content with the mimetype "application/pdf").
 * 
 * @author <NAME> (hohwille at users.sourceforge.net)
 */
public class ContentParserPdf extends AbstractContentParser {

  /**
   * The constructor
   */
  public ContentParserPdf() {

    super();
  }

  /**
   * @see net.sf.mmm.search.parser.api.ContentParser#parse(java.io.InputStream,
   *      long)
   */
  public Properties parse(InputStream inputStream, long filesize) throws Exception {

    Properties properties = new Properties();
    PDFParser parser = new PDFParser(inputStream);
    parser.parse();
    PDDocument pdfDoc = parser.getPDDocument();
    try {
      if (pdfDoc.isEncrypted()) {
        // pdfDoc.decrypt("password");
        return properties;
      }
      PDDocumentInformation info = pdfDoc.getDocumentInformation();
      String title = info.getTitle();
      if (title != null) {
        properties.setProperty(PROPERTY_KEY_TITLE, title);
      }
      String keywords = info.getKeywords();
      if (keywords != null) {
        properties.setProperty(PROPERTY_KEY_KEYWORDS, keywords);
      }
      String author = info.getAuthor();
      if (author != null) {
        properties.setProperty(PROPERTY_KEY_AUTHOR, author);
      }

      PDFTextStripper stripper = new PDFTextStripper();
      properties.setProperty(PROPERTY_KEY_TEXT, stripper.getText(pdfDoc));
    } finally {
      pdfDoc.close();
    }
    return properties;
  }

}<commit_msg>low memory consumption feature implemented<commit_after>/* $Id$ */
package net.sf.mmm.search.parser.impl;

import java.io.InputStream;
import java.util.Properties;

import org.pdfbox.pdfparser.PDFParser;
import org.pdfbox.pdmodel.PDDocument;
import org.pdfbox.pdmodel.PDDocumentInformation;
import org.pdfbox.util.PDFTextStripper;

import net.sf.mmm.search.parser.base.AbstractContentParser;

/**
 * This is the implementation of the
 * {@link net.sf.mmm.search.parser.api.ContentParser} interface for PDF
 * documents (content with the mimetype "application/pdf").
 * 
 * @author <NAME> (hohwille at users.sourceforge.net)
 */
public class ContentParserPdf extends AbstractContentParser {

  /**
   * The constructor
   */
  public ContentParserPdf() {

    super();
  }

  /**
   * @see net.sf.mmm.search.parser.api.ContentParser#parse(java.io.InputStream,
   *      long)
   */
  public Properties parse(InputStream inputStream, long filesize) throws Exception {

    Properties properties = new Properties();
    PDFParser parser = new PDFParser(inputStream);
    parser.parse();
    PDDocument pdfDoc = parser.getPDDocument();
    try {
      if (pdfDoc.isEncrypted()) {
        // pdfDoc.decrypt("password");
        return properties;
      }
      PDDocumentInformation info = pdfDoc.getDocumentInformation();
      String title = info.getTitle();
      if (title != null) {
        properties.setProperty(PROPERTY_KEY_TITLE, title);
      }
      String keywords = info.getKeywords();
      if (keywords != null) {
        properties.setProperty(PROPERTY_KEY_KEYWORDS, keywords);
      }
      String author = info.getAuthor();
      if (author != null) {
        properties.setProperty(PROPERTY_KEY_AUTHOR, author);
      }

      if (filesize < getMaximumBufferSize()) {
        PDFTextStripper stripper = new PDFTextStripper();
        properties.setProperty(PROPERTY_KEY_TEXT, stripper.getText(pdfDoc));        
      }
    } finally {
      pdfDoc.close();
    }
    return properties;
  }

}
{'fraction_non_alphanumeric': 0.11567926455566906, 'fraction_numerical': 0.0, 'mean_word_length': 3.2437703141928496, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7326977', 'n_tokens_mistral': 1153, 'n_tokens_neox': 1152, 'n_words': 239}