<commit_before>/*
PdfContentImporter is part of JabRef. 
Copyright (C) 2011 <NAME>

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
or see http://www.gnu.org/licenses/gpl-2.0.html
*/

package net.sf.jabref.imports;

import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.util.PDFTextStripper;

import net.sf.jabref.BibtexEntry;
import net.sf.jabref.BibtexEntryType;
import net.sf.jabref.Globals;
import net.sf.jabref.OutputPrinter;

/**
 * PdfContentImporter parses data of the first page of the PDF and creates a BibTeX entry.
 * 
 * Currently, Springer and IEEE formats are supported.
 * 
 * Integrating XMP support is future work
 * 
 * @author koppor
 *
 */
public class PdfContentImporter extends ImportFormat {
	
	private static Logger logger = Logger.getLogger(PdfContentImporter.class.getName());

	/* global variables holding the state of the current parse run
	 * needed to be able to generate methods such as "fillCurStringWithNonEmptyLines"
	 */
	
	// input split into several lines
	private String[] split; 
	
	// current index in split
    private int i;
    
    // curent "line" in split.
    // sometimes, a "line" is several lines in split
    private String curString;
    
	private String year = null;

	@Override
	public boolean isRecognizedFormat(InputStream in) throws IOException {
		return false;
	}

	/**
	 * Removes all non-letter characters at the end
	 * 
	 * EXCEPTION: a closing bracket is NOT removed
	 * 
	 * @param input
	 * @return
	 * TODO Additionally repalce multiple subsequent spaces by one space
	 */
	private String removeNonLettersAtEnd(String input) {
		input = input.trim();
		if (input.length() == 0)
			return input;
		char lastC = input.charAt(input.length()-1);
		while (!Character.isLetter(lastC) &&  (lastC!=')')) {
			// if there is an asterix, a dot or something else at the end: remove it
			input = input.substring(0, input.length()-1);
			if (input.length()>0) {
				lastC = input.charAt(input.length()-1);
			} else {
				break;
			}
		}
		return input;
	}
	
	private String streamlineNames(String names) {
		String res;
		// supported formats:
		//   <NAME>, <NAME>, <NAME>, and <NAME>
		if (names.contains(",")) {
			String[] splitNames = names.split(",");
			res = "";
			boolean isFirst = true;
			for (int i=0; i<splitNames.length; i++) {
				String curName = removeNonLettersAtEnd(splitNames[i]);
				if (curName.indexOf("and")==0) {
					// skip possible ands between names
					curName = curName.substring(3).trim();
				} else {
					int posAnd = curName.indexOf(" and "); 
					if (posAnd>=0) {
						String nameBefore = curName.substring(0, posAnd);
						// cannot be first name as "," is contained in the string
						res = res.concat(" and ").concat(removeNonLettersAtEnd(nameBefore));
						curName = curName.substring(posAnd+5);
					}
				}
					
				if (!curName.equals("")) {
					if (curName.equalsIgnoreCase("et al."))
						curName = "others";
					if (isFirst) {
						isFirst = false;
					} else {
						res = res.concat(" and ");
					}
					res = res.concat(curName);
				}
			}
		} else {
			// assumption: names separated by space
			
			String[] splitNames = names.split(" ");
			if (splitNames.length == 0) {
				// empty names... something was really wrong...
				return "";
			}
			
			boolean workedOnFirstOrMiddle = false;
			boolean isFirst = true;
			int i=0;
			res = "";
			do {
				if (!workedOnFirstOrMiddle) {
					if (splitNames[i].equalsIgnoreCase("and")) {
						// do nothing, just increment i at the end of this iteration
					} else {
						if (isFirst) {
							isFirst = false;
						} else {
							res = res.concat(" and ");
						}
						if ((splitNames[i].equalsIgnoreCase("et")) && (splitNames.length>i+1) && (splitNames[i+1].equalsIgnoreCase("al."))) {
							res = res.concat("others");
							break;
						} else {
							res = res.concat(splitNames[i]).concat(" ");
							workedOnFirstOrMiddle = true;
						}
					}
				} else {
					// last item was a first or a middle name
					// we have to check whether we are on a middle name
					// if not, just add the item as last name and add an "and"
					if (splitNames[i].contains(".")) {
						// we found a middle name
						res = res.concat(splitNames[i]).concat(" ");
					} else {
						// last name found
						res = res.concat(removeNonLettersAtEnd(splitNames[i]));
						if (Character.isLowerCase(splitNames[i].charAt(0))) {
							// it is probably be "van", "vom", ...
							// we just rely on the fact that these things are written in lower case letters
							// do NOT finish name
							res = res.concat(" ");
						} else {
							// finish this name
							workedOnFirstOrMiddle = false;
						}
					}
				}
				i++;
			} while (i<splitNames.length);
			
		}
		return res;
	}
	
	private String streamlineTitle(String title) {
		return removeNonLettersAtEnd(title);
	}
	
	private boolean isYear(String yearStr) {
		try {
			Integer.parseInt(yearStr);
			return true;
		} catch (NumberFormatException e) {
			return false;
		}
	}
	
	@Override<commit_msg>fixes PdfContentImporter.java to check if splitNames[i] isn't empty.<commit_after>/*
PdfContentImporter is part of JabRef. 
Copyright (C) 2011 <NAME>

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
or see http://www.gnu.org/licenses/gpl-2.0.html
*/

package net.sf.jabref.imports;

import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.util.PDFTextStripper;

import net.sf.jabref.BibtexEntry;
import net.sf.jabref.BibtexEntryType;
import net.sf.jabref.Globals;
import net.sf.jabref.OutputPrinter;

/**
 * PdfContentImporter parses data of the first page of the PDF and creates a BibTeX entry.
 * 
 * Currently, Springer and IEEE formats are supported.
 * 
 * Integrating XMP support is future work
 * 
 * @author koppor
 *
 */
public class PdfContentImporter extends ImportFormat {
	
	private static Logger logger = Logger.getLogger(PdfContentImporter.class.getName());

	/* global variables holding the state of the current parse run
	 * needed to be able to generate methods such as "fillCurStringWithNonEmptyLines"
	 */
	
	// input split into several lines
	private String[] split; 
	
	// current index in split
    private int i;
    
    // curent "line" in split.
    // sometimes, a "line" is several lines in split
    private String curString;
    
	private String year = null;

	@Override
	public boolean isRecognizedFormat(InputStream in) throws IOException {
		return false;
	}

	/**
	 * Removes all non-letter characters at the end
	 * 
	 * EXCEPTION: a closing bracket is NOT removed
	 * 
	 * @param input
	 * @return
	 * TODO Additionally repalce multiple subsequent spaces by one space
	 */
	private String removeNonLettersAtEnd(String input) {
		input = input.trim();
		if (input.length() == 0)
			return input;
		char lastC = input.charAt(input.length()-1);
		while (!Character.isLetter(lastC) &&  (lastC!=')')) {
			// if there is an asterix, a dot or something else at the end: remove it
			input = input.substring(0, input.length()-1);
			if (input.length()>0) {
				lastC = input.charAt(input.length()-1);
			} else {
				break;
			}
		}
		return input;
	}
	
	private String streamlineNames(String names) {
		String res;
		// supported formats:
		//   <NAME>, <NAME>, <NAME>, and <NAME>
		if (names.contains(",")) {
			String[] splitNames = names.split(",");
			res = "";
			boolean isFirst = true;
			for (int i=0; i<splitNames.length; i++) {
				String curName = removeNonLettersAtEnd(splitNames[i]);
				if (curName.indexOf("and")==0) {
					// skip possible ands between names
					curName = curName.substring(3).trim();
				} else {
					int posAnd = curName.indexOf(" and "); 
					if (posAnd>=0) {
						String nameBefore = curName.substring(0, posAnd);
						// cannot be first name as "," is contained in the string
						res = res.concat(" and ").concat(removeNonLettersAtEnd(nameBefore));
						curName = curName.substring(posAnd+5);
					}
				}
					
				if (!curName.equals("")) {
					if (curName.equalsIgnoreCase("et al."))
						curName = "others";
					if (isFirst) {
						isFirst = false;
					} else {
						res = res.concat(" and ");
					}
					res = res.concat(curName);
				}
			}
		} else {
			// assumption: names separated by space
			
			String[] splitNames = names.split(" ");
			if (splitNames.length == 0) {
				// empty names... something was really wrong...
				return "";
			}
			
			boolean workedOnFirstOrMiddle = false;
			boolean isFirst = true;
			int i=0;
			res = "";
			do {
				if (!workedOnFirstOrMiddle) {
					if (splitNames[i].equalsIgnoreCase("and")) {
						// do nothing, just increment i at the end of this iteration
					} else {
						if (isFirst) {
							isFirst = false;
						} else {
							res = res.concat(" and ");
						}
						if ((splitNames[i].equalsIgnoreCase("et")) && (splitNames.length>i+1) && (splitNames[i+1].equalsIgnoreCase("al."))) {
							res = res.concat("others");
							break;
						} else {
							res = res.concat(splitNames[i]).concat(" ");
							workedOnFirstOrMiddle = true;
						}
					}
				} else {
					// last item was a first or a middle name
					// we have to check whether we are on a middle name
					// if not, just add the item as last name and add an "and"
					if (splitNames[i].contains(".")) {
						// we found a middle name
						res = res.concat(splitNames[i]).concat(" ");
					} else {
						// last name found
						res = res.concat(removeNonLettersAtEnd(splitNames[i]));
						
						if (splitNames[i].length() > 0 && Character.isLowerCase(splitNames[i].charAt(0))) {
							// it is probably be "van", "vom", ...
							// we just rely on the fact that these things are written in lower case letters
							// do NOT finish name
							res = res.concat(" ");
						} else {
							// finish this name
							workedOnFirstOrMiddle = false;
						}
					}
				}
				i++;
			} while (i<splitNames.length);
			
		}
		return res;
	}
	
	private String streamlineTitle(String title) {
		return removeNonLettersAtEnd(title);
	}
	
	private boolean isYear(String yearStr) {
		try {
			Integer.parseInt(yearStr);
			return true;
		} catch (NumberFormatException e) {
			return false;
		}
	}
	
	@Override
{'fraction_non_alphanumeric': 0.11616077384923282, 'fraction_numerical': 0.005920613742494997, 'mean_word_length': 2.911611219830398, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4551202', 'n_tokens_mistral': 4491, 'n_tokens_neox': 3602, 'n_words': 1184}