<commit_before>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package pdfpositional;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.util.List;

import java.io.StringWriter;
import org.apache.pdfbox.exceptions.CryptographyException;

import pdfpositional.exceptions.*;

import org.json.simple.JSONArray;
import org.json.simple.JSONObject;

import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.common.PDStream;
import org.apache.pdfbox.util.PDFTextStripper;
import org.apache.pdfbox.util.TextPosition;
import org.apache.pdfbox.io.RandomAccessFile;


/**
 *
 * @author jonny
 */
public class PdfPositional extends PDFTextStripper {
    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) {
        try {
            // check file param
            if (args.length == 0) {
                throw new ParameterException("No file parameter specified");
            }
            
            String file = args[args.length - 1];
            Pattern patternFile = Pattern.compile("(?i)^[\\w,\\s-()/]+\\.pdf$");
            Matcher matcherFile = patternFile.matcher(file);
        
            // check file is valid format
            if (!matcherFile.find()) {
                throw new ParameterException("File parameter invalid: " + file);
            }

            // check if file exists
            File input = new File(file);
            if(!input.exists()) {
                throw new ParameterException("File does not exist: " + file);
            }

            PdfPositional pdfPositional = new PdfPositional(input);
            pdfPositional.setConversion(new Float(1.388888888889));

            pdfPositional.processParams(args);

            pdfPositional.run();
            
            System.exit(0);
        } catch (ParameterException ex) {
            System.out.println("Parameter Error: " + ex.getMessage());
            System.exit(1);
        } catch (EncryptedDocumentException ex) {
            System.out.println("Encrypted Document Error");
            System.exit(1);
        } catch (IOException ex) {
            System.out.println("IO Error" + ex.getMessage());
            System.exit(1);
        } catch (NumberFormatException ex) {
            System.out.println("NumberFormat Error: " + ex.getMessage());
            System.exit(1);
        }
    }
    
    /**
     * run extraction process
     * @throws FileNotFoundException
     * @throws IOException
     * @throws EncryptedDocumentException
     * @throws ParameterException 
     */
    protected void run () throws FileNotFoundException, IOException, EncryptedDocumentException, ParameterException {
        PDDocument pdfDocument;
        RandomAccessFile scratchFile = null;

        if (this.getMode().equals("scratch")) {
            File tmp = new File("temp.tmp");
            tmp.deleteOnExit();
            scratchFile = new RandomAccessFile(tmp, "rw");
            pdfDocument = PDDocument.loadNonSeq(this.getInputFile(), scratchFile);
        } else {
            pdfDocument = PDDocument.load(this.getInputFile());
        }

        // check for encrypted document
        if (pdfDocument.isEncrypted()) {
            try {
                pdfDocument.decrypt("");
            } catch (CryptographyException | IOException e) {
                pdfDocument.close();
                throw new EncryptedDocumentException();
            }
        }

        List allPages = pdfDocument.getDocumentCatalog().getAllPages();
        if (this.hasPageNumber()) { 
            if (pdfDocument.getNumberOfPages() < this.getPageNumber()) {
                throw new ParameterException("illegal page number");
            }
            PDPage page = (PDPage) allPages.get(this.getPageNumber() - 1);
            PDStream contents = page.getContents();
            if (contents != null) {
                this.processStream(page, page.findResources(), page.getContents().getStream());
                this.addPageDataToPdfData();
                this.writeJSONToOutputStream();
            }
        } else {
            for (int i = 0; i < allPages.size(); i++) {
                System.out.println(i);
                this.setPageNumber(i + 1);
                PDPage page = (PDPage) allPages.get(i);
                try {
                    PDStream contents = page.getContents();

                    if (contents != null) {
                        this.processStream(page, page.findResources(), page.getContents().getStream());
                        this.addPageDataToPdfData();
                        this.writeJSONToOutputStream();
                    }

                } catch (Exception ex) {
                   System.out.println("caught you: " + ex.getMessage()); 
                } finally {
                    page.clear();
                }
            }
        }

        this.destroyOutputStream();
        pdfDocument.close();
        if (scratchFile != null) {
            scratchFile.close();
        }
    }
    
    /**
     * process arguments
     * @param args
     * @throws IOException 
     */
    protected void processParams (String[] args) throws IOException {
        Pattern patternArgument = Pattern.compile("^-{2}([^=]+)[=]([\\s\\S]+)$");
        Matcher matcher;
        for (int i = 0; i < args.length - 1; i++) {<commit_msg>removed debug code<commit_after>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package pdfpositional;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.util.List;

import java.io.StringWriter;
import org.apache.pdfbox.exceptions.CryptographyException;

import pdfpositional.exceptions.*;

import org.json.simple.JSONArray;
import org.json.simple.JSONObject;

import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.common.PDStream;
import org.apache.pdfbox.util.PDFTextStripper;
import org.apache.pdfbox.util.TextPosition;
import org.apache.pdfbox.io.RandomAccessFile;


/**
 *
 * @author jonny
 */
public class PdfPositional extends PDFTextStripper {
    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) {
        try {
            // check file param
            if (args.length == 0) {
                throw new ParameterException("No file parameter specified");
            }
            
            String file = args[args.length - 1];
            Pattern patternFile = Pattern.compile("(?i)^[\\w,\\s-()/]+\\.pdf$");
            Matcher matcherFile = patternFile.matcher(file);
        
            // check file is valid format
            if (!matcherFile.find()) {
                throw new ParameterException("File parameter invalid: " + file);
            }

            // check if file exists
            File input = new File(file);
            if(!input.exists()) {
                throw new ParameterException("File does not exist: " + file);
            }

            PdfPositional pdfPositional = new PdfPositional(input);
            pdfPositional.setConversion(new Float(1.388888888889));

            pdfPositional.processParams(args);

            pdfPositional.run();
            
            System.exit(0);
        } catch (ParameterException ex) {
            System.out.println("Parameter Error: " + ex.getMessage());
            System.exit(1);
        } catch (EncryptedDocumentException ex) {
            System.out.println("Encrypted Document Error");
            System.exit(1);
        } catch (IOException ex) {
            System.out.println("IO Error" + ex.getMessage());
            System.exit(1);
        } catch (NumberFormatException ex) {
            System.out.println("NumberFormat Error: " + ex.getMessage());
            System.exit(1);
        }
    }
    
    /**
     * run extraction process
     * @throws FileNotFoundException
     * @throws IOException
     * @throws EncryptedDocumentException
     * @throws ParameterException 
     */
    protected void run () throws FileNotFoundException, IOException, EncryptedDocumentException, ParameterException {
        PDDocument pdfDocument;
        RandomAccessFile scratchFile = null;

        if (this.getMode().equals("scratch")) {
            File tmp = new File("temp.tmp");
            tmp.deleteOnExit();
            scratchFile = new RandomAccessFile(tmp, "rw");
            pdfDocument = PDDocument.loadNonSeq(this.getInputFile(), scratchFile);
        } else {
            pdfDocument = PDDocument.load(this.getInputFile());
        }

        // check for encrypted document
        if (pdfDocument.isEncrypted()) {
            try {
                pdfDocument.decrypt("");
            } catch (CryptographyException | IOException e) {
                pdfDocument.close();
                throw new EncryptedDocumentException();
            }
        }

        List allPages = pdfDocument.getDocumentCatalog().getAllPages();
        if (this.hasPageNumber()) { 
            if (pdfDocument.getNumberOfPages() < this.getPageNumber()) {
                throw new ParameterException("illegal page number");
            }
            PDPage page = (PDPage) allPages.get(this.getPageNumber() - 1);
            PDStream contents = page.getContents();
            if (contents != null) {
                this.processStream(page, page.findResources(), page.getContents().getStream());
                this.addPageDataToPdfData();
                this.writeJSONToOutputStream();
            }
        } else {
            for (int i = 0; i < allPages.size(); i++) {
                this.setPageNumber(i + 1);
                PDPage page = (PDPage) allPages.get(i);
                try {
                    PDStream contents = page.getContents();

                    if (contents != null) {
                        this.processStream(page, page.findResources(), page.getContents().getStream());
                        this.addPageDataToPdfData();
                        this.writeJSONToOutputStream();
                    }

                } catch (Exception ex) {
                    // do nothing
                } finally {
                    page.clear();
                }
            }
        }

        this.destroyOutputStream();
        pdfDocument.close();
        if (scratchFile != null) {
            scratchFile.close();
        }
    }
    
    /**
     * process arguments
     * @param args
     * @throws IOException 
     */
    protected void processParams (String[] args) throws IOException {
        Pattern patternArgument = Pattern.compile("^-{2}([^=]+)[=]([\\s\\S]+)$");
        Matcher matcher;
        for (int i = 0; i < args.length - 1; i++) {
{'fraction_non_alphanumeric': 0.11343204653622423, 'fraction_numerical': 0.004583112991362595, 'mean_word_length': 1.934316007240755, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '928926', 'n_tokens_mistral': 2988, 'n_tokens_neox': 2885, 'n_words': 658}