<commit_before>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package org.sinarproject.hansardparser;

import com.itextpdf.text.DocumentException;
import java.io.IOException;
import static java.lang.System.out;
import java.util.Set;
import java.util.TreeSet;

import com.itextpdf.text.pdf.PdfDictionary;
import com.itextpdf.text.pdf.PdfName;
import com.itextpdf.text.pdf.PdfReader;
import com.itextpdf.text.pdf.PdfString;
import com.itextpdf.text.pdf.parser.PdfTextExtractor;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 *
 * @author leow
 */
public class HansardParser {

    /**
     * The resulting PDF file.
     */
    public static String SOURCE
            = "example/DR-PARLIMEN/DR-18062015.PDF";
    // PdfReader for multiple uses?
    private static PdfReader my_reader;

    /**
     * @param args the command line arguments
     * @throws java.io.IOException
     */
    public static void main(String[] args) throws IOException {
        // TODO code application logic here
        out.println("Sinar Project's Hansard Parser ..");
        PdfReader reader = new PdfReader(HansardParser.SOURCE);
        // Assign it for later reuse ..
        HansardParser.my_reader = reader;
        Map<Integer, List<String>> myHalamanHash;
        myHalamanHash = HansardParser.getHalaman(reader);
        // Itertae thoug it .. use example here and here ..
        HansardParser.splitHalamanbyTopic(myHalamanHash);

        int n;
        // n = reader.getNumberOfPages();
        // For test of extraction and regexp; use first 5 pages ..
        n = 2;
        for (int i = 2; i < n; i++) {
            // PdfDictionary pageDict = reader.getPageN(i);
            // use location based strategy
            out.println("Page " + i);
            out.println("===========");
            String content = PdfTextExtractor.getTextFromPage(reader, i);
            out.println(content);
        }
    }

    private void copyHalamanbyTopic(Integer start_page, Integer end_page, String topicName)
            throws FileNotFoundException, DocumentException, IOException {
        // process topicName by replacing space with _?? or leave it as is?

        // Apply the offset ..
    }

    private static void splitHalamanbyTopic(Map<Integer, List<String>> myHalamanHash) {
        // Iterate through the hash ..
        // Form final filename
        // Copy over the pages ..
        int topicIndex = 0;
        int currentIndex = 0;
        ArrayList<Integer> Halamans = new ArrayList<>();
        // Halamans[0] = topic, start_page, end_page (1,1)
        // Halamans[1] = topic, start_page, end_page (1,26)
        // Halamans[2] = topic, start_page, end_page (27,60)
        // Halamans[3] = topic, start_page, end_page (61

        // Actual Start page (Start page + 1)
        // Actual End page (Page Number - 1)
        out.println("Sorted Pages and Title");
        out.println("=======================");
        // Current index - 1, fill in the end_page based on start_page - 1; if start_page != 1, else fill 1
        for (Integer myStart_Page : myHalamanHash.keySet()) {
            // Current start will inform the previous end if != 1
            List<String> myTopicList = myHalamanHash.get(myStart_Page);
            for (String myTopic : myTopicList) {
                out.println("Page: " + myStart_Page + " Topic: " + myTopic);
            }
            
        }
    }

    private static Map<Integer, List<String>> getHalaman(PdfReader myreader) {
        try {
            // OPTIONE #1:
            // RegExp for Halaman is .. /(.*)\(Halaman.*(\d+)\)/ig
            // Pattern pattern_halaman = Pattern.compile("(.*)\\(Halaman.*(\\d+)\\)", Pattern.CASE_INSENSITIVE);
            // OPTION #2:
            // Better pattern is --> http://www.regexr.com/3bkfc
            // /(.*?\n?.*?)\(Halaman.*?(\d+)\).*?\n/igm
            Pattern pattern_halaman = Pattern.compile("(.*?\\n?.*?)\\(Halaman.*?(\\d+)\\).*?\\n", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
            // Title name is $1
            // Halaman # is $2
            // Extract out and log every matched items ??
            String content = PdfTextExtractor.getTextFromPage(myreader, 1);

            // Use TreeMap to have native sort in keys
            Map<Integer, List<String>> myHalaman = null;
            myHalaman = new TreeMap<>();

            Matcher halaman_matcher = pattern_halaman.matcher(content);
            while (halaman_matcher.find()) {
                // Replace all until : pattern ..??
                Pattern pattern_header = Pattern.compile(".*\\:", Pattern.CASE_INSENSITIVE);
                // Replace anything NOT valid Title -->  [^\(\)\-\.\w\d]           
                Pattern pattern_valid_title = Pattern.compile("[^\\(\\)\\-\\. \\w\\d]", Pattern.CASE_INSENSITIVE);
                // Replace multiple spaces into one space
                Pattern pattern_single_spacing = Pattern.compile("[ ]+");
                String halaman_title
                        = pattern_single_spacing.matcher(
                                pattern_valid_title.matcher(
                                        pattern_header.matcher(
                                                halaman_matcher.group(1)
                                        ).replaceAll("")
                                ).replaceAll("")
                        ).replaceAll(" ").trim();
                out.println("Title is: " + halaman_title);<commit_msg>Summary: Implemented sorted start/end page map and extract out block topic contents<commit_after>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package org.sinarproject.hansardparser;

import com.itextpdf.text.DocumentException;
import java.io.IOException;
import static java.lang.System.out;
import java.util.Set;
import java.util.TreeSet;

import com.itextpdf.text.pdf.PdfDictionary;
import com.itextpdf.text.pdf.PdfName;
import com.itextpdf.text.pdf.PdfReader;
import com.itextpdf.text.pdf.PdfString;
import com.itextpdf.text.pdf.parser.PdfTextExtractor;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 *
 * @author leow
 */
public class HansardParser {

    /**
     * The resulting PDF file.
     */
    public static String SOURCE
            = "example/DR-PARLIMEN/DR-18062015.PDF";
    // PdfReader for multiple uses?
    private static PdfReader my_reader;

    /**
     * @param args the command line arguments
     * @throws java.io.IOException
     */
    public static void main(String[] args) throws IOException {
        // TODO code application logic here
        out.println("Sinar Project's Hansard Parser ..");
        PdfReader reader = new PdfReader(HansardParser.SOURCE);
        // Assign it for later reuse ..
        HansardParser.my_reader = reader;
        // Below gets the Topics associated with each page ..
        Map<Integer, List<String>> myHalamanHash;
        myHalamanHash = HansardParser.getHalaman(reader);
        // Below gets the start page and end page mappings
        Map<Integer, Integer> myHalamanStartEnd;
        myHalamanStartEnd = HansardParser.splitHalamanbyTopic(myHalamanHash);
        // Identify the playas
        HansardParser.identifySpeakersinTopic(myHalamanStartEnd, myHalamanHash);

        // Below copies out the files and split them ..
        /*
         try {
         HansardParser.copyHalamanbyTopic(myHalamanStartEnd, myHalamanHash);
         } catch (FileNotFoundException ex) {
         Logger.getLogger(HansardParser.class.getName()).log(Level.SEVERE, null, ex);
         } catch (DocumentException ex) {
         Logger.getLogger(HansardParser.class.getName()).log(Level.SEVERE, null, ex);
         }
         */
    }

    private static void copyHalamanbyTopic(Map<Integer, Integer> myHalamanStartEnd, Map<Integer, List<String>> myHalamanHash)
            throws FileNotFoundException, DocumentException, IOException {
        // process topicName by replacing space with _?? or leave it as is?
        // Iterate through the hash ..
        // Form final filename
        // Copy over the pages ..
        int topicIndex = 0;
        int currentIndex = 0;

        // Apply the offset ..
    }

    private static void identifySpeakersinTopic(Map<Integer, Integer> myHalamanStartEnd, Map<Integer, List<String>> myHalamanHash) throws IOException {
        // NOTE: Assumes: first page is index; and there is offset .. does it apply across the spectrum??
        // Should probably put a safe guard to test actual start/end ...
        // Look out for the DR pattern and its page number ..
        // TODO: Above ..
        for (Integer current_page : myHalamanStartEnd.keySet()) {
            int start_page = current_page + 1;
            int end_page = myHalamanStartEnd.get(current_page) + 1;
            out.println("For current block with title; start page is " + start_page + " and end page is " + end_page);
            for (int i = start_page; i <= end_page; i++) {
                // PdfDictionary pageDict = reader.getPageN(i);
                // use location based strategy
                out.println("Page " + i);
                out.println("===========");
                String content = PdfTextExtractor.getTextFromPage(HansardParser.my_reader, i);
                // out.println(content);
                // Identify people ..
                // ... and what they say??
                // How to regexp detect paragraph ..
            }
            // Rescan needed? no need; just give CMS to tag their speech ..
            // for demo; break out after first cycle ..
            break;
        }
    }

    private static Map<Integer, Integer> splitHalamanbyTopic(Map<Integer, List<String>> myHalamanHash) {
        Map<Integer, Integer> Halamans = new TreeMap<>();
        // Halamans[0] = topic, start_page, end_page (1,1)
        // Halamans[1] = topic, start_page, end_page (1,26)
        // Halamans[2] = topic, start_page, end_page (27,60)
        // Halamans[3] = topic, start_page, end_page (61

        // Actual Start page (Start page + 1)
        // Actual End page (Page Number - 1)
        out.println("Sorted Pages and Title");
        out.println("=======================");
        // Send back ordered HashMap of Start page and mapped End page
        int current_start_page = 0;
        int current_end_page = 0;
        int previous_page = 0;
        // Current index - 1, fill in the end_page based on start_page - 1; if start_page != 1, else fill 1
        for (Integer myStart_Page : myHalamanHash.keySet()) {
            current_start_page = myStart_Page;
            if (previous_page == 0) {
                // do nothing
            } else {
                if (current_start_page == 1) {
                    // special case: remain as 1
                    current_end_page = 1;
                } else {
                    // it will be start_page - 1
                    current_end_page = current_start_page - 1;
                }
                // Attach to previous page index the value
                out.println("Calculated block starting at page " + previous_page + " end at page " + current_end_page);
                Halamans.put(previous_page, current_end_page);
            }
            // Upddate previous_page_index to the current start_page
            previous_page = current_start_page;
            // Below for debugging purposes only ..
            /*
             List<String> myTopicList = myHalamanHash.get(myStart_Page);
             for (String myTopic : myTopicList) {
             out.println("Page: " + myStart_Page + " Topic: " + myTopic);
             }
             */

        }
        // At the end; we need to know that the previous page; the endpage is size of the whole doc - 1
        current_end_page = HansardParser.my_reader.getNumberOfPages() - 1;
        out.println("FINALLY: Block starting at page " + previous_page + " end at page " + current_end_page);
        Halamans.put(previous_page, current_end_page);

        return Halamans;
    }

    private static Map<Integer, List<String>> getHalaman(PdfReader myreader) {
        try {
            // OPTIONE #1:
            // RegExp for Halaman is .. /(.*)\(Halaman.*(\d+)\)/ig
            // Pattern pattern_halaman = Pattern.compile("(.*)\\(Halaman.*(\\d+)\\)", Pattern.CASE_INSENSITIVE);
            // OPTION #2:
            // Better pattern is --> http://www.regexr.com/3bkfc
            // /(.*?\n?.*?)\(Halaman.*?(\d+)\).*?\n/igm
            Pattern pattern_halaman = Pattern.compile("(.*?\\n?.*?)\\(Halaman.*?(\\d+)\\).*?\\n", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
            // Title name is $1
            // Halaman # is $2
            // Extract out and log every matched items ??
            String content = PdfTextExtractor.getTextFromPage(myreader, 1);
            // Use TreeMap to have native sort in keys
            Map<Integer, List<String>> myHalaman = null;
            myHalaman = new TreeMap<>();

            Matcher halaman_matcher = pattern_halaman.matcher(content);
            while (halaman_matcher.find()) {
                // Replace all until : pattern ..??
                Pattern pattern_header = Pattern.compile(".*\\:", Pattern.CASE_INSENSITIVE);
                // Replace anything NOT valid Title -->  [^\(\)\-\.\w\d]           
                Pattern pattern_valid_title = Pattern.compile("[^\\(\\)\\-\\. \\w\\d]", Pattern.CASE_INSENSITIVE);
                // Replace multiple spaces into one space
                Pattern pattern_single_spacing = Pattern.compile("[ ]+");
                String halaman_title
                        = pattern_single_spacing.matcher(
                                pattern_valid_title.matcher(
                                        pattern_header.matcher(
                                                halaman_matcher.group(1)
                                        ).replaceAll("")
                                ).replaceAll("")
                        ).replaceAll(" ").trim();
                out.println("Title is: " + halaman_title);
{'fraction_non_alphanumeric': 0.11695866874112155, 'fraction_numerical': 0.006291009943854427, 'mean_word_length': 1.9800443458980044, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1694199', 'n_tokens_mistral': 4182, 'n_tokens_neox': 4026, 'n_words': 1088}