<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# ScraperWiki Limited
# <NAME>, 2013-06-04

from __future__ import unicode_literals
"""
Some experiments with pdfminer
http://www.unixuser.org/~euske/python/pdfminer/programming.html
Some help here:
http://denis.papathanasiou.org/2010/08/04/extracting-text-images-from-pdf-files
"""

# TODO Identify multi-column text, for multicolumn text detect per column
# TODO Dynamic / smarter thresholding
# TODO Handle argentina_diputados_voting_record.pdf automatically
# TODO Handle multiple tables on one page


import sys
import codecs

from pdfminer.pdfparser import PDFParser, PDFDocument
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfdevice import PDFDevice
from pdfminer.layout import LAParams, LTPage
from pdfminer.converter import PDFPageAggregator

import collections

from tree import Leaf, LeafList
import requests  # TODO: remove this dependency
from cStringIO import StringIO
import math
import numpy

Y_THRESHOLD = 3      # TODO: name more helpfully
COUNT_THRESHOLD = 3  # TODO: name more helpfully


class TableDiagnosticData(object):
    def __init__(self, box_list, top_plot, left_plot, x_comb, y_comb):
        self.box_list = box_list
        self.top_plot = top_plot
        self.left_plot = left_plot
        self.x_comb = x_comb
        self.y_comb = y_comb

# index 0 = left, 1 = top, 2 = right, 3 = bottom
# TODO: WTF is above comment and why does it disagree with below 'enum'?
LEFT = 0
TOP = 3
RIGHT = 2
BOTTOM = 1


def get_tables(fh):
    """
    Return a list of 'tables' from the given file handle, where a table is a
    list of rows, and a row is a list of strings.
    """
    result = []
    doc, interpreter, device = initialize_pdf_miner(fh)
    for i, pdf_page in enumerate(doc.get_pages()):
        print("Trying page {}".format(i + 1))
        if not page_contains_tables(pdf_page, interpreter, device):
            print("Skipping page {}: no tables.".format(i + 1))
            continue

        # receive the LTPage object for the page.
        interpreter.process_page(pdf_page)<commit_msg>Made get_tables not print<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# ScraperWiki Limited
# <NAME>, 2013-06-04

from __future__ import unicode_literals
"""
Some experiments with pdfminer
http://www.unixuser.org/~euske/python/pdfminer/programming.html
Some help here:
http://denis.papathanasiou.org/2010/08/04/extracting-text-images-from-pdf-files
"""

# TODO Identify multi-column text, for multicolumn text detect per column
# TODO Dynamic / smarter thresholding
# TODO Handle argentina_diputados_voting_record.pdf automatically
# TODO Handle multiple tables on one page


import sys
import codecs

from pdfminer.pdfparser import PDFParser, PDFDocument
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfdevice import PDFDevice
from pdfminer.layout import LAParams, LTPage
from pdfminer.converter import PDFPageAggregator

import collections

from tree import Leaf, LeafList
import requests  # TODO: remove this dependency
from cStringIO import StringIO
import math
import numpy

Y_THRESHOLD = 3      # TODO: name more helpfully
COUNT_THRESHOLD = 3  # TODO: name more helpfully


class TableDiagnosticData(object):
    def __init__(self, box_list, top_plot, left_plot, x_comb, y_comb):
        self.box_list = box_list
        self.top_plot = top_plot
        self.left_plot = left_plot
        self.x_comb = x_comb
        self.y_comb = y_comb

# index 0 = left, 1 = top, 2 = right, 3 = bottom
# TODO: WTF is above comment and why does it disagree with below 'enum'?
LEFT = 0
TOP = 3
RIGHT = 2
BOTTOM = 1


def get_tables(fh):
    """
    Return a list of 'tables' from the given file handle, where a table is a
    list of rows, and a row is a list of strings.
    """
    result = []
    doc, interpreter, device = initialize_pdf_miner(fh)
    for i, pdf_page in enumerate(doc.get_pages()):
        #print("Trying page {}".format(i + 1))
        if not page_contains_tables(pdf_page, interpreter, device):
            #print("Skipping page {}: no tables.".format(i + 1))
            continue

        # receive the LTPage object for the page.
        interpreter.process_page(pdf_page)
{'fraction_non_alphanumeric': 0.0847378277153558, 'fraction_numerical': 0.013576779026217229, 'mean_word_length': 3.6445652173913046, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7423912', 'n_tokens_mistral': 1403, 'n_tokens_neox': 1330, 'n_words': 434}