# Path: common/utils.py
# def read_file(filename):
# def write_file(filename, text):
# def save_obj(filename, data):
# def load_obj(filename):
# def sha1(filename):
# def write_sha1(sha1, filename):
# def url_opener():
# def copy_file_from_url(url, out_file, expect_sha1 = None, max_retry = 4):
# def compress_file_data(out_filename, data, compress_type):
# def compress_file(out_filename, in_filename, compress_type):
# def uncompress_file(filename, compress_type):
# def _retry_on_eintr(func, *args):
# def safe_read(fd):
# def safe_write(fd, text):
# def print_traceback(*kwargs):
# def readline_backward(filename, buf_size=8192):


import subprocess
import resource
import os
import sys
import tempfile
import xml.etree.ElementTree as etree
import urllib
import json
import shutil
from common import utils
        text += get_deep_text(subelement)
    text += element.tail or ''
    return text[:-1] + ' '

# This is much more accurate than using pdftotext
def pdf_text_to_djvu_with_xml(xml_file, out_file):
    fd = open(xml_file)
    last_text = ''
    page_nr = 1
    for event, elem in etree.iterparse(fd):
        if event == 'end':
            if elem.tag.lower() == 'line':
                last_text += get_deep_text(elem)
            elif elem.tag.lower() == 'paragraph':
                last_text += '\n'
            elif elem.tag.lower() == 'object':
                write_page_text(page_nr, last_text.encode('utf-8'))
                page_nr += 1
                last_text = ''
                elem.clear()
    fd.close()

    ret_code = add_text_layer(page_nr - 1, out_file)

    for r in range(1, page_nr):
        os.remove('page_%04d.temp.txt' % r)

    return ret_code

def get_ia_files(ia_id):

# Path: common/utils.py
# def read_file(filename):
# def write_file(filename, text):
# def save_obj(filename, data):
# def load_obj(filename):
# def sha1(filename):
# def write_sha1(sha1, filename):
# def url_opener():
# def copy_file_from_url(url, out_file, expect_sha1 = None, max_retry = 4):
# def compress_file_data(out_filename, data, compress_type):
# def compress_file(out_filename, in_filename, compress_type):
# def uncompress_file(filename, compress_type):
# def _retry_on_eintr(func, *args):
# def safe_read(fd):
# def safe_write(fd, text):
# def print_traceback(*kwargs):
# def readline_backward(filename, buf_size=8192):


# Path: ocr/pdf_to_djvu.py
import subprocess
import resource
import os
import sys
import tempfile
import xml.etree.ElementTree as etree
import urllib
import json
import shutil
from common import utils

        text += get_deep_text(subelement)
    text += element.tail or ''
    return text[:-1] + ' '

# This is much more accurate than using pdftotext
def pdf_text_to_djvu_with_xml(xml_file, out_file):
    fd = open(xml_file)
    last_text = ''
    page_nr = 1
    for event, elem in etree.iterparse(fd):
        if event == 'end':
            if elem.tag.lower() == 'line':
                last_text += get_deep_text(elem)
            elif elem.tag.lower() == 'paragraph':
                last_text += '\n'
            elif elem.tag.lower() == 'object':
                write_page_text(page_nr, last_text.encode('utf-8'))
                page_nr += 1
                last_text = ''
                elem.clear()
    fd.close()

    ret_code = add_text_layer(page_nr - 1, out_file)

    for r in range(1, page_nr):
        os.remove('page_%04d.temp.txt' % r)

    return ret_code

def get_ia_files(ia_id):

    result = {