<commit_before>#!/usr/bin/env python
# -*- encoding: UTF-8 -*-

"""
pdf-factory
~~~~~~~~~~~

Parse a JSON to generate several PDFs then merge them in a single file.
"""

import sys
import os
import traceback
import shutil
from time import sleep
import json
import logging
import subprocess
import tempfile

import requests
import pypdftk

__author__ = "LaurentMox"
__copyright__ = "Copyright 2013, Revolunet"
__credits__ = ["Revolunet"]
__license__ = "MIT"
__version__ = "0.1.0"
__maintainer__ = "LaurentMox"
__email__ = "<EMAIL>"
__status__ = "Development"

log = logging.getLogger()


####
# Module configuration
# Full path to WKHTMLTOPDF binary
WKHTMLTOPDF = 'wkhtmltopdf-i386'
# Path where the final PDF files are stored
BASE_OUTPUT_DIR = "./outputs/"
# WKHTMLTOPDF conversion timeout 
TIMEOUT = 2400
###


def usage():
    ''' print CLI usage '''
    print "usage:\t", sys.argv[0], "foo.json"
    print "\t", sys.argv[0], "http://www.foo.bar/document.json"


def clean_failure(tmp_dir, callback_url=None):
    ''' clean tmp folder and call callback '''
    if callback_url:
        success_callback(False, callback_url)
    clean_tmp(tmp_dir)
    sys.exit()


def clean_tmp(folder):
    ''' clean the temp directory '''
    log.info("Deleting temporary folder: \033[34m'%s'\033[m...", folder)
    shutil.rmtree(folder)


def check_output_folder(root, path, create_folder=False):
    ''' raise exception if path is not inside root '''
    if not os.path.abspath(path).startswith(os.path.abspath(root)):
        raise Exception('incorrect path:%s', path)
    if create_folder:
        outdir = os.path.dirname(path)
        if not os.path.lexists(outdir):
            os.makedirs(outdir)


def make_tmp_file(tmp_dir, suffix='.pdf'):
    ''' shortcut to create a new PDF file '''
    handle, filename = tempfile.mkstemp(dir=tmp_dir, suffix=suffix)
    os.close(handle)
    return filename


def success_callback(success, callback_url):
    ''' emit a POST request to the callback_url with the generation result '''
    if callback_url is not None:
        payload = '{"success":%s}' % str(success).lower()
        try:
            log.info("\033[33mInforming callback\033[m")
            requests.post(callback_url, data=payload, timeout=5)
        except:
            log.error("\033[33mCan't post to callback !\033[m\n%s", traceback.format_exc())


def call_wkhtmltopdf(uri, tmp_dir, options=[]):
    ''' call WKHTMLTOPDF binary with a timeout and return generated file path '''

    pdf_filename = make_tmp_file(tmp_dir)

    log.info("\033[33mCalling WKHTMLTOPDF on '%s'\033[m", uri)

    wk_options = [
        WKHTMLTOPDF,
        # '--debug-javascript',
        '--disable-smart-shrinking',
        '--print-media-type',
        '--use-xserver',
    ]
    if options:
        wk_options += options

    wk_options.append(uri)
    wk_options.append(pdf_filename)

    p = subprocess.Popen(wk_options)
    timeout = TIMEOUT
    while p.poll() is None:
        if timeout == 0:
            p.terminate()
            log.error('033[31mWkhtmltopdf process took too much time. Aborting.033[m')
            raise Exception('WKHTMLTOPDF_TIMEOUT')
        timeout -= 1
        sleep(0.05)
    log.info("\033[33mWKHTMLTOPDF ended.\033[m")
    return pdf_filename


def process_item(item, tmp_dir):
    '''Process a single item from the JSON config
          - download remote resources
          - convert HTML to PDF<commit_msg>Added exception if wkhtmltopdf failed somehow<commit_after>#!/usr/bin/env python
# -*- encoding: UTF-8 -*-

"""
pdf-factory
~~~~~~~~~~~

Parse a JSON to generate several PDFs then merge them in a single file.
"""

import sys
import os
import traceback
import shutil
from time import sleep
import json
import logging
import subprocess
import tempfile

import requests
import pypdftk

__author__ = "LaurentMox"
__copyright__ = "Copyright 2013, Revolunet"
__credits__ = ["Revolunet"]
__license__ = "MIT"
__version__ = "0.1.0"
__maintainer__ = "LaurentMox"
__email__ = "<EMAIL>"
__status__ = "Development"

log = logging.getLogger()


####
# Module configuration
# Full path to WKHTMLTOPDF binary
WKHTMLTOPDF = 'wkhtmltopdf-i386'
# Full path to PDFTK binary
# pypdftk.PDFTK_PATH = '/usr/bin/pdftk'
# Path where the final PDF files are stored
BASE_OUTPUT_DIR = "./outputs/"
# WKHTMLTOPDF conversion timeout
TIMEOUT = 2400
###


def usage():
    ''' print CLI usage '''
    print "usage:\t", sys.argv[0], "foo.json"
    print "\t", sys.argv[0], "http://www.foo.bar/document.json"


def clean_failure(tmp_dir, callback_url=None):
    ''' clean tmp folder and call callback '''
    if callback_url:
        success_callback(False, callback_url)
    clean_tmp(tmp_dir)
    sys.exit()


def clean_tmp(folder):
    ''' clean the temp directory '''
    log.info("Deleting temporary folder: \033[34m'%s'\033[m...", folder)
    shutil.rmtree(folder)


def check_output_folder(root, path, create_folder=False):
    ''' raise exception if path is not inside root '''
    if not os.path.abspath(path).startswith(os.path.abspath(root)):
        raise Exception('incorrect path:%s', path)
    if create_folder:
        outdir = os.path.dirname(path)
        if not os.path.lexists(outdir):
            os.makedirs(outdir)


def make_tmp_file(tmp_dir, suffix='.pdf'):
    ''' shortcut to create a new PDF file '''
    handle, filename = tempfile.mkstemp(dir=tmp_dir, suffix=suffix)
    os.close(handle)
    return filename


def success_callback(success, callback_url):
    ''' emit a POST request to the callback_url with the generation result '''
    if callback_url is not None:
        payload = '{"success":%s}' % str(success).lower()
        try:
            log.info("\033[33mInforming callback\033[m")
            requests.post(callback_url, data=payload, timeout=5)
        except:
            log.error("\033[33mCan't post to callback !\033[m\n%s", traceback.format_exc())


def call_wkhtmltopdf(uri, tmp_dir, options=[]):
    ''' call WKHTMLTOPDF binary with a timeout and return generated file path '''

    pdf_filename = make_tmp_file(tmp_dir)

    log.info("\033[33mCalling WKHTMLTOPDF on '%s'\033[m", uri)

    wk_options = [
        WKHTMLTOPDF,
        # '--debug-javascript',
        '--disable-smart-shrinking',
        '--print-media-type',
        '--use-xserver',
    ]
    if options:
        wk_options += options

    wk_options.append(uri)
    wk_options.append(pdf_filename)

    p = subprocess.Popen(wk_options)
    timeout = TIMEOUT
    while p.poll() is None:
        if timeout == 0:
            p.terminate()
            log.error('033[31mWkhtmltopdf process took too much time. Aborting.033[m')
            raise Exception('WKHTMLTOPDF_TIMEOUT')
        timeout -= 1
        sleep(0.05)
    if p.returncode != 0:
        log.error('033[31mWkhtmltopdf return code is not zero. Aborting.033[m')
        raise Exception('WKHTMLTOPDF_ERROR')
    log.info("\033[33mWKHTMLTOPDF ended successfully.\033[m")
    return pdf_filename


def process_item(item, tmp_dir):
    '''Process a single item from the JSON config
          - download remote resources
          - convert HTML to PDF
{'fraction_non_alphanumeric': 0.1252633057154894, 'fraction_numerical': 0.021204886954079485, 'mean_word_length': 3.172231985940246, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3894594', 'n_tokens_mistral': 2436, 'n_tokens_neox': 2268, 'n_words': 596}