<commit_before># -*- encoding: utf-8 -*-

##############################################################################
#
# Copyright (c) 2009-2011 Alistek, SIA. (http://www.alistek.com) All Rights Reserved.
#                    General contacts <<EMAIL>>
# Copyright (C) 2009  Domsense s.r.l.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import os, sys, traceback
import tempfile
from openerp import report
from openerp.report.report_sxw import report_sxw, report_rml
from pyPdf import PdfFileWriter, PdfFileReader
#import zipfile
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
from xml.dom import minidom
import base64
from openerp.osv import osv
from openerp.tools.translate import _
from openerp import tools
import time
import re
import copy

import openerp.modules as addons
from openerp import release

import aeroolib
from aeroolib.plugins.opendocument import Template, OOSerializer
from genshi.template import NewTextTemplate
from openerp.modules.registry import RegistryManager

import logging
logger = logging.getLogger(__name__)

from ExtraFunctions import ExtraFunctions

AEROO_TIMEOUT = 1
AEROO_RETRIES = 3

def fixPdf(data,ret_reader=False):
    if data:
        try:
            reader = PdfFileReader(StringIO(data))
            if ret_reader:
                return reader
            return data
        except Exception, e:
            error = hasattr(e, "message") and e.message or ""
            logger.error("PDF corrupted and should be fixed: %s",error)
            if os.path.exists("/usr/bin/pdftk"):
                pdfFile = None
                pdfFileFixed = None
                try:
                    pdfFile = tempfile.mktemp(".pdf")
                    fp = open(pdfFile,"wb")
                    try:
                        fp.write(data)
                    finally:
                        fp.close()

                    pdfFileFixed = tempfile.mktemp(".pdf", "fixed")
                    cmd = "pdftk %s output %s" % (pdfFile,pdfFileFixed)
                    cmd_fp = os.popen(cmd,"r",1)
                    try:
                        logger.info("PDF Fixed %s",cmd_fp.read())
                    finally:
                        cmd_fp.close()

                    if os.path.exists(pdfFileFixed):
                        data_fp = open(pdfFileFixed,"r")
                        try:
                            data = data_fp.read()
                        finally:
                            data_fp.close()

                        reader = PdfFileReader(StringIO(data))
                        if ret_reader:
                            return reader
                        return data
                finally:
                    if pdfFile:
                        os.remove(pdfFile)
                    if pdfFileFixed:
                        os.remove(pdfFileFixed)
    return None


class Counter(object):
    def __init__(self, name, start=0, interval=1):
        self.name = name
        self._number = start
        self._interval = interval

    def next(self):
        curr_number = self._number
        self._number += self._interval
        return curr_number

    def get_inc(self):<commit_msg>[FIX] delete only if file exists<commit_after># -*- encoding: utf-8 -*-

##############################################################################
#
# Copyright (c) 2009-2011 Alistek, SIA. (http://www.alistek.com) All Rights Reserved.
#                    General contacts <<EMAIL>>
# Copyright (C) 2009  <NAME>.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import os, sys, traceback
import tempfile
from openerp import report
from openerp.report.report_sxw import report_sxw, report_rml
from pyPdf import PdfFileWriter, PdfFileReader
#import zipfile
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
from xml.dom import minidom
import base64
from openerp.osv import osv
from openerp.tools.translate import _
from openerp import tools
import time
import re
import copy

import openerp.modules as addons
from openerp import release

import aeroolib
from aeroolib.plugins.opendocument import Template, OOSerializer
from genshi.template import NewTextTemplate
from openerp.modules.registry import RegistryManager

import logging
logger = logging.getLogger(__name__)

from ExtraFunctions import ExtraFunctions

AEROO_TIMEOUT = 1
AEROO_RETRIES = 3

def fixPdf(data,ret_reader=False):
    if data:
        try:
            reader = PdfFileReader(StringIO(data))
            if ret_reader:
                return reader
            return data
        except Exception, e:
            error = hasattr(e, "message") and e.message or ""
            logger.error("PDF corrupted and should be fixed: %s",error)
            if os.path.exists("/usr/bin/pdftk"):
                pdfFile = None
                pdfFileFixed = None
                try:
                    pdfFile = tempfile.mktemp(".pdf")
                    fp = open(pdfFile,"wb")
                    try:
                        fp.write(data)
                    finally:
                        fp.close()

                    pdfFileFixed = tempfile.mktemp(".pdf", "fixed")
                    cmd = "pdftk %s output %s" % (pdfFile,pdfFileFixed)
                    cmd_fp = os.popen(cmd,"r",1)
                    try:
                        logger.info("PDF Fixed %s",cmd_fp.read())
                    finally:
                        cmd_fp.close()

                    if os.path.exists(pdfFileFixed):
                        data_fp = open(pdfFileFixed,"r")
                        try:
                            data = data_fp.read()
                        finally:
                            data_fp.close()

                        reader = PdfFileReader(StringIO(data))
                        if ret_reader:
                            return reader
                        return data
                finally:
                    if pdfFile and os.path.exists(pdfFile):
                        os.remove(pdfFile)
                    if pdfFileFixed and os.path.exists(pdfFileFixed):
                        os.remove(pdfFileFixed)
    return None


class Counter(object):
    def __init__(self, name, start=0, interval=1):
        self.name = name
        self._number = start
        self._interval = interval

    def next(self):
        curr_number = self._number
        self._number += self._interval
        return curr_number

    def get_inc(self):
{'fraction_non_alphanumeric': 0.10181151112130245, 'fraction_numerical': 0.008025682182985553, 'mean_word_length': 2.100959829363669, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5001862', 'n_tokens_mistral': 2291, 'n_tokens_neox': 2127, 'n_words': 783}