<commit_before># -*- coding: utf-8 -*-
##
##
## This file is part of CDS Indico.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Indico; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

import tempfile, os, base64, socket
import mimetypes, urlparse, simplejson
from MaKaC.common.Configuration import Config
from MaKaC import conference
from MaKaC.common.httpTimeout import HTTPWithTimeout

#URL_RESPONSE = "http://pcdh20.cern.ch/indico/getConvertedFile.py"
#SERVER = 'http://pcdh20.cern.ch/getSegFile.py'#'http://pcuds01.cern.ch/getSegFile.py'

SEGMENT_SIZE = 500000
DEFAUTL_ENCODING = "utf-8"

def get_content_type(filename):
    return mimetypes.guess_type(filename)[0] or 'application/octet-stream'

def writeLog(m):
    ERROR_LOG_FOLDER="c:/tmp"
    if os.path.isdir(ERROR_LOG_FOLDER):
        l=open(os.path.join(ERROR_LOG_FOLDER, "FILE-CONVERTER-ERROR.log"),"a")
        l.write("%s\n"%m)
        l.close()


class FileConverter:

    def convert(filepath, converter, uid):
        pass
    convert=staticmethod(convert)

    def storeConvertedFile(params):
        pass
    storeConvertedFile=staticmethod(storeConvertedFile)

    def hasAvailableConversionsFor(ext):
        pass
    hasAvailableConversionsFor=staticmethod(hasAvailableConversionsFor)

class CDSConvFileConverter(FileConverter):

    _availableExt=[".ppt", ".doc", ".pptx", ".docx", ".odp", ".sxi" ]

    def convert(filepath, converter, material):
        # Material UID
        materialUID="%s"%material.getLocator()
        # Getting variables
        responseURL = Config.getInstance().getFileConverterResponseURL()
        serverURL = Config.getInstance().getFileConverterServerURL()
        up = urlparse.urlparse(serverURL)
        if up[0] == "":
            #writeLog("Wrong conversion server URL")
            return
        host = up[1]
        selector = up[2]
        # Checking the file
        localFile = filepath.strip()
        localFile = localFile.replace("\\", "/")
        filename = localFile.split("/")[-1]
        if not os.access(localFile, os.F_OK):
            #writeLog("Local file to upload invalid")
            return False
        fic=open(localFile, "rb")
        segnum=0
        segsize = SEGMENT_SIZE
        filekey = 'segfile'
        try :
            h = HTTPWithTimeout(host, timeout = 5)
        except Exception, e:
            #writeLog("Error : Can't connect to host:%s"%host)
            return False

        while 1:

            temp = fic.read(segsize)

            segnum = segnum + 1
            BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
            CRLF = '\r\n'
            L = []

            if len(temp) < segsize:
                lastseg = 1

                L.append('--' + BOUNDARY)
                L.append('Content-Disposition: form-data; name="dirresponse"')
                L.append('')
                L.append(materialUID)

                L.append('--' + BOUNDARY)
                L.append('Content-Disposition: form-data; name="urlresponse"')
                L.append('')
                L.append(responseURL)

                L.append('--' + BOUNDARY)
                L.append('Content-Disposition: form-data; name="converter"')
                L.append('')
                L.append(converter)
            else:
                lastseg = 0
            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name="filename"')
            L.append('')
            L.append(filename)

            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name="segnum"')
            L.append('')
            L.append('%i' % segnum)

            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name="lastseg"')
            L.append('')
            L.append('%i' % lastseg)

            """
            add the segment
            """

            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (filekey, filename))
            L.append('Content-Type: %s' % get_content_type(filename))
            L.append('')
            L.append(temp)

            L.append('--' + BOUNDARY + '--')
            L.append('')
            body = CRLF.join(L)
            content_type = 'multipart/form-data; boundary=%s' % BOUNDARY

            try:
                h.putrequest('POST', selector)
                h.putheader('content-type', content_type)
                h.putheader('content-length', str(len(body)))
                h.endheaders()
                h.send(body)
            except:
                #writeLog("Exception while sending file to the CDSconv server")
                pass
            try:
                h.getreply()
            except socket.timeout:
                break
            except Exception:
                #writeLog(e)
                pass
            if lastseg == 1 :<commit_msg>[FIX] Surviving HTTPWithTimeout<commit_after># -*- coding: utf-8 -*-
##
##
## This file is part of CDS Indico.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Indico; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

import tempfile, os, base64, socket
import mimetypes, urlparse, simplejson
import httplib

from MaKaC.common.logger import Logger
from MaKaC.common.Configuration import Config
from MaKaC import conference

#URL_RESPONSE = "http://pcdh20.cern.ch/indico/getConvertedFile.py"
#SERVER = 'http://pcdh20.cern.ch/getSegFile.py'#'http://pcuds01.cern.ch/getSegFile.py'

SEGMENT_SIZE = 500000
DEFAUTL_ENCODING = "utf-8"

def get_content_type(filename):
    return mimetypes.guess_type(filename)[0] or 'application/octet-stream'

def writeLog(m):
    ERROR_LOG_FOLDER="c:/tmp"
    if os.path.isdir(ERROR_LOG_FOLDER):
        l=open(os.path.join(ERROR_LOG_FOLDER, "FILE-CONVERTER-ERROR.log"),"a")
        l.write("%s\n"%m)
        l.close()


class FileConverter:

    def convert(filepath, converter, uid):
        pass
    convert=staticmethod(convert)

    def storeConvertedFile(params):
        pass
    storeConvertedFile=staticmethod(storeConvertedFile)

    def hasAvailableConversionsFor(ext):
        pass
    hasAvailableConversionsFor=staticmethod(hasAvailableConversionsFor)

class CDSConvFileConverter(FileConverter):

    _availableExt=[".ppt", ".doc", ".pptx", ".docx", ".odp", ".sxi" ]

    def convert(filepath, converter, material):
        # Material UID
        materialUID="%s"%material.getLocator()
        # Getting variables
        responseURL = Config.getInstance().getFileConverterResponseURL()
        serverURL = Config.getInstance().getFileConverterServerURL()
        up = urlparse.urlparse(serverURL)
        if up[0] == "":
            #writeLog("Wrong conversion server URL")
            return
        host = up[1]
        selector = up[2]
        # Checking the file
        localFile = filepath.strip()
        localFile = localFile.replace("\\", "/")
        filename = localFile.split("/")[-1]
        if not os.access(localFile, os.F_OK):
            #writeLog("Local file to upload invalid")
            return False
        fic = open(localFile, "rb")
        segnum = 0
        segsize = SEGMENT_SIZE
        filekey = 'segfile'

        while 1:

            temp = fic.read(segsize)

            segnum = segnum + 1
            BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
            CRLF = '\r\n'
            L = []

            if len(temp) < segsize:
                lastseg = 1

                L.append('--' + BOUNDARY)
                L.append('Content-Disposition: form-data; name="dirresponse"')
                L.append('')
                L.append(materialUID)

                L.append('--' + BOUNDARY)
                L.append('Content-Disposition: form-data; name="urlresponse"')
                L.append('')
                L.append(responseURL)

                L.append('--' + BOUNDARY)
                L.append('Content-Disposition: form-data; name="converter"')
                L.append('')
                L.append(converter)
            else:
                lastseg = 0
            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name="filename"')
            L.append('')
            L.append(filename)

            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name="segnum"')
            L.append('')
            L.append('%i' % segnum)

            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name="lastseg"')
            L.append('')
            L.append('%i' % lastseg)

            """
            add the segment
            """

            L.append('--' + BOUNDARY)
            L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (filekey, filename))
            L.append('Content-Type: %s' % get_content_type(filename))
            L.append('')
            L.append(temp)

            L.append('--' + BOUNDARY + '--')
            L.append('')
            body = CRLF.join(L)
            content_type = 'multipart/form-data; boundary=%s' % BOUNDARY

            try:
                Logger.get('cdsconv').info("Uploading %s to %s" % (localFile, host))
                h = httplib.HTTPConnection(host, timeout=5)
                h.request('POST', selector, body, {'content-type': content_type})
                h.getresponse()
            except socket.timeout:
                Logger.get('cdsconv').exception("Timeout connecting to %s" % host)
                break
            except Exception:
                Logger.get('cdsconv').exception("Error connecting to %s" % host)
            if lastseg == 1 :
{'fraction_non_alphanumeric': 0.1274141876430206, 'fraction_numerical': 0.012265446224256293, 'mean_word_length': 2.0846979107848673, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4608912', 'n_tokens_mistral': 3388, 'n_tokens_neox': 3175, 'n_words': 770}