<commit_before># -*- coding: utf-8 -*-
'''
    Gmvault: a tool to backup and restore your gmail account.
    Copyright (C) <2011-2013>  <<NAME> (guillaume dot aubert at gmail do com)>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    Contains the class monkey patching IMAPClient and imaplib

'''
import zlib
import time
import datetime
import re
import socket
import ssl
import cStringIO
import os

import imaplib  #for the exception
import imapclient

#enable imap debugging if GMV_IMAP_DEBUG is set 
if os.getenv("GMV_IMAP_DEBUG"):
    imaplib.Debug = 4 #enable debugging

#to enable imap debugging and see all command
#imaplib.Debug = 4 #enable debugging

INTERNALDATE_RE = re.compile(r'.*INTERNALDATE "'
r'(?P<day>[ 0123][0-9])-(?P<mon>[A-Z][a-z][a-z])-(?P<year>[0-9][0-9][0-9][0-9])'
r' (?P<hour>[0-9][0-9]):(?P<min>[0-9][0-9]):(?P<sec>[0-9][0-9])'
r' (?P<zonen>[-+])(?P<zoneh>[0-9][0-9])(?P<zonem>[0-9][0-9])'
r'"')

MON2NUM = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6,
        'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12}

#need to monkey patch _convert_INTERNALDATE to work with imaplib2
#modification of IMAPClient
def mod_convert_INTERNALDATE(date_string, normalise_times=True):#pylint: disable=C0103
    """
       monkey patched convert_INTERNALDATE
    """
    mon = INTERNALDATE_RE.match('INTERNALDATE "%s"' % date_string)
    if not mon:
        raise ValueError("couldn't parse date %r" % date_string)
    
    zoneh = int(mon.group('zoneh'))
    zonem = (zoneh * 60) + int(mon.group('zonem'))
    if mon.group('zonen') == '-':
        zonem = -zonem
    timez = imapclient.fixed_offset.FixedOffset(zonem)
    
    year    = int(mon.group('year'))
    the_mon = MON2NUM[mon.group('mon')]
    day     = int(mon.group('day'))
    hour    = int(mon.group('hour'))
    minute  = int(mon.group('min'))
    sec = int(mon.group('sec'))
    
    the_dt = datetime.datetime(year, the_mon, day, hour, minute, sec, 0, timez)
    
    if normalise_times:
        # Normalise to host system's timezone
        return the_dt.astimezone(imapclient.fixed_offset.FixedOffset.for_system()).replace(tzinfo=None)
    return the_dt

#monkey patching is done here
imapclient.response_parser._convert_INTERNALDATE = mod_convert_INTERNALDATE

#monkey patching add compress in COMMANDS of imap
imaplib.Commands['COMPRESS'] = ('AUTH', 'SELECTED')

class IMAP4COMPSSL(imaplib.IMAP4_SSL): #pylint:disable=R0904
    """
       Add support for compression inspired by inspired by http://www.janeelix.com/piers/python/py2html.cgi/piers/python/imaplib2
    """
    SOCK_TIMEOUT = 70 # set a socket timeout of 70 sec to avoid for ever blockage in ssl.read

    def __init__(self, host = '', port = imaplib.IMAP4_SSL_PORT, keyfile = None, certfile = None):
        """
           constructor
        """
        self.compressor = None
        self.decompressor = None
        
        imaplib.IMAP4_SSL.__init__(self, host, port, keyfile, certfile)
        
    def activate_compression(self):
        """
           activate_compressing()
           Enable deflate compression on the socket (RFC 4978).
        """
        # rfc 1951 - pure DEFLATE, so use -15 for both windows
        self.decompressor = zlib.decompressobj(-15)
        self.compressor   = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15)
        
    def open(self, host = '', port = imaplib.IMAP4_SSL_PORT): 
        """Setup connection to remote server on "host:port".
           (default: localhost:standard IMAP4 SSL port).
           This connection will be used by the routines:
           read, readline, send, shutdown.
        """
        self.host   = host
        self.port   = port

        self.sock   = socket.create_connection((host, port), self.SOCK_TIMEOUT) #add so_timeout  

        #self.sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) #try to set TCP NO DELAY to increase performances

        self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile)
        
        # This is the last correction added to avoid memory fragmentation in imaplib
        # makefile creates a file object that makes use of cStringIO to avoid mem fragmentation
        # it could be used without the compression 
        # (maybe make 2 set of methods without compression and with compression)
        #self.file   = self.sslobj.makefile('rb')
    
    def read(self, size):
        """
            Read 'size' bytes from remote.
            Call _intern_read that takes care of the compression
        """
        
        chunks = cStringIO.StringIO() #use cStringIO.cStringIO to avoir too much fragmentation
        read = 0
        while read < size:
            data = self._intern_read(min(size-read, 16384)) #never ask more than 16384 because imaplib can do it
            if not data: 
                #to avoid infinite looping due to empty string returned
                raise self.abort('Gmvault ssl socket error: EOF') 
            read += len(data)
            chunks.write(data)
        
        return chunks.getvalue() #return the cStringIO content
  
    def _intern_read(self, size):
        """
            Read at most 'size' bytes from remote.
            Takes care of the compression
        """
        if self.decompressor is None:
            return self.sslobj.read(size)

        if self.decompressor.unconsumed_tail:
            data = self.decompressor.unconsumed_tail
        else:
            data = self.sslobj.read(8192) #maybe change to 16384

        return self.decompressor.decompress(data, size)
        
    def readline(self):<commit_msg>finalize mod_imap<commit_after># -*- coding: utf-8 -*-
'''
    Gmvault: a tool to backup and restore your gmail account.
    Copyright (C) <2011-2013>  <<NAME> (guillaume dot aubert at gmail do com)>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    Contains the class monkey patching IMAPClient and imaplib

'''
import zlib
import time
import datetime
import re
import socket
import ssl
import cStringIO
import os

import imaplib  #for the exception
import imapclient

#enable imap debugging if GMV_IMAP_DEBUG is set 
if os.getenv("GMV_IMAP_DEBUG"):
    imaplib.Debug = 4 #enable debugging

#to enable imap debugging and see all command
#imaplib.Debug = 4 #enable debugging

INTERNALDATE_RE = re.compile(r'.*INTERNALDATE "'
r'(?P<day>[ 0123][0-9])-(?P<mon>[A-Z][a-z][a-z])-(?P<year>[0-9][0-9][0-9][0-9])'
r' (?P<hour>[0-9][0-9]):(?P<min>[0-9][0-9]):(?P<sec>[0-9][0-9])'
r' (?P<zonen>[-+])(?P<zoneh>[0-9][0-9])(?P<zonem>[0-9][0-9])'
r'"')

MON2NUM = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6,
        'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12}

#need to monkey patch _convert_INTERNALDATE to work with imaplib2
#modification of IMAPClient
def mod_convert_INTERNALDATE(date_string, normalise_times=True):#pylint: disable=C0103
    """
       monkey patched convert_INTERNALDATE
    """
    mon = INTERNALDATE_RE.match('INTERNALDATE "%s"' % date_string)
    if not mon:
        raise ValueError("couldn't parse date %r" % date_string)
    
    zoneh = int(mon.group('zoneh'))
    zonem = (zoneh * 60) + int(mon.group('zonem'))
    if mon.group('zonen') == '-':
        zonem = -zonem
    timez = imapclient.fixed_offset.FixedOffset(zonem)
    
    year    = int(mon.group('year'))
    the_mon = MON2NUM[mon.group('mon')]
    day     = int(mon.group('day'))
    hour    = int(mon.group('hour'))
    minute  = int(mon.group('min'))
    sec = int(mon.group('sec'))
    
    the_dt = datetime.datetime(year, the_mon, day, hour, minute, sec, 0, timez)
    
    if normalise_times:
        # Normalise to host system's timezone
        return the_dt.astimezone(imapclient.fixed_offset.FixedOffset.for_system()).replace(tzinfo=None)
    return the_dt

#monkey patching is done here
imapclient.response_parser._convert_INTERNALDATE = mod_convert_INTERNALDATE

#monkey patching add compress in COMMANDS of imap
imaplib.Commands['COMPRESS'] = ('AUTH', 'SELECTED')

class IMAP4COMPSSL(imaplib.IMAP4_SSL): #pylint:disable=R0904
    """
       Add support for compression inspired by inspired by http://www.janeelix.com/piers/python/py2html.cgi/piers/python/imaplib2
    """
    SOCK_TIMEOUT = 70 # set a socket timeout of 70 sec to avoid for ever blockage in ssl.read

    def __init__(self, host = '', port = imaplib.IMAP4_SSL_PORT, keyfile = None, certfile = None):
        """
           constructor
        """
        self.compressor = None
        self.decompressor = None
        
        imaplib.IMAP4_SSL.__init__(self, host, port, keyfile, certfile)
        
    def activate_compression(self):
        """
           activate_compressing()
           Enable deflate compression on the socket (RFC 4978).
        """
        # rfc 1951 - pure DEFLATE, so use -15 for both windows
        self.decompressor = zlib.decompressobj(-15)
        self.compressor   = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15)
        
    def open(self, host = '', port = imaplib.IMAP4_SSL_PORT): 
        """Setup connection to remote server on "host:port".
           (default: localhost:standard IMAP4 SSL port).
           This connection will be used by the routines:
           read, readline, send, shutdown.
        """
        self.host   = host
        self.port   = port

        self.sock   = socket.create_connection((host, port), self.SOCK_TIMEOUT) #add so_timeout  

        #self.sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) #try to set TCP NO DELAY to increase performances

        self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile)
        
        # This is the last correction added to avoid memory fragmentation in imaplib
        # makefile creates a file object that makes use of cStringIO to avoid mem fragmentation
        # it could be used without the compression 
        # (maybe make 2 set of methods without compression and with compression)
        #self.file   = self.sslobj.makefile('rb')
    
    def read(self, size):
        """
            Read 'size' bytes from remote.
            Call _intern_read that takes care of the compression
        """
        
        chunks = cStringIO.StringIO() #use cStringIO.cStringIO to avoir too much fragmentation
        read = 0
        while read < size:
            data = self._intern_read(min(size-read, 16384)) #never ask more than 16384 because imaplib can do it
            if not data: 
                #to avoid infinite looping due to empty string returned
                raise self.abort('Gmvault ssl socket error: EOF') 
            read += len(data)
            chunks.write(data)
        
        return chunks.getvalue() #return the cStringIO content
  
    def _intern_read(self, size):
        """
            Read at most 'size' bytes from remote.
            Takes care of the compression
        """
        if self.decompressor is None:
            return self.sslobj.read(size)

        if self.decompressor.unconsumed_tail:
            data = self.decompressor.unconsumed_tail
        else:
            data = self.sslobj.read(8192) #Fixed buffer size. maybe change to 16384

        return self.decompressor.decompress(data, size)
        
    def readline(self):
{'fraction_non_alphanumeric': 0.10814055871287921, 'fraction_numerical': 0.01969102697510606, 'mean_word_length': 2.7496998799519807, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7444792', 'n_tokens_mistral': 4092, 'n_tokens_neox': 3897, 'n_words': 1187}