<commit_before>'''
    Gmvault: a tool to backup and restore your gmail account.
    Copyright (C) <2011-2012>  <<NAME> (<EMAIL> dot <EMAIL> at gmail do com)>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

'''
import socket
import sys
import datetime
import os
import signal
import traceback

import argparse
import imaplib
import gmv.log_utils as log_utils
import gmv.gmvault_utils as gmvault_utils
import gmv.gmvault as gmvault

from gmv.cmdline_utils  import CmdLineParser
from gmv.credential_utils import CredentialHelper

GMVAULT_VERSION = gmvault_utils.GMVAULT_VERSION

GLOBAL_HELP_EPILOGUE = """Examples:

a) Get help for each of the individual commands

#> gmvault sync -h
#> gmvault restore --help
#> gmvault check -h

"""

REST_HELP_EPILOGUE = """Examples:

a) Complete restore of your gmail account (backed up in ~/gmvault-db) into <EMAIL> 

#> gmvault restore -d ~/gmvault-db <EMAIL>

b) Quick restore (restore only the last 2 months to make regular updates) of your gmail account into <EMAIL> 

#> gmvault restore --type quick -d ~/gmvault-db <EMAIL>

c) Restart a restore after a previous error (Gmail can cut the connection if it is too long)

#> gmvault restore -d ~/gmvault-db <EMAIL> --resume

d) Add label to all restored emails

#> gmvault restore --label "20120422-gmvault-restore" -d ~/gmvault-db <EMAIL>
"""

SYNC_HELP_EPILOGUE = """Examples:

a) Full synchronisation with email and oauth login in ./gmvault-db

#> gmvault sync <EMAIL>

b) Quick daily synchronisation (only the last 2 months are scanned)

#> gmvault sync --type quick <EMAIL>

c) Resume Full synchronisation from where it failed to not go through your mailbox again

#> gmvault sync <EMAIL> --resume

d) Encrypt stored emails to save them safely anywhere

#> gmvault sync <EMAIL> --encrypt

d) Custom synchronisation with an IMAP request for advance users

#> gmvault sync --type custom --imap-req "Since 1-Nov-2011 Before 10-Nov-2011" <EMAIL>

e) Custom synchronisation with an Gmail request for advance users.
   Get all emails with label work and sent by foo.

#> gmvault sync --type custom --gmail-req "in:work from:foo" <EMAIL>





"""

LOG = log_utils.LoggerFactory.get_logger('gmv')

class NotSeenAction(argparse.Action): #pylint:disable=R0903
    """
       to differenciate between a seen and non seen command
    """
    def __call__(self, parser, namespace, values, option_string=None):
        if values:
            setattr(namespace, self.dest, 'empty')
        else:
            setattr(namespace, self.dest, values)

class GMVaultLauncher(object):
    """
       GMVault launcher handling the command parsing
    """
    
    SYNC_TYPES    = ['full', 'quick', 'custom']
    RESTORE_TYPES = ['full', 'quick']
    CHECK_TYPES   = ['full']
    
    DEFAULT_GMVAULT_DB = "%s/gmvault-db" % (os.getenv("HOME", "."))
    
    def __init__(self):
        """ constructor """
        super(GMVaultLauncher, self).__init__()
        
    @gmvault_utils.memoized
    def _create_parser(self):
        """
           Create the argument parser
           Return the created parser
        """
        parser = CmdLineParser()
        
        parser.epilogue = GLOBAL_HELP_EPILOGUE

        parser.add_argument("-v", '--version', action='version', version='Gmvault v%s' % (GMVAULT_VERSION))
        
        subparsers = parser.add_subparsers(title='subcommands', help='valid subcommands.')
         
        # A sync command
        sync_parser = subparsers.add_parser('sync', \
                                            help='synchronize with a given gmail account.')
        #email argument can be optional so it should be an option
        sync_parser.add_argument('email', \
                                 action='store', default='empty_$_email', help='email to sync with.')
        # sync typ
        sync_parser.add_argument('-t', '-type', '--type', \
                                 action='store', dest='type', \
                                 default='full', help='type of synchronisation: full|quick|custom. (default: full)')
        
        sync_parser.add_argument("-d", "--db-dir", \
                                 action='store', help="Database root directory. (default: $HOME/gmvault-db)",\
                                 dest="db_dir", default= self.DEFAULT_GMVAULT_DB)
               
        # for both when seen add const empty otherwise not_seen
        # this allow to distinguish between an empty value and a non seen option
        
        
        sync_parser.add_argument("-o", "--oauth", \
                          help="use oauth for authentication. (default recommended method)",\
                          action='store_const', dest="oauth_token", const='empty', default='not_seen')
        
        sync_parser.add_argument("-p", "--passwd", \
                          help="use interactive password authentication. (not recommended)",
                          action= 'store_const' , dest="passwd", const='empty', default='not_seen')
        
        sync_parser.add_argument("--renew-oauth-tok", \
                          help="renew the stored oauth token via an interactive authentication session.",
                          action= 'store_const' , dest="oauth_token", const='renew')
         
        sync_parser.add_argument("--renew-passwd", \
                          help="renew the stored password via an interactive authentication session. (not recommended)",
                          action= 'store_const' , dest="passwd", const='renew')
        
        sync_parser.add_argument("--store-passwd", \
                          help="use interactive password authentication, encrypt and store the password. (not recommended)",
                          action= 'store_const' , dest="passwd", const='store')
        
        sync_parser.add_argument("-r", "--imap-req", metavar = "REQ", \
                                 help="Imap request to restrict sync.",\
                                 dest="imap_request", default=None)
        
        sync_parser.add_argument("-g", "--gmail-req", metavar = "REQ", \
                                 help="Gmail search request to restrict sync as defined in"\
                                      "https://support.google.com/mail/bin/answer.py?hl=en&answer=7190",\
                                 dest="gmail_request", default=None)
        
        # activate the resume mode --restart is deprecated
        sync_parser.add_argument("--resume", "--restart", \
                                 action='store_true', dest='restart', \
                                 default=False, help= 'Resume the sync action from the last saved gmail id.')
        
        # activate the resume mode --restart is deprecated
        sync_parser.add_argument("--emails-only", \
                                 action='store_true', dest='only_emails', \
                                 default=False, help= 'Only sync emails.')
        
        # activate the resume mode --restart is deprecated
        sync_parser.add_argument("--chats-only", \
                                 action='store_true', dest='only_chats', \
                                 default=False, help= 'Only sync chats.')
        
        sync_parser.add_argument("-e", "--encrypt", \
                                 help="encrypt stored email messages in the database.",\
                                 action='store_true',dest="encrypt", default=False)
        
        sync_parser.add_argument("-c", "--check-db", metavar = "VAL", \
                          help="Enable/disable the removal from the gmvault db of the emails "\
                               "that have been deleted from the given gmail account. VAL = yes or no.",\
                          dest="db_cleaning", default=None)
        
        sync_parser.add_argument("-m", "--multiple-db-owner", \
                                 help="Allow the email database to be synchronized with emails from multiple accounts.",\
                                 action='store_true',dest="allow_mult_owners", default=False)
        
        # activate the restart mode
        sync_parser.add_argument("--no-compression", \
                                 action='store_false', dest='compression', \
                                 default=True, help= 'disable email storage compression (gzip).')
        
        sync_parser.add_argument("--server", metavar = "HOSTNAME", \
                              action='store', help="Gmail imap server hostname. (default: imap.gmail.com)",\
                              dest="host", default="imap.gmail.com")
            
        sync_parser.add_argument("--port", metavar = "PORT", \
                              action='store', help="Gmail imap server port. (default: 993)",\
                              dest="port", default=993)
        
        sync_parser.add_argument("--debug", \
                              action='store_true', help="Activate debugging info",\
                              dest="debug", default=False)
        
        
        sync_parser.set_defaults(verb='sync')
    
        sync_parser.epilogue = SYNC_HELP_EPILOGUE
        
        # restore command
        rest_parser = subparsers.add_parser('restore', \
                                            help='restore gmvault-db to a given email account.')
        #email argument can be optional so it should be an option
        rest_parser.add_argument('email', \
                                 action='store', default='empty_$_email', help='email account to restore.')
        
        # restore typ
        rest_parser.add_argument('-t', '-type', '--type', \
                                 action='store', dest='type', \
                                 default='full', help='type of restoration: full|quick. (default: full)')
        
        # add a label
        rest_parser.add_argument('-l', '--label', \
                                 action='store', dest='label', \
                                 default=None, help='Apply a label to restored emails')
        
        # activate the resume mode --restart is deprecated
        rest_parser.add_argument("--resume", "--restart", \
                                 action='store_true', dest='restart', \
                                 default=False, help= 'Restart from the last saved gmail id.')
                                 
        # activate the resume mode --restart is deprecated
        rest_parser.add_argument("--emails-only", \
                                 action='store_true', dest='only_emails', \
                                 default=False, help= 'Only sync emails.')
        
        # activate the resume mode --restart is deprecated
        rest_parser.add_argument("--chats-only", \
                                 action='store_true', dest='only_chats', \
                                 default=False, help= 'Only sync chats.')
        
        rest_parser.add_argument("-d", "--db-dir", \
                                 action='store', help="Database root directory. (default: ./gmvault-db)",\
                                 dest="db_dir", default= self.DEFAULT_GMVAULT_DB)
               
        # for both when seen add const empty otherwise not_seen
        # this allow to distinguish between an empty value and a non seen option
        rest_parser.add_argument("-o", "--oauth", \
                          help="use oauth for authentication. (default method)",\
                          action='store_const', dest="oauth_token", const='empty', default='not_seen')
        
        rest_parser.add_argument("-p", "--passwd", \
                          help="use interactive password authentication. (not recommended)",
                          action='store_const', dest="passwd", const='empty', default='not_<PASSWORD>')
        
        rest_parser.add_argument("--server", metavar = "HOSTNAME", \
                              action='store', help="Gmail imap server hostname. (default: imap.gmail.com)",\
                              dest="host", default="imap.gmail.com")
            
        rest_parser.add_argument("--port", metavar = "PORT", \
                              action='store', help="Gmail imap server port. (default: 993)",\
                              dest="port", default=993)
        
        rest_parser.add_argument("--debug", \
                              action='store_true', help="Activate debugging info",\
                              dest="debug", default=False)
        
        rest_parser.set_defaults(verb='restore')
    
        rest_parser.epilogue = REST_HELP_EPILOGUE
        
        # check_db command
        check_parser = subparsers.add_parser('check', \
                                            help='Check and clean the gmvault-db disk database.')

        #email argument
        check_parser.add_argument('email', \
                                 action='store', default='empty_$_email', help='gmail account against which to check.')
        
        check_parser.add_argument("-d", "--db-dir", \
                                 action='store', help="Database root directory. (default: ./gmvault-db)",\
                                 dest="db_dir", default= self.DEFAULT_GMVAULT_DB)
     
        # for both when seen add const empty otherwise not_seen
        # this allow to distinguish between an empty value and a non seen option
        check_parser.add_argument("-o", "--oauth", \
                          help="use oauth for authentication. (default method)",\
                          action='store_const', dest="oauth_token", const='empty', default='not_seen')
        
        check_parser.add_argument("-p", "--passwd", \
                          help="use interactive password authentication. (not recommended)",
                          action='store_const', dest="passwd", const='empty', default='<PASSWORD>')
        
        check_parser.add_argument("--server", metavar = "HOSTNAME", \
                              action='store', help="Gmail imap server hostname. (default: imap.gmail.com)",\
                              dest="host", default="imap.gmail.com")
            
        check_parser.add_argument("--port", metavar = "PORT", \
                              action='store', help="Gmail imap server port. (default: 993)",\
                              dest="port", default=993)
        
        check_parser.add_argument("--debug", \
                              action='store_true', help="Activate debugging info",\
                              dest="debug", default=False)
        
        check_parser.set_defaults(verb='check')
        
        return parser
      
    @classmethod
    def _parse_common_args(cls, options, parser, parsed_args, list_of_types = []): #pylint:disable=W0102
        """
           Parse the common arguments for sync and restore
        """
        #add email
        parsed_args['email']            = options.email
        
        parsed_args['debug']            = options.debug
        
        parsed_args['restart']          = options.restart
        
        #user entered both authentication methods
        if options.passwd == 'empty' and options.oauth_token == 'empty':
            parser.error('You have to use one authentication method. '\
                         'Please choose between XOAuth and password (recommend XOAuth).')
        
        # user entered no authentication methods => go to default oauth
        if options.passwd == '<PASSWORD>' and options.oauth_token == 'not_seen':
            #default to xoauth
            options.oauth_token = 'empty'
            
        # add passwd
        parsed_args['passwd']           = options.passwd
        
        # add oauth tok
        parsed_args['oauth']            = options.oauth_token
        
        #add ops type
        if options.type:
            if options.type.lower() in list_of_types:
                parsed_args['type'] = options.type.lower()
            else:
                parser.error('Unknown type for command %s. The type should be one of %s' \
                             % (parsed_args['command'], list_of_types))
        
        #add db_dir
        parsed_args['db-dir']           = options.db_dir

        LOG.critical("Use gmvault-db located in %s.\n" % (parsed_args['db-dir'])) 
        
        # add host
        parsed_args['host']             = options.host
        
        #convert to int if necessary
        port_type = type(options.port)
        
        try:
            if port_type == type('s') or port_type == type("s"):
                port = int(options.port)
            else:
                port = options.port
        except Exception, _: #pylint:disable=W0703
            parser.error("--port option %s is not a number. Please check the port value" % (port))
            
        # add port
        parsed_args['port']             = port
             
        return parsed_args
    
    def parse_args(self):
        """ Parse command line arguments 
            
            :returns: a dict that contains the arguments
               
            :except Exception Error
            
        """
        
        parser = self._create_parser()
          
        options = parser.parse_args()
        
        LOG.debug("Namespace = %s\n" % (options))
        
        parsed_args = { }
                
        parsed_args['command'] = options.verb
        
        if parsed_args.get('command', '') == 'sync':
            
            # parse common arguments for sync and restore
            self._parse_common_args(options, parser, parsed_args, self.SYNC_TYPES)
            
            # handle the search requests (IMAP or GMAIL dialect)
            if options.imap_request and options.gmail_request:
                parser.error('Please use only one search request type. You can use --imap-req or --gmail-req.')
            elif not options.imap_request and not options.gmail_request:
                LOG.debug("No search request type passed: Get everything.")
                parsed_args['request']   = {'type': 'imap', 'req':'ALL'}
            elif options.gmail_request and not options.imap_request:
                parsed_args['request']  = { 'type': 'gmail', 'req' : self._clean_imap_or_gm_request(options.gmail_request)}
            else:
                parsed_args['request']  = { 'type':'imap',  'req' : self._clean_imap_or_gm_request(options.imap_request)}
                
            # handle emails or chats only
            if options.only_emails and options.only_chats:
                parser.error("--emails-only and --chats-only cannot be used together. Please choose one.")
           
            parsed_args['emails_only'] = options.only_emails
            parsed_args['chats_only']  = options.only_chats
        
            # add db-cleaning
            # if request passed put it False unless it has been forced by the user
            # default is True (db-cleaning done)
            #default 
            parsed_args['db-cleaning'] = True
            
            # if there is a value then it is forced
            if options.db_cleaning: 
                parsed_args['db-cleaning'] = parser.convert_to_boolean(options.db_cleaning)
            
            #elif parsed_args['request']['req'] != 'ALL' and not options.db_cleaning:
            #    #else if we have a request and not forced put it to false
            #    parsed_args['db-cleaning'] = False
                
            if parsed_args['db-cleaning']:
                LOG.critical("Activate Gmvault db cleaning.")
            else:
                LOG.critical("Disable deletion of emails that are in Gmvault db and not anymore in Gmail.")
                
            #add encryption option
            parsed_args['encrypt'] = options.encrypt

            #add ownership checking
            parsed_args['ownership_control'] = not options.allow_mult_owners
            
            #compression flag
            parsed_args['compression'] = options.compression
                
                
        elif parsed_args.get('command', '') == 'restore':
            
            # parse common arguments for sync and restore
            self._parse_common_args(options, parser, parsed_args, self.RESTORE_TYPES)
            
            # add restore label if there is any
            parsed_args['label'] = options.label
            
            parsed_args['restart'] = options.restart
            
            # handle emails or chats only
            if options.only_emails and options.only_chats:
                parser.error("--emails-only and --chats-only cannot be used together. Please choose one.")
           
            parsed_args['emails_only'] = options.only_emails
            parsed_args['chats_only']  = options.only_chats
            
        elif parsed_args.get('command', '') == 'check':
            
            #add defaults for type
            options.type    = 'full'
            options.restart = False
            
            # parse common arguments for sync and restore
            self._parse_common_args(options, parser, parsed_args, self.CHECK_TYPES)
    
        elif parsed_args.get('command', '') == 'config':
            pass
    
        #add parser
        parsed_args['parser']           = parser
        
        return parsed_args
    
    @classmethod
    def _clean_imap_or_gm_request(cls, request):
        """
           Clean request passed by the user with the option --imap-req or --gmail-req.
           Windows batch script preserve the single quote and unix shell doesn't.
           If the request starts and ends with single quote eat them.
        """
        LOG.debug("clean_imap_or_gm_request. original request = %s\n" % (request))
        
        if request and (len(request) > 2) and (request[0] == "'" and request[-1] == "'"):
            request =  request[1:-1]
            
        LOG.debug("clean_imap_or_gm_request. processed request = %s\n" % (request))
        return request
    
    @classmethod
    def _restore(cls, args, credential):
        """
           Execute All restore operations
        """
        LOG.critical("Connect to Gmail server.\n")
        # Create a gmvault releaving read_only_access
        restorer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
                                       args['email'], credential, read_only_access = False)
        
        #full sync is the first one
        if args.get('type', '') == 'full':
            
            #call restore
            labels = [args['label']] if args['label'] else []
            restorer.restore(extra_labels = labels, restart = args['restart'], \
                             emails_only = args['emails_only'], chats_only = args['chats_only'])
            
        elif args.get('type', '') == 'quick':
            
            #take the last two to 3 months depending on the current date
            
            # today - 2 months
            today = datetime.date.today()
            begin = today - datetime.timedelta(gmvault_utils.get_conf_defaults().getint("Restore", "quick_days", 8))
            
            starting_dir = gmvault_utils.get_ym_from_datetime(begin)
            
            #call restore
            labels = [args['label']] if args['label'] else []
            restorer.restore(pivot_dir = starting_dir, extra_labels = labels, restart = args['restart'],\
                             emails_only = args['emails_only'], chats_only = args['chats_only'])
        
        else:
            raise ValueError("Unknown synchronisation mode %s. Please use full (default), quick.")
        
        #print error report
        LOG.critical(restorer.get_error_report()) 
            
    @classmethod        
    def _sync(cls, args, credential):
        """
           Execute All synchronisation operations
        """
        LOG.critical("Connect to Gmail server.\n")
        
        # handle credential in all levels
        syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
                                       args['email'], credential, read_only_access = True, use_encryption = args['encrypt'])
        
        
        
        #full sync is the first one
        if args.get('type', '') == 'full':
        
            #choose full sync. Ignore the request
            syncer.sync({ 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
                        db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
                        restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only'])
            
        elif args.get('type', '') == 'quick':
            
            #sync only the last x days (taken in defaults) in order to be quick (cleaning is import here because recent days might move again
            
            # today - 2 months
            today = datetime.date.today()
            begin = today - datetime.timedelta(gmvault_utils.get_conf_defaults().getint("Sync", "quick_days", 8))
            
            LOG.critical("Quick sync mode. Check for new emails since %s." % (begin.strftime('%d-%b-%Y')))
            
            # today + 1 day
            end   = today + datetime.timedelta(1)
            
            syncer.sync( { 'type': 'imap', 'req': syncer.get_imap_request_btw_2_dates(begin, end) }, \
                           compress_on_disk = args['compression'], \
                           db_cleaning = args['db-cleaning'], \
                           ownership_checking = args['ownership_control'], restart = args['restart'], \
                           emails_only = args['emails_only'], chats_only = args['chats_only'])
            
        elif args.get('type', '') == 'custom':
            <commit_msg>pylinting<commit_after>'''
    Gmvault: a tool to backup and restore your gmail account.
    Copyright (C) <2011-2012>  <<NAME> (guillaume dot aubert at gmail do com)>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

'''
import socket
import sys
import datetime
import os
import signal
import traceback

import argparse
import imaplib
import gmv.log_utils as log_utils
import gmv.gmvault_utils as gmvault_utils
import gmv.gmvault as gmvault

from gmv.cmdline_utils  import CmdLineParser
from gmv.credential_utils import CredentialHelper

GMVAULT_VERSION = gmvault_utils.GMVAULT_VERSION

GLOBAL_HELP_EPILOGUE = """Examples:

a) Get help for each of the individual commands

#> gmvault sync -h
#> gmvault restore --help
#> gmvault check -h

"""

REST_HELP_EPILOGUE = """Examples:

a) Complete restore of your gmail account (backed up in ~/gmvault-db) into <EMAIL> 

#> gmvault restore -d ~/gmvault-db <EMAIL>

b) Quick restore (restore only the last 2 months to make regular updates) of your gmail account into <EMAIL> 

#> gmvault restore --type quick -d ~/gmvault-db <EMAIL>

c) Restart a restore after a previous error (Gmail can cut the connection if it is too long)

#> gmvault restore -d ~/gmvault-db <EMAIL> --resume

d) Add label to all restored emails

#> gmvault restore --label "20120422-gmvault-restore" -d ~/gmvault-db <EMAIL>
"""

SYNC_HELP_EPILOGUE = """Examples:

a) Full synchronisation with email and oauth login in ./gmvault-db

#> gmvault sync <EMAIL>

b) Quick daily synchronisation (only the last 2 months are scanned)

#> gmvault sync --type quick <EMAIL>

c) Resume Full synchronisation from where it failed to not go through your mailbox again

#> gmvault sync <EMAIL> --resume

d) Encrypt stored emails to save them safely anywhere

#> gmvault sync <EMAIL> --encrypt

d) Custom synchronisation with an IMAP request for advance users

#> gmvault sync --type custom --imap-req "Since 1-Nov-2011 Before 10-Nov-2011" <EMAIL>

e) Custom synchronisation with an Gmail request for advance users.
   Get all emails with label work and sent by foo.

#> gmvault sync --type custom --gmail-req "in:work from:foo" <EMAIL>





"""

LOG = log_utils.LoggerFactory.get_logger('gmv')

class NotSeenAction(argparse.Action): #pylint:disable=R0903
    """
       to differenciate between a seen and non seen command
    """
    def __call__(self, parser, namespace, values, option_string=None):
        if values:
            setattr(namespace, self.dest, 'empty')
        else:
            setattr(namespace, self.dest, values)

class GMVaultLauncher(object):
    """
       GMVault launcher handling the command parsing
    """
    
    SYNC_TYPES    = ['full', 'quick', 'custom']
    RESTORE_TYPES = ['full', 'quick']
    CHECK_TYPES   = ['full']
    
    DEFAULT_GMVAULT_DB = "%s/gmvault-db" % (os.getenv("HOME", "."))
    
    def __init__(self):
        """ constructor """
        super(GMVaultLauncher, self).__init__()
        
    @gmvault_utils.memoized
    def _create_parser(self): #pylint: disable=R0915
        """
           Create the argument parser
           Return the created parser
        """
        parser = CmdLineParser()
        
        parser.epilogue = GLOBAL_HELP_EPILOGUE

        parser.add_argument("-v", '--version', action='version', version='Gmvault v%s' % (GMVAULT_VERSION))
        
        subparsers = parser.add_subparsers(title='subcommands', help='valid subcommands.')
         
        # A sync command
        sync_parser = subparsers.add_parser('sync', \
                                            help='synchronize with a given gmail account.')
        #email argument can be optional so it should be an option
        sync_parser.add_argument('email', \
                                 action='store', default='empty_$_email', help='email to sync with.')
        # sync typ
        sync_parser.add_argument('-t', '-type', '--type', \
                                 action='store', dest='type', \
                                 default='full', help='type of synchronisation: full|quick|custom. (default: full)')
        
        sync_parser.add_argument("-d", "--db-dir", \
                                 action='store', help="Database root directory. (default: $HOME/gmvault-db)",\
                                 dest="db_dir", default= self.DEFAULT_GMVAULT_DB)
               
        # for both when seen add const empty otherwise not_seen
        # this allow to distinguish between an empty value and a non seen option
        
        
        sync_parser.add_argument("-o", "--oauth", \
                          help="use oauth for authentication. (default recommended method)",\
                          action='store_const', dest="oauth_token", const='empty', default='not_seen')
        
        sync_parser.add_argument("-p", "--passwd", \
                          help="use interactive password authentication. (not recommended)",
                          action= 'store_const' , dest="passwd", const='empty', default='not_seen')
        
        sync_parser.add_argument("--renew-oauth-tok", \
                          help="renew the stored oauth token via an interactive authentication session.",
                          action= 'store_const' , dest="oauth_token", const='renew')
         
        sync_parser.add_argument("--renew-passwd", \
                          help="renew the stored password via an interactive authentication session. (not recommended)",
                          action= 'store_const' , dest="passwd", const='renew')
        
        sync_parser.add_argument("--store-passwd", \
                          help="use interactive password authentication, encrypt and store the password. (not recommended)",
                          action= 'store_const' , dest="passwd", const='store')
        
        sync_parser.add_argument("-r", "--imap-req", metavar = "REQ", \
                                 help="Imap request to restrict sync.",\
                                 dest="imap_request", default=None)
        
        sync_parser.add_argument("-g", "--gmail-req", metavar = "REQ", \
                                 help="Gmail search request to restrict sync as defined in"\
                                      "https://support.google.com/mail/bin/answer.py?hl=en&answer=7190",\
                                 dest="gmail_request", default=None)
        
        # activate the resume mode --restart is deprecated
        sync_parser.add_argument("--resume", "--restart", \
                                 action='store_true', dest='restart', \
                                 default=False, help= 'Resume the sync action from the last saved gmail id.')
        
        # activate the resume mode --restart is deprecated
        sync_parser.add_argument("--emails-only", \
                                 action='store_true', dest='only_emails', \
                                 default=False, help= 'Only sync emails.')
        
        # activate the resume mode --restart is deprecated
        sync_parser.add_argument("--chats-only", \
                                 action='store_true', dest='only_chats', \
                                 default=False, help= 'Only sync chats.')
        
        sync_parser.add_argument("-e", "--encrypt", \
                                 help="encrypt stored email messages in the database.",\
                                 action='store_true',dest="encrypt", default=False)
        
        sync_parser.add_argument("-c", "--check-db", metavar = "VAL", \
                          help="Enable/disable the removal from the gmvault db of the emails "\
                               "that have been deleted from the given gmail account. VAL = yes or no.",\
                          dest="db_cleaning", default=None)
        
        sync_parser.add_argument("-m", "--multiple-db-owner", \
                                 help="Allow the email database to be synchronized with emails from multiple accounts.",\
                                 action='store_true',dest="allow_mult_owners", default=False)
        
        # activate the restart mode
        sync_parser.add_argument("--no-compression", \
                                 action='store_false', dest='compression', \
                                 default=True, help= 'disable email storage compression (gzip).')
        
        sync_parser.add_argument("--server", metavar = "HOSTNAME", \
                              action='store', help="Gmail imap server hostname. (default: imap.gmail.com)",\
                              dest="host", default="imap.gmail.com")
            
        sync_parser.add_argument("--port", metavar = "PORT", \
                              action='store', help="Gmail imap server port. (default: 993)",\
                              dest="port", default=993)
        
        sync_parser.add_argument("--debug", \
                              action='store_true', help="Activate debugging info",\
                              dest="debug", default=False)
        
        
        sync_parser.set_defaults(verb='sync')
    
        sync_parser.epilogue = SYNC_HELP_EPILOGUE
        
        # restore command
        rest_parser = subparsers.add_parser('restore', \
                                            help='restore gmvault-db to a given email account.')
        #email argument can be optional so it should be an option
        rest_parser.add_argument('email', \
                                 action='store', default='empty_$_email', help='email account to restore.')
        
        # restore typ
        rest_parser.add_argument('-t', '-type', '--type', \
                                 action='store', dest='type', \
                                 default='full', help='type of restoration: full|quick. (default: full)')
        
        # add a label
        rest_parser.add_argument('-l', '--label', \
                                 action='store', dest='label', \
                                 default=None, help='Apply a label to restored emails')
        
        # activate the resume mode --restart is deprecated
        rest_parser.add_argument("--resume", "--restart", \
                                 action='store_true', dest='restart', \
                                 default=False, help= 'Restart from the last saved gmail id.')
                                 
        # activate the resume mode --restart is deprecated
        rest_parser.add_argument("--emails-only", \
                                 action='store_true', dest='only_emails', \
                                 default=False, help= 'Only sync emails.')
        
        # activate the resume mode --restart is deprecated
        rest_parser.add_argument("--chats-only", \
                                 action='store_true', dest='only_chats', \
                                 default=False, help= 'Only sync chats.')
        
        rest_parser.add_argument("-d", "--db-dir", \
                                 action='store', help="Database root directory. (default: ./gmvault-db)",\
                                 dest="db_dir", default= self.DEFAULT_GMVAULT_DB)
               
        # for both when seen add const empty otherwise not_seen
        # this allow to distinguish between an empty value and a non seen option
        rest_parser.add_argument("-o", "--oauth", \
                          help="use oauth for authentication. (default method)",\
                          action='store_const', dest="oauth_token", const='empty', default='not_seen')
        
        rest_parser.add_argument("-p", "--passwd", \
                          help="use interactive password authentication. (not recommended)",
                          action='store_const', dest="passwd", const='empty', default='not_seen')
        
        rest_parser.add_argument("--server", metavar = "HOSTNAME", \
                              action='store', help="Gmail imap server hostname. (default: imap.gmail.com)",\
                              dest="host", default="imap.gmail.com")
            
        rest_parser.add_argument("--port", metavar = "PORT", \
                              action='store', help="Gmail imap server port. (default: 993)",\
                              dest="port", default=993)
        
        rest_parser.add_argument("--debug", \
                              action='store_true', help="Activate debugging info",\
                              dest="debug", default=False)
        
        rest_parser.set_defaults(verb='restore')
    
        rest_parser.epilogue = REST_HELP_EPILOGUE
        
        # check_db command
        check_parser = subparsers.add_parser('check', \
                                            help='Check and clean the gmvault-db disk database.')

        #email argument
        check_parser.add_argument('email', \
                                 action='store', default='empty_$_email', help='gmail account against which to check.')
        
        check_parser.add_argument("-d", "--db-dir", \
                                 action='store', help="Database root directory. (default: ./gmvault-db)",\
                                 dest="db_dir", default= self.DEFAULT_GMVAULT_DB)
     
        # for both when seen add const empty otherwise not_seen
        # this allow to distinguish between an empty value and a non seen option
        check_parser.add_argument("-o", "--oauth", \
                          help="use oauth for authentication. (default method)",\
                          action='store_const', dest="oauth_token", const='empty', default='not_seen')
        
        check_parser.add_argument("-p", "--passwd", \
                          help="use interactive password authentication. (not recommended)",
                          action='store_const', dest="passwd", const='empty', default='not_seen')
        
        check_parser.add_argument("--server", metavar = "HOSTNAME", \
                              action='store', help="Gmail imap server hostname. (default: imap.gmail.com)",\
                              dest="host", default="imap.gmail.com")
            
        check_parser.add_argument("--port", metavar = "PORT", \
                              action='store', help="Gmail imap server port. (default: 993)",\
                              dest="port", default=993)
        
        check_parser.add_argument("--debug", \
                              action='store_true', help="Activate debugging info",\
                              dest="debug", default=False)
        
        check_parser.set_defaults(verb='check')
        
        return parser
      
    @classmethod
    def _parse_common_args(cls, options, parser, parsed_args, list_of_types = []): #pylint:disable=W0102
        """
           Parse the common arguments for sync and restore
        """
        #add email
        parsed_args['email']            = options.email
        
        parsed_args['debug']            = options.debug
        
        parsed_args['restart']          = options.restart
        
        #user entered both authentication methods
        if options.passwd == 'empty' and options.oauth_token == 'empty':
            parser.error('You have to use one authentication method. '\
                         'Please choose between XOAuth and password (recommend XOAuth).')
        
        # user entered no authentication methods => go to default oauth
        if options.passwd == '<PASSWORD>' and options.oauth_token == 'not_seen':
            #default to xoauth
            options.oauth_token = 'empty'
            
        # add passwd
        parsed_args['passwd']           = options.passwd
        
        # add oauth tok
        parsed_args['oauth']            = options.oauth_token
        
        #add ops type
        if options.type:
            if options.type.lower() in list_of_types:
                parsed_args['type'] = options.type.lower()
            else:
                parser.error('Unknown type for command %s. The type should be one of %s' \
                             % (parsed_args['command'], list_of_types))
        
        #add db_dir
        parsed_args['db-dir']           = options.db_dir

        LOG.critical("Use gmvault-db located in %s.\n" % (parsed_args['db-dir'])) 
        
        # add host
        parsed_args['host']             = options.host
        
        #convert to int if necessary
        port_type = type(options.port)
        
        try:
            if port_type == type('s') or port_type == type("s"):
                port = int(options.port)
            else:
                port = options.port
        except Exception, _: #pylint:disable=W0703
            parser.error("--port option %s is not a number. Please check the port value" % (port))
            
        # add port
        parsed_args['port']             = port
             
        return parsed_args
    
    def parse_args(self): #pylint: disable=R0912
        """ Parse command line arguments 
            
            :returns: a dict that contains the arguments
               
            :except Exception Error
            
        """
        
        parser = self._create_parser()
          
        options = parser.parse_args()
        
        LOG.debug("Namespace = %s\n" % (options))
        
        parsed_args = { }
                
        parsed_args['command'] = options.verb
        
        if parsed_args.get('command', '') == 'sync':
            
            # parse common arguments for sync and restore
            self._parse_common_args(options, parser, parsed_args, self.SYNC_TYPES)
            
            # handle the search requests (IMAP or GMAIL dialect)
            if options.imap_request and options.gmail_request:
                parser.error('Please use only one search request type. You can use --imap-req or --gmail-req.')
            elif not options.imap_request and not options.gmail_request:
                LOG.debug("No search request type passed: Get everything.")
                parsed_args['request']   = {'type': 'imap', 'req':'ALL'}
            elif options.gmail_request and not options.imap_request:
                parsed_args['request']  = { 'type': 'gmail', 'req' : self._clean_imap_or_gm_request(options.gmail_request)}
            else:
                parsed_args['request']  = { 'type':'imap',  'req' : self._clean_imap_or_gm_request(options.imap_request)}
                
            # handle emails or chats only
            if options.only_emails and options.only_chats:
                parser.error("--emails-only and --chats-only cannot be used together. Please choose one.")
           
            parsed_args['emails_only'] = options.only_emails
            parsed_args['chats_only']  = options.only_chats
        
            # add db-cleaning
            # if request passed put it False unless it has been forced by the user
            # default is True (db-cleaning done)
            #default 
            parsed_args['db-cleaning'] = True
            
            # if there is a value then it is forced
            if options.db_cleaning: 
                parsed_args['db-cleaning'] = parser.convert_to_boolean(options.db_cleaning)
            
            #elif parsed_args['request']['req'] != 'ALL' and not options.db_cleaning:
            #    #else if we have a request and not forced put it to false
            #    parsed_args['db-cleaning'] = False
                
            if parsed_args['db-cleaning']:
                LOG.critical("Activate Gmvault db cleaning.")
            else:
                LOG.critical("Disable deletion of emails that are in Gmvault db and not anymore in Gmail.")
                
            #add encryption option
            parsed_args['encrypt'] = options.encrypt

            #add ownership checking
            parsed_args['ownership_control'] = not options.allow_mult_owners
            
            #compression flag
            parsed_args['compression'] = options.compression
                
                
        elif parsed_args.get('command', '') == 'restore':
            
            # parse common arguments for sync and restore
            self._parse_common_args(options, parser, parsed_args, self.RESTORE_TYPES)
            
            # add restore label if there is any
            parsed_args['label'] = options.label
            
            parsed_args['restart'] = options.restart
            
            # handle emails or chats only
            if options.only_emails and options.only_chats:
                parser.error("--emails-only and --chats-only cannot be used together. Please choose one.")
           
            parsed_args['emails_only'] = options.only_emails
            parsed_args['chats_only']  = options.only_chats
            
        elif parsed_args.get('command', '') == 'check':
            
            #add defaults for type
            options.type    = 'full'
            options.restart = False
            
            # parse common arguments for sync and restore
            self._parse_common_args(options, parser, parsed_args, self.CHECK_TYPES)
    
        elif parsed_args.get('command', '') == 'config':
            pass
    
        #add parser
        parsed_args['parser']           = parser
        
        return parsed_args
    
    @classmethod
    def _clean_imap_or_gm_request(cls, request):
        """
           Clean request passed by the user with the option --imap-req or --gmail-req.
           Windows batch script preserve the single quote and unix shell doesn't.
           If the request starts and ends with single quote eat them.
        """
        LOG.debug("clean_imap_or_gm_request. original request = %s\n" % (request))
        
        if request and (len(request) > 2) and (request[0] == "'" and request[-1] == "'"):
            request =  request[1:-1]
            
        LOG.debug("clean_imap_or_gm_request. processed request = %s\n" % (request))
        return request
    
    @classmethod
    def _restore(cls, args, credential):
        """
           Execute All restore operations
        """
        LOG.critical("Connect to Gmail server.\n")
        # Create a gmvault releaving read_only_access
        restorer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
                                       args['email'], credential, read_only_access = False)
        
        #full sync is the first one
        if args.get('type', '') == 'full':
            
            #call restore
            labels = [args['label']] if args['label'] else []
            restorer.restore(extra_labels = labels, restart = args['restart'], \
                             emails_only = args['emails_only'], chats_only = args['chats_only'])
            
        elif args.get('type', '') == 'quick':
            
            #take the last two to 3 months depending on the current date
            
            # today - 2 months
            today = datetime.date.today()
            begin = today - datetime.timedelta(gmvault_utils.get_conf_defaults().getint("Restore", "quick_days", 8))
            
            starting_dir = gmvault_utils.get_ym_from_datetime(begin)
            
            #call restore
            labels = [args['label']] if args['label'] else []
            restorer.restore(pivot_dir = starting_dir, extra_labels = labels, restart = args['restart'], \
                             emails_only = args['emails_only'], chats_only = args['chats_only'])
        
        else:
            raise ValueError("Unknown synchronisation mode %s. Please use full (default), quick.")
        
        #print error report
        LOG.critical(restorer.get_error_report()) 
            
    @classmethod        
    def _sync(cls, args, credential):
        """
           Execute All synchronisation operations
        """
        LOG.critical("Connect to Gmail server.\n")
        
        # handle credential in all levels
        syncer = gmvault.GMVaulter(args['db-dir'], args['host'], args['port'], \
                                       args['email'], credential, read_only_access = True, use_encryption = args['encrypt'])
        
        
        
        #full sync is the first one
        if args.get('type', '') == 'full':
        
            #choose full sync. Ignore the request
            syncer.sync({ 'type': 'imap', 'req': 'ALL' } , compress_on_disk = args['compression'], \
                        db_cleaning = args['db-cleaning'], ownership_checking = args['ownership_control'],\
                        restart = args['restart'], emails_only = args['emails_only'], chats_only = args['chats_only'])
            
        elif args.get('type', '') == 'quick':
            
            #sync only the last x days (taken in defaults) in order to be quick 
            #(cleaning is import here because recent days might move again
            
            # today - 2 months
            today = datetime.date.today()
            begin = today - datetime.timedelta(gmvault_utils.get_conf_defaults().getint("Sync", "quick_days", 8))
            
            LOG.critical("Quick sync mode. Check for new emails since %s." % (begin.strftime('%d-%b-%Y')))
            
            # today + 1 day
            end   = today + datetime.timedelta(1)
            
            syncer.sync( { 'type': 'imap', 'req': syncer.get_imap_request_btw_2_dates(begin, end) }, \
                           compress_on_disk = args['compression'], \
                           db_cleaning = args['db-cleaning'], \
                           ownership_checking = args['ownership_control'], restart = args['restart'], \
                           emails_only = args['emails_only'], chats_only = args['chats_only'])
            
        elif args.get('type', '') == 'custom':
            
{'fraction_non_alphanumeric': 0.10570564897033818, 'fraction_numerical': 0.0030606461364065746, 'mean_word_length': 1.720968488150928, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 71, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 92, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7514003', 'n_tokens_mistral': 13865, 'n_tokens_neox': 13708, 'n_words': 3724}