<commit_before>'''

Note: This module uses source code provided by Google Inc.
The original oauth2.py script can be found at:
    https://github.com/google/gmail-oauth2-tools/blob/master/python/oauth2.py

'''

import imaplib
import email
import lxml
import sys
import urllib
import json

from optparse import OptionParser

# Gmail credentials file path
CREDENTIALS_PATH = "./creds.data"

# The URL root for accessing Google Accounts.
GOOGLE_ACCOUNTS_BASE_URL = 'https://accounts.google.com'

# Hardcoded dummy redirect URI for non-web apps.
REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'

#SCOPE= 'https://www.googleapis.com/auth/gmail.readonly'  
SCOPE= 'https://mail.google.com/'

ENDOFHEADER= "Number\r\n"

class ClientExtractor:
    def __init__(self):
        self.auth_string= ""

    def InitializeCredentials(self):
        credentials = MiscTools.GetGmailCreds(CREDENTIALS_PATH)
        self.username = credentials['USERNAME']
        self.client_id = credentials['CLIENTID']
        self.client_token = credentials['CLIENTTOKEN']

    def GetRawClientList(self):
        if self.auth_string ==  "":
            print 'To authorize token, visit this url and follow the directions:'
            print '  %s' % OAuth2Tools.GeneratePermissionUrl(self.cliend_id, SCOPE)
            authorization_code = raw_input('Enter verification code: ')
            auth_tokens= OAuth2Tools.AuthorizeTokens(self.client_id, self.client_token, authorization_code)
            self.auth_string= OAuth2Tools.GenerateOAuth2String(self.username, auth_tokens['access_token'], base64_encode=False)
        latestRawEmail= EmailTools.GetLatestEmail(self.username,  self.auth_string)
        latestEmail= EmailTools.ConvertRawToEmailMessage(latestRawEmail)
        emailData= EmailTools.ConvertEmailMessageToData(latestEmail, 0)
        dataList = DataTools.BreakDataStringToDataList(emailData)

class DataTools:

    @staticmethod
    def BreakDataStringToDataList(dataString):
        dataList = re.split('[0-9][0-9][0-9][0-9]\-[0-9][0-9][0-9][0-9]', dataString)
        return dataList

class MiscTools:
    
    @staticmethod
    def GetGmailCreds(path_to_data_file):
        credentials = {}
        with open(path_to_data_file, 'r') as credsFile:
            for line in credsFile:
                (key, val) = line.split('=')
                key = key.replace(" ","")
                val = val.replace("\n","")
                val = val.replace(" ","")
                credentials[key] = val
        return credentials

class EmailTools:

    @staticmethod
    def GetLatestEmail(EMAILUSER,  auth_string):
        imap_conn = imaplib.IMAP4_SSL('imap.gmail.com')
        imap_conn.debug = 4
        imap_conn.authenticate('XOAUTH2', lambda x: auth_string)
        imap_conn.select('INBOX')
        result, data = imap_conn.uid('search', None, "ALL") # search and return uids instead
        latest_email_uid = data[0].split()[-1]
        result, data = imap_conn.uid('fetch', latest_email_uid, '(RFC822)')
        raw_email = data[0][1]
        return raw_email

    @staticmethod
    def ConvertRawToEmailMessage(raw_email):
        return email.message_from_string(raw_email)

#TODO:save text files of both raw emails to avoid data cap
    @staticmethod
    def ConvertEmailMessageToData(email_message, payload_index):
	emailPayload= email_message.get_payload(payload_index)
        dataDecodeable= emailPayload.get_payload(decode= True)
        dataDecoded= dataDecodeable.decode('utf-8')
        startDataIndex= dataDecoded.find(ENDOFHEADER)
        return dataDecoded[(startDataIndex + len(ENDOFHEADER)):]



class OAuth2Tools:

    @staticmethod
    def AccountsUrl(command):
        """Generates the Google Accounts URL.

        Args:
            command: The command to execute.

        Returns:
            A URL for the given command.
        """
        return '%s/%s' % (GOOGLE_ACCOUNTS_BASE_URL, command)


    @staticmethod
    def UrlEscape(text):
        # See OAUTH 5.1 for a definition of which characters need to be escaped.
        return urllib.quote(text, safe='~-._')

<commit_msg>Dates can be extracted for clients using regex expressions/searches<commit_after>'''

Note: This module uses source code provided by Google Inc.
The original oauth2.py script can be found at:
    https://github.com/google/gmail-oauth2-tools/blob/master/python/oauth2.py

'''

import imaplib
import email
import lxml
import sys
import urllib
import json
import re
import time

from optparse import OptionParser

# Gmail credentials file path
CREDENTIALS_PATH = "./creds_filled.data"

# The URL root for accessing Google Accounts.
GOOGLE_ACCOUNTS_BASE_URL = 'https://accounts.google.com'

# Hardcoded dummy redirect URI for non-web apps.
REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'

#SCOPE= 'https://www.googleapis.com/auth/gmail.readonly'  
SCOPE= 'https://mail.google.com/'

ENDOFHEADER= "Number\r\n"

class Client
    
    def setDateLastUpdated:
        pass

    def setDateCreated:
        pass



class ClientExtractor:
    def __init__(self):
        self.auth_string= ""
        self.clientSet = []

    def InitializeCredentials(self):
        credentials = MiscTools.GetGmailCreds(CREDENTIALS_PATH)
        self.username = credentials['USERNAME']
        self.client_id = credentials['CLIENTID']
        self.client_token = credentials['CLIENTTOKEN']

    def GetRawClientList(self):
        if self.auth_string ==  "":
            print 'To authorize token, visit this url and follow the directions:'
            print '  %s' % OAuth2Tools.GeneratePermissionUrl(self.client_id, SCOPE)
            authorization_code = raw_input('Enter verification code: ')
            auth_tokens= OAuth2Tools.AuthorizeTokens(self.client_id, self.client_token, authorization_code)
            self.auth_string= OAuth2Tools.GenerateOAuth2String(self.username, auth_tokens['access_token'], base64_encode=False)
        latestRawEmail= EmailTools.GetLatestEmail(self.username,  self.auth_string)
        latestEmail= EmailTools.ConvertRawToEmailMessage(latestRawEmail)
        emailData= EmailTools.ConvertEmailMessageToData(latestEmail, 0)
        self.dataList = DataTools.BreakDataStringToDataList(emailData)

    def ConvertListToClients
        for eachStudent in self.dataList:
            entryDates = re.search('([0-9][0-9]\/[0-9][0-9]\/[0-9][0-9][0-9][0-9]).*?([0-9][0-9]\/[0-9][0-9]\/[0-9][0-9][0-9][0-9])', eachStudent)
            dateUpdated = entryDates.group(1)
            dateCreated = entryDates.group(2)
            


class DataTools:

    @staticmethod
    def BreakDataStringToDataList(dataString):
        dataList = re.split('[0-9][0-9][0-9][0-9][0-9][0-9]\-[0-9][0-9][0-9][0-9][0-9][0-9]', dataString)
        return dataList

class MiscTools:
    
    @staticmethod
    def GetGmailCreds(path_to_data_file):
        credentials = {}
        with open(path_to_data_file, 'r') as credsFile:
            for line in credsFile:
                (key, val) = line.split('=')
                key = key.replace(" ","")
                val = val.replace("\n","")
                val = val.replace(" ","")
                credentials[key] = val
        return credentials

class EmailTools:

    @staticmethod
    def GetLatestEmail(EMAILUSER,  auth_string):
        imap_conn = imaplib.IMAP4_SSL('imap.gmail.com')
        imap_conn.debug = 4
        imap_conn.authenticate('XOAUTH2', lambda x: auth_string)
        imap_conn.select('INBOX')
        result, data = imap_conn.uid('search', None, "ALL") # search and return uids instead
        latest_email_uid = data[0].split()[-1]
        result, data = imap_conn.uid('fetch', latest_email_uid, '(RFC822)')
        raw_email = data[0][1]
        return raw_email

    @staticmethod
    def ConvertRawToEmailMessage(raw_email):
        return email.message_from_string(raw_email)

#TODO:save text files of both raw emails to avoid data cap
    @staticmethod
    def ConvertEmailMessageToData(email_message, payload_index):
	emailPayload= email_message.get_payload(payload_index)
        dataDecodeable= emailPayload.get_payload(decode= True)
        dataDecoded= dataDecodeable.decode('utf-8')
        return dataDecoded
        #startDataIndex= dataDecoded.find(ENDOFHEADER)
        #return dataDecoded[(startDataIndex + len(ENDOFHEADER)):]



class OAuth2Tools:

    @staticmethod
    def AccountsUrl(command):
        """Generates the Google Accounts URL.

        Args:
            command: The command to execute.

        Returns:
            A URL for the given command.
        """
        return '%s/%s' % (GOOGLE_ACCOUNTS_BASE_URL, command)


    @staticmethod
    def UrlEscape(text):
        # See OAUTH 5.1 for a definition of which characters need to be escaped.
        return urllib.quote(text, safe='~-._')


{'fraction_non_alphanumeric': 0.1094391244870041, 'fraction_numerical': 0.014363885088919288, 'mean_word_length': 3.0428571428571427, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1502134', 'n_tokens_mistral': 2790, 'n_tokens_neox': 2723, 'n_words': 584}