<commit_before>'''Process member's mailing list email bounces by logging into the mail server
mailbox which receives the bounces, inspecting the bounced mails,
and unsubcribing the members associated with the bounced mails.
Also deletes over-quota and holiday replies from the mailbox.
invoke with python2 mailoutomatic.py
'''
import os
import sys
import datetime
import imaplib
import email
import string
import random
import StringIO
import rfc822
import re
from urllib import urlopen
import logging

'''
File: imaplib-example-1.py
from http://effbot.org/librarybook/imaplib.htm

See also
http://www.doughellmann.com/PyMOTW/imaplib/index.html

Recipe 52560: Remove duplicates from a sequence
http://code.activestate.com/recipes/52560/
http://python.about.com/od/pythonstandardlibrary/ss/email_whitelist_5.htm

TODO
* Rewrite for python3
* Cope with Re: in subject line
* Unacceptable Mail Content
* spam detected
* "Message contains spam or virus"
* Use MD5 password
'''

mailserver_default = 'localhost'
user_default = 'admin'
mailbox_default = 'Inbox.logs.mailout bounces'
site = 'cubecinema.com'

verbose = True
deleteStuff = True  # Delete processed mails from mailbox
unsubscribe = True  # Unsubscribe members from toolkit
expunge = True  # Expunge mailbox on completion
maxMesgNo = 1000  # Max number of mails to process. Use a lower number for testing


def emailHits(domain, domains):
    if domain in domains:
        domains[domain] = domains[domain] + 1
    else:
        domains[domain] = 1


'''
Examples of gone away strings

Mailbox disabled
mailbox unavailable
This is a permanent error
Unrouteable address  (bris.ac.uk)
recipient rejected
The email account that you tried to reach does not exist (google)
This user doesn't have a yahoo.co.uk account
Rcpt <johnsmith<EMAIL>> does not exist
unable to validate recipient  (virgin/blueyonder)
Delivery has failed
'''

GoneAway = 'Mailbox disabled|mailbox unavailable|Mailbox is inactive|This is a permanent error'
GoneAway += '|Unrouteable address|recipient rejected|unknown or illegal alias'
GoneAway += '|The email account that you tried to reach does not exist'
GoneAway += "|This user doesn't have a .* account|Address rejected"
GoneAway += "|Rcpt .* does not exist|recipient.*known|Recipient address rejected"
GoneAway += "|unable to validate recipient|Invalid recipient"
GoneAway += "|account has been disabled or discontinued"
GoneAway += "|User unknown|Unknown user|User is unknown|account has been disabled"
GoneAway += "|No such user|Recipient address rejected|User does not exist"
GoneAway += "|No such recipient|no mailbox|No such mailbox"
GoneAway += "|Delivery.*failed|inactive user"
GoneAway += "|email account that you tried to reach is disabled"
GoneAway += "|email account that you tried to reach does not exist"
GoneAway += "|Unknown local part|MAILBOX NOT FOUND"
GoneAway += "|This email address is not known to this system"
GoneAway += "|unknown or illegal alias|disabled mailbox"
GoneAway += "|cannot find address on system|is a deactivated mailbox"
GoneAway += "|Incoming mail not allowed to this address"
GoneAway += "|Account Inactive"
GoneAway += "|dd Requested mail action aborted"
GoneAway += "|This mailbox is disabled"

SpamAlleged = "Your email was detected as spam"
SpamAlleged += "|Message contains spam"
SpamAlleged += "|Message is spam"
SpamAlleged += "|detected as spam"
SpamAlleged += "|looks like spam"
SpamAlleged += "|rejected by recipients spam filter"
SpamAlleged += "|spam detected"
SpamAlleged += "|detected as spam"
SpamAlleged += "|Message rejected as spam"
SpamAlleged += "|this message looked like spam"

'''Examples of over quota strings
full, over quota
mailbox is full
Over quota'''

OverQuota = 'over quota|mailbox.*full|overquota|too many messages|quota exceeded|user has exhausted allowed storage space|full mailbox'

SPF = "SPF verification failed for host"

# Name server errors
NameServerError = 'name service error'

LOG_DIR = '/var/log/cubetoolkit'
LOG_FILENAME = 'mailoutomatic-%s.log' % datetime.datetime.now().strftime("%d-%b-%Y-%H:%m:%S")

'''logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s',
                    filename=LOG_FILENAME,
                    filemode='w')
'''
# https://docs.python.org/2/howto/logging.html
# create logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)

# create formatter
formatter = logging.Formatter(<commit_msg>Updated over quota messages<commit_after>'''Process member's mailing list email bounces by logging into the mail server
mailbox which receives the bounces, inspecting the bounced mails,
and unsubcribing the members associated with the bounced mails.
Also deletes over-quota and holiday replies from the mailbox.
invoke with python2 mailoutomatic.py
'''
import os
import sys
import datetime
import imaplib
import email
import string
import random
import StringIO
import rfc822
import re
from urllib import urlopen
import logging

'''
File: imaplib-example-1.py
from http://effbot.org/librarybook/imaplib.htm

See also
http://www.doughellmann.com/PyMOTW/imaplib/index.html

Recipe 52560: Remove duplicates from a sequence
http://code.activestate.com/recipes/52560/
http://python.about.com/od/pythonstandardlibrary/ss/email_whitelist_5.htm

TODO
* Rewrite for python3
* Cope with Re: in subject line
* Unacceptable Mail Content
* spam detected
* "Message contains spam or virus"
* Use MD5 password
'''

mailserver_default = 'localhost'
user_default = 'admin'
mailbox_default = 'Inbox.logs.mailout bounces'
site = 'cubecinema.com'

verbose = True
deleteStuff = True  # Delete processed mails from mailbox
unsubscribe = True  # Unsubscribe members from toolkit
expunge = True  # Expunge mailbox on completion
maxMesgNo = 10000  # Max number of mails to process. Use a lower number for testing


def emailHits(domain, domains):
    if domain in domains:
        domains[domain] = domains[domain] + 1
    else:
        domains[domain] = 1


'''
Examples of gone away strings

Mailbox disabled
mailbox unavailable
This is a permanent error
Unrouteable address  (bris.ac.uk)
recipient rejected
The email account that you tried to reach does not exist (google)
This user doesn't have a yahoo.co.uk account
Rcpt <<EMAIL>> does not exist
unable to validate recipient  (virgin/blueyonder)
Delivery has failed
'''

GoneAway = 'Mailbox disabled|mailbox unavailable|Mailbox is inactive|This is a permanent error'
GoneAway += '|Unrouteable address|recipient rejected|unknown or illegal alias'
GoneAway += '|The email account that you tried to reach does not exist'
GoneAway += "|This user doesn't have a .* account|Address rejected"
GoneAway += "|Rcpt .* does not exist|recipient.*known|Recipient address rejected"
GoneAway += "|unable to validate recipient|Invalid recipient"
GoneAway += "|account has been disabled or discontinued"
GoneAway += "|User unknown|Unknown user|User is unknown|account has been disabled"
GoneAway += "|No such user|Recipient address rejected|User does not exist"
GoneAway += "|No such recipient|no mailbox|No such mailbox"
GoneAway += "|Delivery.*failed|inactive user"
GoneAway += "|email account that you tried to reach is disabled"
GoneAway += "|email account that you tried to reach does not exist"
GoneAway += "|Unknown local part|MAILBOX NOT FOUND"
GoneAway += "|This email address is not known to this system"
GoneAway += "|unknown or illegal alias|disabled mailbox"
GoneAway += "|cannot find address on system|is a deactivated mailbox"
GoneAway += "|Incoming mail not allowed to this address"
GoneAway += "|Account Inactive"
GoneAway += "|dd Requested mail action aborted"
GoneAway += "|This mailbox is disabled"

SpamAlleged = "Your email was detected as spam"
SpamAlleged += "|Message contains spam"
SpamAlleged += "|Message is spam"
SpamAlleged += "|detected as spam"
SpamAlleged += "|looks like spam"
SpamAlleged += "|rejected by recipients spam filter"
SpamAlleged += "|spam detected"
SpamAlleged += "|detected as spam"
SpamAlleged += "|Message rejected as spam"
SpamAlleged += "|this message looked like spam"

'''Examples of over quota strings
full, over quota
mailbox is full
Over quota'''

OverQuota = 'over quota|mailbox.*full|overquota|too many messages|quota exceeded|user has exhausted allowed storage space|full mailbox|temporarily deferred'

SPF = "SPF verification failed for host"

# Name server errors
NameServerError = 'name service error'

LOG_DIR = '/var/log/cubetoolkit'
LOG_FILENAME = 'mailoutomatic-%s.log' % datetime.datetime.now().strftime("%d-%b-%Y-%H:%m:%S")

'''logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s',
                    filename=LOG_FILENAME,
                    filemode='w')
'''
# https://docs.python.org/2/howto/logging.html
# create logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)

# create formatter
formatter = logging.Formatter(
{'fraction_non_alphanumeric': 0.09481530198399649, 'fraction_numerical': 0.005590266359750082, 'mean_word_length': 5.014502307185234, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7021780', 'n_tokens_mistral': 2924, 'n_tokens_neox': 2770, 'n_words': 989}