<commit_before>#!/usr/bin/env python

import argparse
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
import configparser
import imaplib
import json
import os.path
import pickle
import re
import traceback

HAVE_READ_FILE = 'have-processed.json'

# If modifying these scopes, delete the file token.pickle.
SCOPES = ['https://www.googleapis.com/auth/spreadsheets']

# The ID and range of a sample spreadsheet.
SPREADSHEET_ID = '1h-GBpn__5CG-jlG1LuQbooNaOm_rLQmBmz6OS1GdaeM'
RANGE_NAME = 'Transactions!A1:B'

def main(dry_run, proc_all):
    print(dry_run, proc_all)
    config = configparser.ConfigParser()
    config.read('imap-creds.ini');

    username = config.get('creds', 'username')
    password = config.get('creds', 'password')
    hostname = config.get('server', 'hostname')

    have_processed = {}
    try:
        if not proc_all:
            have_processed = json.load(open(HAVE_READ_FILE))
    except:
        pass

    with imaplib.IMAP4_SSL(hostname) as M:
        M.login(username, password)
        M.select('INBOX', readonly=True)
        message_ids_from_chase = M.search('NONE', 'FROM', '"chase.com"')[1][0].split()

        entries = []
        for mid in message_ids_from_chase:
            d_mid = mid.decode("utf-8")
            if d_mid not in have_processed:
                try:
                    message = M.fetch(d_mid, '(BODY.PEEK[TEXT])')[1][0][1]
                    decoded_message = message.decode("UTF-8")
                    scrubbed_message = decoded_message.replace("=", "").replace("\n", "").lower()
                    if "credit card statement is ready" in scrubbed_message:
                        continue
                    amount, vendor, datestr = re.findall("\(\$USD\) ([0-9.]*) at (.*) has .* authorized on (.*) at", decoded_message)[0]
                    entries.append([vendor, amount])
                    have_processed[d_mid] = True
                    print(d_mid, amount, vendor, datestr)
                except Exception as e:
                    print("Couldn't process: {}".format(decoded_message))
                    traceback.print_exc()

        if not dry_run:
            add_to_spreadsheet(entries)

    if not dry_run:
        with open(HAVE_READ_FILE, "w+") as f:
            json.dump(have_processed, f)

def add_to_spreadsheet(entries):
    creds = None
    # The file token.pickle stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    if os.path.exists('token.pickle'):
        with open('token.pickle', 'rb') as token:
            creds = pickle.load(token)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(<commit_msg>add more skipped strings to email-transaction-to-budget<commit_after>#!/usr/bin/env python

import argparse
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
import configparser
import imaplib
import json
import os.path
import pickle
import re
import traceback

HAVE_READ_FILE = 'have-processed.json'

# If modifying these scopes, delete the file token.pickle.
SCOPES = ['https://www.googleapis.com/auth/spreadsheets']

# The ID and range of a sample spreadsheet.
SPREADSHEET_ID = '1h-GBpn__5CG-jlG1LuQbooNaOm_rLQmBmz6OS1GdaeM'
RANGE_NAME = 'Transactions!A1:B'

skip_strings_in_body = [
    "Your credit card payment is due soon",
    "as you requested, we are notifying you of an international charge",
    "credit card statement is ready",
]

def main(dry_run, proc_all):
    config = configparser.ConfigParser()
    config.read('imap-creds.ini');

    username = config.get('creds', 'username')
    password = config.get('creds', 'password')
    hostname = config.get('server', 'hostname')

    have_processed = {}
    try:
        if not proc_all:
            have_processed = json.load(open(HAVE_READ_FILE))
    except:
        pass

    with imaplib.IMAP4_SSL(hostname) as M:
        M.login(username, password)
        M.select('INBOX', readonly=True)
        message_ids_from_chase = M.search('NONE', 'FROM', '"chase.com"')[1][0].split()

        entries = []
        for mid in message_ids_from_chase:
            d_mid = mid.decode("utf-8")
            if d_mid not in have_processed:
                try:
                    message = M.fetch(d_mid, '(BODY.PEEK[TEXT])')[1][0][1]
                    decoded_message = message.decode("UTF-8")
                    scrubbed_message = decoded_message.replace("=", "").replace("\n", "")

                    matching_skipped = [s for s in skip_strings_in_body if s.lower() in scrubbed_message.lower()]
                    if len(matching_skipped) > 0:
                        continue

                    amount, vendor, datestr = re.findall("\(\$USD\) ([0-9.]*) at (.*) has .* authorized on (.*) at", decoded_message)[0]
                    entries.append([vendor, amount])
                    have_processed[d_mid] = True
                    print(d_mid, amount, vendor, datestr)
                except Exception as e:
                    print("Couldn't process: {}".format(decoded_message))
                    traceback.print_exc()

        if not dry_run:
            add_to_spreadsheet(entries)

    if not dry_run:
        with open(HAVE_READ_FILE, "w+") as f:
            json.dump(have_processed, f)

def add_to_spreadsheet(entries):
    creds = None
    # The file token.pickle stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    if os.path.exists('token.pickle'):
        with open('token.pickle', 'rb') as token:
            creds = pickle.load(token)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
{'fraction_non_alphanumeric': 0.10877470355731225, 'fraction_numerical': 0.005533596837944664, 'mean_word_length': 2.516397998888271, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1792257', 'n_tokens_mistral': 2004, 'n_tokens_neox': 1906, 'n_words': 486}