<commit_before>#!/usr/bin/python
# This is the main client-side scraping program.
# It will download a list of accounts, and run their scrapers.
import os
import sys
import api
import json
import getpass
import traceback
from selenium import webdriver

sys.path.append((os.path.dirname(__file__) or ".") + "/../")

import config

# Banks
banks = {}
for bank in config.banks:
    exec "import %s" % (bank)
    banks[bank] = eval(bank)

print "Login"
username = raw_input("Username: ")
password = getpass.getpass()

if not api.callapi("login",{"username": username, "password": password}):
    print "Login failed"
    sys.exit(1)

todo = api.callapi("accountstodo")

b = webdriver.Chrome()

for account in todo:
    if account["bankname"] not in banks:
        print "No scraper for %s!" % (account["bankname"])
        continue
    print "Scraping %s..." % (account["bankname"])
    try:
        if os.getenv("DATAFILE") and os.path.exists(account["bankname"]+".json"):
            data = open(account["bankname"]+".json").read()
        else:
            data = json.dumps(banks[account["bankname"]].downloadaccount(b, account),default=str)
            if os.getenv("DATAFILE"):
                open(account["bankname"]+".json","w").write(data)
        if not api.callapi("newtransactions", {"data": data}):
            print "Error uploading transactions"
    except Exception, e:
        print e
        traceback.print_exc()

b.quit()

api.callapi("logout")<commit_msg>Ask for passwords upfront before browser starts.<commit_after>#!/usr/bin/python
# This is the main client-side scraping program.
# It will download a list of accounts, and run their scrapers.
import os
import sys
import api
import json
import getpass
import traceback
from selenium import webdriver

sys.path.append((os.path.dirname(__file__) or ".") + "/../")

import config

# Banks
banks = {}
for bank in config.banks:
    exec "import %s" % (bank)
    banks[bank] = eval(bank)

print "Login"
username = raw_input("Username: ")
password = getpass.getpass()

if not api.callapi("login",{"username": username, "password": password}):
    print "Login failed"
    sys.exit(1)

todo = api.callapi("accountstodo")

for account in todo:
    if account.get("username") and "password" not in account:
        account["password"] = getpass.getpass("Password for %s (%s): " % (account["name"], account["username"]))

b = webdriver.Chrome()

for account in todo:
    if account["bankname"] not in banks:
        print "No scraper for %s!" % (account["bankname"])
        continue
    print "Scraping %s..." % (account["bankname"])
    try:
        if os.getenv("DATAFILE") and os.path.exists(account["bankname"]+".json"):
            data = open(account["bankname"]+".json").read()
        else:
            data = json.dumps(banks[account["bankname"]].downloadaccount(b, account),default=str)
            if os.getenv("DATAFILE"):
                open(account["bankname"]+".json","w").write(data)
        if not api.callapi("newtransactions", {"data": data}):
            print "Error uploading transactions"
    except Exception, e:
        print e
        traceback.print_exc()

b.quit()

api.callapi("logout")
{'fraction_non_alphanumeric': 0.15198237885462554, 'fraction_numerical': 0.0006293266205160479, 'mean_word_length': 3.378787878787879, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '781282', 'n_tokens_mistral': 990, 'n_tokens_neox': 969, 'n_words': 267}