<commit_before>var page = require('webpage').create();
var system = require('system');
var fs = require('fs');

var options = {};
system.args.slice(1).forEach(function(arg) {
  var match = arg.match(/--([\w-]+)="?(.*)"?$/);
  if (match) {
    var key = match[1],
        value = match[2].replace(/(^")|("$)/g, '');
    options[key] = value;
  }
});

var state = 'login';
var accounts = [];
var currentAccount;
var urls = {
  login: 'https://www.schwab.com/public/schwab/client_home',
  loggedIn: 'https://client.schwab.com/Accounts/Summary/Summary.aspx?ShowUN=YES',
  loginError: 'https://client.schwab.com/Login/SignOn/CustomerCenterLogin.aspx?ErrorCode=ErrorInvalidCredentials'
};

page.onUrlChanged = defaultHandler;

var errorBlacklist = [
  /btn.onclick/
];
page.onError = function(msg) {
  var blacklist = errorBlacklist.some(function(pattern) { return pattern.test(msg); });
  if (blacklist) return;
  log('ERROR', msg.replace(/[\r\n]+/, ' '));
};

page.open(urls.login, function login(status) {
  if (status !== 'success') {
    die('Unable to load login page:', status);
  }

  log('Logging in as', options.username, '...');

  page.evaluate(function(username, password) {
    $('#SignonAccountNumber').val(username);
    $('#SignonPassword').val(password);
    submitLogin(); // this is a global function on schwab.com
  }, options.username, options.password);

  state = 'auth';
  nextPage(function(url) {
    if (url === urls.loggedIn) {
      log('Logged in!');
      listAccounts();
    } else {
      defaultHandler(url);
    }
  });
});

function listAccounts() {
  waitFor('#tblCharlesSchwabBank', function(selector) {
    var allAccounts = getAccounts(selector);
    if (allAccounts) {
      // XXX because you can't shift() an array returned by page.evaluate()
      accounts = [].slice.call(allAccounts);
      if (options.acct) {
        var lookup = makeMap(options.acct.split(','));
        accounts = accounts.filter(function(d) {
          return lookup.hasOwnProperty(d.name);
        });
      }

      log('accounts:', allAccounts
        .map(function(d) { return d.name; })
        .join(', '));
      nextAccount();
    } else {
      return die('Unable to load any accounts:', selector);
    }
  });
}

function nextPage(done) {
  page.onUrlChanged = function(url) {
    done(url);
    page.onUrlChanged = defaultHandler;
  };
}

function defaultHandler(url) {
  log('URL changed:', url);
  switch (url) {
    case urls.loginError:
      return die('Login error! Try re-typing your username and password.');
  }
};

function getAccounts(selector) {
  var accounts = page.evaluate(function(selector) {
    var table = document.querySelector(selector);
    var rows = table.querySelectorAll('tr.data-row');
    return [].map.call(rows, function(tr, i) {
      var a = tr.querySelector('td:first-child a');
      return {
        name: a.textContent,
        action: a.href.replace(/^javascript:/, '')
      };
    });
  }, selector);
  return [].slice.call(accounts);
}

function nextAccount() {
  state = 'account';
  if (!accounts.length) {
    return exit('All done!');
  }
  log('Loading next account of', accounts.length);
  var acct = accounts.shift();
  currentAccount = acct;
  log('Account:', acct.name);
  var submitted = page.evaluate(function(action) {
    return eval(action);
  }, acct.action);
  log('submitted:', submitted);
  nextPage(function(url) {
    loadTransactions();
  });
}

function loadTransactions() {
  log('Waiting for data on', page.url);
  waitFor('#tbldata', function(selector) {
    log('Reading transactions from:', selector);
    var data = page.evaluate(function(selector) {<commit_msg>add account number (last 4 chars) support<commit_after>var page = require('webpage').create();
var system = require('system');
var fs = require('fs');

var options = {};
system.args.slice(1).forEach(function(arg) {
  var match = arg.match(/--([\w-]+)="?(.*)"?$/);
  if (match) {
    var key = match[1],
        value = match[2].replace(/(^")|("$)/g, '');
    options[key] = value;
  }
});

var state = 'login';
var accounts = [];
var currentAccount;
var urls = {
  login: 'https://www.schwab.com/public/schwab/client_home',
  loggedIn: 'https://client.schwab.com/Accounts/Summary/Summary.aspx?ShowUN=YES',
  loginError: 'https://client.schwab.com/Login/SignOn/CustomerCenterLogin.aspx?ErrorCode=ErrorInvalidCredentials'
};

page.onUrlChanged = defaultHandler;

var errorBlacklist = [
  /btn.onclick/
];
page.onError = function(msg) {
  var blacklist = errorBlacklist.some(function(pattern) { return pattern.test(msg); });
  if (blacklist) return;
  log('ERROR', msg.replace(/[\r\n]+/, ' '));
};

page.open(urls.login, function login(status) {
  if (status !== 'success') {
    die('Unable to load login page:', status);
  }

  log('Logging in as', options.username, '...');

  page.evaluate(function(username, password) {
    $('#SignonAccountNumber').val(username);
    $('#SignonPassword').val(password);
    submitLogin(); // this is a global function on schwab.com
  }, options.username, options.password);

  state = 'auth';
  nextPage(function(url) {
    if (url === urls.loggedIn) {
      log('Logged in!');
      listAccounts();
    } else {
      defaultHandler(url);
    }
  });
});

function listAccounts() {
  waitFor('#tblCharlesSchwabBank', function(selector) {
    var allAccounts = getAccounts(selector);
    if (allAccounts) {
      // XXX because you can't shift() an array returned by page.evaluate()
      accounts = [].slice.call(allAccounts);
      if (options.account) {
        var lookup = makeMap(options.account.split(','));
        accounts = accounts.filter(function(d) {
          return lookup.hasOwnProperty(d.name)
              || lookup.hasOwnProperty(d.number);
        });
      }

      log('accounts:', accounts
        .map(function(d) { return d.name; })
        .join(', '));
      nextAccount();
    } else {
      return die('unable to load any accounts:', selector);
    }
  });
}

function nextPage(done) {
  page.onUrlChanged = function(url) {
    done(url);
    page.onUrlChanged = defaultHandler;
  };
}

function defaultHandler(url) {
  log('URL changed:', url);
  switch (url) {
    case urls.loginError:
      return die('Login error! Try re-typing your username and password.');
  }
};

function getAccounts(selector) {
  var accounts = page.evaluate(function(selector) {
    var table = document.querySelector(selector);
    var rows = table.querySelectorAll('tr.data-row');
    return [].map.call(rows, function(tr, i) {
      var name = tr.querySelector('td:first-child a'),
          number = tr.querySelector('td:nth-child(2) a');
      return {
        name: name.textContent,
        action: name.href.replace(/^javascript:/, ''),
        number: number.textContent.slice(-4)
      };
    });
  }, selector);
  return [].slice.call(accounts);
}

function nextAccount() {
  state = 'account';
  if (!accounts.length) {
    return exit('All done!');
  }
  log('Loading next account of', accounts.length);
  var acct = accounts.shift();
  currentAccount = acct;
  log('Account:', acct.name);
  var submitted = page.evaluate(function(action) {
    return eval(action);
  }, acct.action);
  log('submitted:', submitted);
  nextPage(function(url) {
    loadTransactions();
  });
}

function loadTransactions() {
  log('Waiting for data on', page.url);
  waitFor('#tbldata', function(selector) {
    log('Reading transactions from:', selector);
    var data = page.evaluate(function(selector) {
{'fraction_non_alphanumeric': 0.1743767497666978, 'fraction_numerical': 0.0011998400213304892, 'mean_word_length': 3.4654761904761906, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1288880', 'n_tokens_mistral': 2427, 'n_tokens_neox': 2420, 'n_words': 494}