<jupyter_start><jupyter_text>Getting data from OpenLab<jupyter_code>import json

with open('Accounts') as json_file:  
    data = json.load(json_file)<jupyter_output><empty_output><jupyter_text>In data: (1) Power Users; (2) Customers<jupyter_code>import requests
import csv

# It seems that most of the customers in the beginning are not loaded with transaction data 

for user in data[1][::-1][0:10]:
    # login
    url = 'https://openlab.openbankproject.com/my/logins/direct'
    headers = {'Authorization': 'DirectLogin username="'+user['user_name']+'", password="'+user['password']+'",  consumer_key="<KEY>"',
               'Content-Type': 'application/json'}
    r=requests.post(url, headers=headers)
    tok = json.loads(r.content.decode('utf-8'))['token']
    # get accounts held by user
    baseURL = 'https://openlab.openbankproject.com'
    url = baseURL + '/obp/v3.1.0/banks/hsbc.02.hk.hsbc/accounts-held'
    headers ={ 'Authorization': 'DirectLogin token="'+ tok +'"'}
    r=requests.get(url, headers=headers)

    result=json.loads(r.content.decode('utf-8'))
    accounts = result['accounts']
    if(len(accounts)<1):
        pass
    print('processing: '+user['user_name'])
    # Get transaction for each account
    for ind in range(len(accounts)):
        account = accounts[ind]
        
        # Get account balance
        url = 'https://openlab.openbankproject.com/obp/v3.1.0/banks/hsbc.02.hk.hsbc/accounts/'+ account['id'] +'/owner/account'
        headers = {
            #'limit' : '100',
            #'to_date' : '2019-06-12T00:00:00.000Z',
            'Authorization': 'DirectLogin token="'+ tok +'"'
        }
        r=requests.get(url, headers=headers)
        results=json.loads(r.content.decode('utf-8'))
        print(results)
        # Get transaction
        url = 'https://openlab.openbankproject.com/obp/v3.1.0/banks/hsbc.02.hk.hsbc/accounts/'+ account['id'] +'/owner/transactions'
        headers = {
            'limit' : '100',
            'to_date' : '2019-06-11T00:00:00.000Z',
            'Authorization': 'DirectLogin token="'+ tok +'"'
        }
        r=requests.get(url, headers=headers)
        
        print('no account: ' + str(ind+1))
        fileName = 't_'+user['user_name']+'a_'+str(ind)+'.csv'
        results=json.loads(r.content.decode('utf-8'))
        
        fp=open(fileName, 'a+')
        fp.write('description,completed,value,new_balance\n')
        for result in results['transactions']:
            result=result['details']
            a=[result[x] for x in ['description', 'completed', 'value', 'new_balance']]
            a[2]=a[2]['amount'] + a[2]['currency']
            a[3]=a[3]['amount'] + a[3]['currency']
            wr = csv.writer(fp, dialect='excel')
            wr.writerow(a)
        fp.close()

# Each transaction consist of:
results['transactions'][0].keys()

# Only used details here
results['transactions'][0]['details']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17358747447243023, 'fraction_numerical': 0.025867937372362152, 'mean_word_length': 3.0150273224043715, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '70942', 'license': 'Apache-2.0', 'n_tokens_mistral': 1014, 'n_tokens_neox': 973, 'n_words': 168, 'path': 'Data/DataGrab.ipynb', 'repo_name': 'ludwigHoon/PoohWiser', 'size': '2973'}