<commit_before>import json
import requests
from collections import defaultdict

lists_url = "https://api.crowdtangle.com/lists"

headers = {
    'x-api-token': "<KEY>" , #Add CT dashboard token here
    'Cache-Control': "no-cache",
    }

lists = requests.request("GET", lists_url, headers=headers)
lists = json.loads(lists.text)
# print(lists_response)
list_dict = {}
#storing list ids and list names
for obj in lists["result"]["lists"]:
    # print(obj["id"])
    list_dict[str(obj["id"])] = {"title": obj["title"]}

# print(json.dumps(list_dict, indent=4, ensure_ascii=False))

#prepping for paginations
paginations = []
for l_name in list_dict.keys():
    paginations.append({"list_id":str(l_name), "request":"https://api.crowdtangle.com/lists/{}/accounts".format(str(l_name))})


while paginations:
    acc_obj = paginations.pop(0)
    list_id = acc_obj["list_id"]
    acc_request = acc_obj["request"]
    print("Current request: {}\n".format(acc_request))
    accounts_response = requests.request("GET", acc_request, headers=headers)
    accounts_response = json.loads(accounts_response.text)
    if "result" in accounts_response:
        for acc in accounts_response["result"]["accounts"]:
            if "accounts" not in list_dict[list_id]:
                list_dict[list_id]["accounts"] = [acc["id"]]<commit_msg>removed token<commit_after>import json
import requests
from collections import defaultdict

lists_url = "https://api.crowdtangle.com/lists"

headers = {
    'x-api-token': "" , #Add CT dashboard token here
    'Cache-Control': "no-cache",
    }

lists = requests.request("GET", lists_url, headers=headers)
lists = json.loads(lists.text)
# print(lists_response)
list_dict = {}
#storing list ids and list names
for obj in lists["result"]["lists"]:
    # print(obj["id"])
    list_dict[str(obj["id"])] = {"title": obj["title"]}

# print(json.dumps(list_dict, indent=4, ensure_ascii=False))

#prepping for paginations
paginations = []
for l_name in list_dict.keys():
    paginations.append({"list_id":str(l_name), "request":"https://api.crowdtangle.com/lists/{}/accounts".format(str(l_name))})


while paginations:
    acc_obj = paginations.pop(0)
    list_id = acc_obj["list_id"]
    acc_request = acc_obj["request"]
    print("Current request: {}\n".format(acc_request))
    accounts_response = requests.request("GET", acc_request, headers=headers)
    accounts_response = json.loads(accounts_response.text)
    if "result" in accounts_response:
        for acc in accounts_response["result"]["accounts"]:
            if "accounts" not in list_dict[list_id]:
                list_dict[list_id]["accounts"] = [acc["id"]]
{'fraction_non_alphanumeric': 0.15687760515346721, 'fraction_numerical': 0.0015157256536566881, 'mean_word_length': 4.689655172413793, 'pattern_counts': {'":': 6, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3731373', 'n_tokens_mistral': 869, 'n_tokens_neox': 874, 'n_words': 173}