<jupyter_start><jupyter_text>Get all categories from various online websites.  This data can be used for categorization in machine learning<jupyter_code>import requests
from time import sleep
import random
import json
import pprint as pp
url = "http://developer.shopclues.com/api/v1/category"

headers = {
    'authorization': "Bearer 11111112172617652167452716",
    'content-type': "application/json",
    'cache-control': "no-cache"
}
visited = {}
unvisited = {}

while True:
    response = requests.request("GET", url, headers=headers)
    if response.status_code == 200 and not isinstance(json.loads(response.text)['data'], basestring):
        data = json.loads(response.text)
        break

for cat in data['data']:
    if cat['is_leaf_category'] == 'N':
        unvisited.update({
                cat["category_id"]: {
                    "name":cat['category'],
                    "parent":0
                }
            })
    else:
        visited.update({
                cat["category_id"]: {
                    "name":cat['category'],
                    "parent":0
                }
            })
print(len(visited))
print(len(unvisited))
while unvisited:
    unv = unvisited.keys()[0]
    print(unv)
    while True:
        sleep_time = random.randint(1, 2000)
        querystring = {"category_id":unv}

        response = requests.request("GET", url, headers=headers, params=querystring)
        sleep(sleep_time/1000)
        
        if response.status_code == 200 and not isinstance(json.loads(response.text)['data'], basestring):
            break
        print("status " + str(response.status_code))
    
    data = json.loads(response.text)
    
    for cat in data['data']:
        if cat['is_leaf_category'] == 'N':
            unvisited.update({
                    cat["category_id"]: {
                        "name":cat['category'],
                        "parent":int(unv)
                    }
                })
        else:
            visited.update({
                    cat["category_id"]: {
                        "name":cat['category'],
                        "parent":int(unv)
                    }
                })
            
    visited.update({unv:unvisited.get(unv)})
    del unvisited[unv]
    print(str(len(visited)) + " " + str(len(unvisited)))
#pp.pprint(visited)
#pp.pprint(unvisited)
list_categories = []
for x in visited:
    parent_id = str(visited[x]['parent'])
    parent_cat = ""
    while parent_id != '0' and parent_id in visited:
        parent_cat = str(visited[parent_id]['name']) + " > " + parent_cat
        parent_id = str(visited[parent_id]['parent'])
    list_categories.append([
            x,
            visited[x]['name'],
            parent_cat,
            visited[x]['parent']
        ])
import csv

myfile = open("sclues_categories_"+str(random.randint(1, 2000))+".csv", 'wb')
wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
wr.writerows(list_categories)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1365015166835187, 'fraction_numerical': 0.017189079878665317, 'mean_word_length': 2.114375655823715, 'pattern_counts': {'":': 9, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '616954', 'license': 'MIT', 'n_tokens_mistral': 945, 'n_tokens_neox': 891, 'n_words': 164, 'path': 'get_related_categories_online.ipynb', 'repo_name': 'xRahul/ClusterSMS', 'size': '2967'}