<jupyter_start><jupyter_text>Get info of each gameHere are some examples of responded API:- https://api.rawg.io/api/games/rimworld- https://api.rawg.io/api/games/grand-theft-auto-v- https://rawg.io/games/grand-theft-auto-v<jupyter_code>import csv
import requests
import json
from pprint import pprint
from time import time
import concurrent.futures
import functools
import os<jupyter_output><empty_output><jupyter_text>Load CSV file which has game's id and its name<jupyter_code>csv_data = []
with open("../data/game_id.csv", "r") as f:
    csv_data = list(csv.reader(f))

# Preview
for i, val in enumerate(csv_data):
    print(val)
    if i==10: break<jupyter_output>['3498', 'grand-theft-auto-v']
['4200', 'portal-2']
['3328', 'the-witcher-3-wild-hunt']
['5286', 'tomb-raider']
['5679', 'the-elder-scrolls-v-skyrim']
['12020', 'left-4-dead-2']
['802', 'borderlands-2']
['4062', 'bioshock-infinite']
['13536', 'portal']
['3439', 'life-is-strange-episode-1-2']
['4291', 'counter-strike-global-offensive']<jupyter_text>Multithreading  This function is responsible for requesting each game and save as a JSON file in `/data/game_info/`<jupyter_code>def worker(start_index, games_per_worker, urls, downloaded_files, headers):
    for url in urls[start_index : start_index + games_per_worker]:
        if url.rsplit("/")[-1] in downloaded_files: continue
        try:
            # Request API
            json_data = json.loads(requests.get(url, headers=headers).text)

            # Only include wanted keys
            D = {k:v for k,v in json_data.items() if k in include}

            # Clean up dictionary
            D["platforms"] = []
            for platform in json_data["platforms"]:
                D["platforms"].append(platform["platform"]["name"])

            for key in ("developers", "genres", "publishers"):
                D[key] = []
                for data in json_data[key]:
                    D[key].append(data["name"])

            if json_data["esrb_rating"]:
                D["esrb_rating"] = json_data["esrb_rating"]["name"]
            
            # Save as JSON file
            name = D["id"]
            with open(f"../data/game_info/{name}.json","w", encoding="utf-8") as f:
                json.dump(D, f)
        except:
            print(f"Error with {url}")
# Create folder if not existed
if not os.path.exists('../data/game_info/'):
    os.makedirs('../data/game_info/')<jupyter_output><empty_output><jupyter_text>Threading Preparation<jupyter_code>headers = { 'User-Agent': 'App Name: Education purpose',}
include = {"id",
           "slug",
           "name",
           "metacritic",
           "released",
           "tba",
           "updated",
           "website",
           "rating",
           "rating_top",
           "added_by_status",
           "playtime",
           "achievements_count",
           "ratings_count",
           "suggestions_count",
           "game_series_count",
           "reviews_count",
           "platforms",
           "developers",
           "genres",
           "publishers",
           "esrb_rating",
           }

# Set up number of workers
max_workers = 64
start_game_index = 0
end_game_index = len(csv_data)
number_of_games = end_game_index - start_game_index
games_per_worker = int(number_of_games/max_workers) + 1 
start_index = range(start_game_index, end_game_index, games_per_worker)

# Make urls
base_url = "https://api.rawg.io/api/games/"
urls = [base_url + csv_data[i][0] for i in range(len(csv_data))]
# Skip downloaded files
downloaded_files = {file.split(".",1)[0] for file in os.listdir("../data/game_info/")}

# Time
t0 = time()
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
    temp = functools.partial(worker,
                             games_per_worker=games_per_worker,
                             urls=urls,
                             downloaded_files=downloaded_files,
                             headers=headers,
                            )
    executor.map(temp, start_index)
    
# Time
print(f"Time taken: {time()-t0}")<jupyter_output>Error with https://api.rawg.io/api/games/55494
Error with https://api.rawg.io/api/games/55172
Error with https://api.rawg.io/api/games/267083
Error with https://api.rawg.io/api/games/471035
Error with https://api.rawg.io/api/games/440682
Error with https://api.rawg.io/api/games/367202
Error with https://api.rawg.io/api/games/29079
Error with https://api.rawg.io/api/games/517088
Error with https://api.rawg.io/api/games/312611
Error with https://api.rawg.io/api/games/28446
Error with https://api.rawg.io/api/games/79200
Error with https://api.rawg.io/api/games/413880
Error with https://api.rawg.io/api/games/29123
Error with https://api.rawg.io/api/games/55027
Error with https://api.rawg.io/api/games/59025
Error with https://api.rawg.io/api/games/28703
Error with https://api.rawg.io/api/games/275734
Error with https://api.rawg.io/api/games/266581
Error with https://api.rawg.io/api/games/517387
Time taken: 1.2313072681427002
{'fraction_non_alphanumeric': 0.14319856602270464, 'fraction_numerical': 0.03724357697669787, 'mean_word_length': 2.728285077951002, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 23, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '406519', 'license': 'MIT', 'n_tokens_mistral': 1839, 'n_tokens_neox': 1694, 'n_words': 331, 'path': 'src/Get_game_info.ipynb', 'repo_name': 'trung-hn/game-encyclopedia', 'size': '5021'}