<jupyter_start><jupyter_text>To test my nofaction validation tool I'll have to find a game that actually has them<jupyter_code>def find_nofaction_game():
    for json_file in json_dir.glob("*.json"):
        with open(json_file) as f:
            data = json.load(f) 
        for game in data:
            if any(faction["faction"].startswith("nofaction") for faction in game["factions"]):
                return game
nofaction_game = find_nofaction_game()
tmg2 = make_dataset.TerraMysticaGame(nofaction_game)
tmg2.factions
tmg2.game_options
tmg2._has_nofaction_player()<jupyter_output><empty_output><jupyter_text>OK, next let's identify expansion games<jupyter_code>def find_expansion_game():
    for json_file in json_dir.glob("*.json"):
        with open(json_file) as f:
            data = json.load(f) 
        for game in data:
            if any(faction["faction"] == "icemaidens" for faction in game["factions"]):
                return game

expansion_game = make_dataset.TerraMysticaGame(find_expansion_game())
expansion_game.factions
expansion_game.game_options<jupyter_output><empty_output><jupyter_text>I don't see anything in options that would identify it, I'll probably just go off factions<jupyter_code>expansion_game._is_played_on_original_map()<jupyter_output><empty_output><jupyter_text>That might be something but I'll keep that logic separate<jupyter_code>expansion_game._has_expansion_factions()
tmg2._has_expansion_factions()
tmg._has_expansion_factions()<jupyter_output><empty_output><jupyter_text>Seems to be working Ok, now let's identify if there are dropped players. Once that's done I think we're ready to build a dataset<jupyter_code>def find_drop_faction_game():
    for json_file in json_dir.glob("*.json"):
        with open(json_file) as f:
            data = json.load(f) 
        for game in data:
            if any(key.startswith("drop") for key in game["events"]["global"]):
                return game

drop_game = make_dataset.TerraMysticaGame(find_drop_faction_game())
list(drop_game._raw_game["events"]["global"].keys())
drop_game._raw_game["events"]["global"]["drop-faction"]
drop_game._has_dropped_faction()
tmg._has_dropped_faction()<jupyter_output><empty_output><jupyter_text>Cool, coolcoolcool. Let's wrap all these tests into a nice dataset validator<jupyter_code>tmg.is_valid_game
tmg2.is_valid_game
drop_game.is_valid_game
expansion_game.is_valid_game<jupyter_output><empty_output><jupyter_text>DataFrame Parsing<jupyter_code>game_df = pd.DataFrame()
player_df = pd.DataFrame()
make_dataset._game_to_dfs(tmg, game_df, player_df)
game_df
player_df
make_dataset._game_to_dfs(tmg2, game_df, player_df)
game_df<jupyter_output><empty_output><jupyter_text>Sweeeeeeeet  Let's fire it up Have this here for troubleshooting, can delete after parsing works<jupyter_code>def find_error_game(game_name):
    for json_file in json_dir.glob("*.json"):
        with open(json_file) as f:
            data = json.load(f) 
        for game in data:
            if game_name == game["game"]:
                return game

error_game = make_dataset.TerraMysticaGame(find_error_game("10"))
game_df, player_df = make_dataset.games_to_dfs()
game_df
player_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09947155735156979, 'fraction_numerical': 0.0027976375505129004, 'mean_word_length': 5.060263653483992, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '103783', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1128, 'n_tokens_neox': 1109, 'n_words': 225, 'path': 'notebooks/04_ip_even_more_json_parsing.ipynb', 'repo_name': 'ianepreston/terra-mystica-models', 'size': '3217'}