<jupyter_start><jupyter_text>Lichess Puzzles Ein Überblick über den Datensatz<jupyter_code>import numpy
import pandas
import pickle<jupyter_output><empty_output><jupyter_text>Das Lichess Puzzle Dataset kann man sich von https://database.lichess.org/puzzles herunterladen.<jupyter_code>column_names = ('PuzzleId','FEN','Moves','Rating','RatingDeviation','Popularity','NbPlays','Themes','GameUrl')
lichess_puzzles = pandas.read_csv("data/lichess_db_puzzle.csv", header=None, names=column_names)
lichess_puzzles.head()
lichess_puzzles.describe()<jupyter_output><empty_output><jupyter_text>Suche nach schwierigen Aufgaben<jupyter_code># This flag allows to disable the use of already computed data after
# changing filter parameters or updating the puzzle database
use_cache = True

# Relative path to cache file
cache_file = "pickles/hard_puzzles.pickle"
def find_hard_puzzles(puzzles_df: pandas.DataFrame, min_nbPlays=100, min_rating=2500, max_deviation=80, deviation_scaling=0.1):
    hard_puzzles = []
    for index, puzzle in puzzles_df.iterrows():
        if puzzle["NbPlays"] >= min_nbPlays and puzzle["Rating"] >= min_rating and \
                puzzle["RatingDeviation"] <= max_deviation + (puzzle["Rating"] - min_rating) * deviation_scaling:
            hard_puzzles.append(puzzle)
    return hard_puzzles
if use_cache:
    try:
        with open(cache_file, "rb") as fp:  # Unpickling if already computed
            hard_puzzles = pickle.load(fp)
    except:
        hard_puzzles = find_hard_puzzles(lichess_puzzles)
else:
    hard_puzzles = find_hard_puzzles(lichess_puzzles)

# save hard puzzles for later
with open(cache_file, "wb") as fp:
    pickle.dump(hard_puzzles, fp)
def print_puzzle_list(puzzles, show_move_number=False, show_game_url=True):
    head = "ID    Rating  Deviation  NbPlays  Popularity"
    if show_move_number:
        head += "  Moves"
    if show_game_url:
        head += "  GameURL" 
    print(head)
    for puzzle in puzzles:
        line = puzzle["PuzzleId"] + "   " + str.rjust(str(puzzle["Rating"]),4) + "        " + \
                    str.rjust(str(puzzle["RatingDeviation"]), 3) + \
                    str.rjust(str(puzzle["NbPlays"]), 9) + "          " + str(puzzle["Popularity"])
        if show_move_number:
            line += str.rjust(str((puzzle["Moves"].count(' ') + 1) // 2), 7)
        if show_game_url:
            line += "  " + puzzle["GameUrl"]
        print(line)
    print("Insgesamt:", len(puzzles))
print_puzzle_list(hard_puzzles, show_game_url=False)
hard_puzzles_df = pandas.DataFrame.from_records(hard_puzzles, columns=column_names)

superhard_puzzles = find_hard_puzzles(hard_puzzles_df, \
                                      min_nbPlays=140, min_rating=2600, max_deviation=75, deviation_scaling=0.08)

superhard_puzzles.sort(reverse=True, key=lambda p: p["Rating"])
print_puzzle_list(superhard_puzzles)<jupyter_output>ID    Rating  Deviation  NbPlays  Popularity  GameURL
mW0J3   2878         92      348          89  https://lichess.org/Fz1ew5a9#97
F9PRJ   2799         90      172          83  https://lichess.org/Gqql5MbC/black#38
GX19F   2799         86      602          93  https://lichess.org/PgMsNJ4S/black#78
Wiw7f   2799         87      334          91  https://lichess.org/z3JKtf3o/black#76
1rQVB   2798         87      339          88  https://lichess.org/mhJp2mxn/black#58
54zQR   2798         90      427          95  https://lichess.org/oQ0eCQaX/black#32
ojaVO   2798         88      696          94  https://lichess.org/zmlMrzs0#83
GrsBq   2797         90      673          93  https://lichess.org/9glDsAzH#61
UajG5   2797         90      177          84  https://lichess.org/0RISYYGF#99
pzMts   2797         90      177          83  https://lichess.org/eZuXKyjA/black#64
v1MdQ   2797         89      157          95  https://lichess.org/zBt1MEio#79
G4dQn   2796         88      172          80  https:[...]<jupyter_text>Sehr lange AufgabenHier suche ich anhand der Zuganzahl in der Lösung unter den schwierigsten Aufgaben, nach denen mit den meisten erzwungenen Zügen und nach den längsten Matts.<jupyter_code>def find_long_puzzles(puzzles_df: pandas.DataFrame, only_mates=False, min_moves=8):
    '''Filter a puzzle dataset for puzzles that require the player to make many correct single moves in a row.'''
    long_puzzles = []
    for index, puzzle in puzzles_df.iterrows():
        if (not only_mates or "mate" in puzzle["Themes"]) and puzzle["Moves"].count(' ') >= 2 * min_moves -1:
            long_puzzles.append(puzzle)
    return long_puzzles
long_hard_puzzles = find_long_puzzles(hard_puzzles_df)
long_hard_puzzles.sort(reverse=True, key=lambda p: 10000 * p["Moves"].count(' ') + p["Rating"])
print_puzzle_list(long_hard_puzzles, show_move_number=True)
long_mates = find_long_puzzles(hard_puzzles_df, only_mates=True, min_moves=7)
long_mates.sort(reverse=True, key=lambda p: 10000 * p["Moves"].count(' ') + p["Rating"])
print_puzzle_list(long_mates, show_move_number=True)<jupyter_output>ID    Rating  Deviation  NbPlays  Popularity  Moves  GameURL
kuxpS   2701         97     1296          94     10  https://lichess.org/2ZGZECbS#65
kxyJ7   2758         95     1624          95      8  https://lichess.org/vkyuwHBc#41
QGtzX   2679         94     1746          93      8  https://lichess.org/HydDdc2X/black#70
uxBsV   2678         96      208          89      8  https://lichess.org/MsPdISIs/black#38
Swhm6   2583         84     1880          92      8  https://lichess.org/xqa9Ospm/black#20
42guN   2761         93     1150          94      7  https://lichess.org/BYYWwm4m#47
eoA0f   2754        101      761          86      7  https://lichess.org/5SvlonmZ/black#58
Lk9ZY   2726         92     1058          93      7  https://lichess.org/sz6bmlph/black#40
5QUgd   2685         92     1831          97      7  https://lichess.org/jcPI9orx#65
88g2C   2684         96      205          86      7  https://lichess.org/CKUjthOO/black#36
73N3v   2682         93      649          86      7  [...]<jupyter_text>Eine Überprüfung auf dem gesamten Aufgaben-Set hat ergeben, dass dieses Matt in 10 tatsächlich auch das längest Matt in den Lichess-Aufgaben ist.  Beliebte und unbeliebte AufgabenDer mittlere Beliebtheitswert liegt bei 89 (der Durchschnitt wird von einigen klaren Ausreißern auf 81 heruntergezogen), das 75-Prozentil bei 94. Daher wird bei unbeliebten Aufgaben vermutlich deutlich klarer sein, wieso sie als unbefriedigend markiert wurden, während die beliebtesten nur minimal herausstechen werden.<jupyter_code>def find_popular_puzzles(puzzles_df: pandas.DataFrame, min_popularity=100, min_nbPlays=150):
    popular_puzzles = []
    for index, puzzle in puzzles_df.iterrows():
        if puzzle["NbPlays"] >= min_nbPlays and puzzle["Popularity"] >= min_popularity:
            popular_puzzles.append(puzzle)
    return popular_puzzles

def find_unpopular_puzzles(puzzles_df: pandas.DataFrame, max_popularity=-1, min_nbPlays=150):
    unpopular_puzzles = []
    for index, puzzle in puzzles_df.iterrows():
        if puzzle["NbPlays"] >= min_nbPlays and puzzle["Popularity"] <= max_popularity:
            unpopular_puzzles.append(puzzle)
    return unpopular_puzzles
popular_hard_puzzles = find_popular_puzzles(hard_puzzles_df)
popular_hard_puzzles.sort(reverse=True, key=lambda p: p["Rating"])
print_puzzle_list(popular_hard_puzzles)
unpopular_hard_puzzles = find_unpopular_puzzles(hard_puzzles_df, max_popularity=50, min_nbPlays=100)
unpopular_hard_puzzles.sort(reverse=False, key=lambda p: p["Popularity"]*1000 - p["Rating"])
print_puzzle_list(unpopular_hard_puzzles)
# Über den gesamten Datensatz
unpopular_puzzles = find_unpopular_puzzles(lichess_puzzles)
unpopular_puzzles.sort(reverse=False, key=lambda p: p["Popularity"]*1000 - p["Rating"])
print_puzzle_list(unpopular_puzzles)<jupyter_output>ID    Rating  Deviation  NbPlays  Popularity  GameURL
8BkZm   1315         78      416          -52  https://lichess.org/RmcJsyAV/black#96
imMRZ   1400         80      153          -43  https://lichess.org/QcZb2rvs#47
CTLPO   1131         77      216          -43  https://lichess.org/hV6QIRdj/black#44
UQ80X   1240         79      162          -33  https://lichess.org/7hBf5FUo/black#56
gD7mJ    689         88      191          -30  https://lichess.org/P16dgGrN/black#76
IhAMK    598        104      157          -29  https://lichess.org/cWTribHK#27
HLEwA   1336         78      153          -26  https://lichess.org/cdnXFHsK#29
6kWcc   1295         78      155          -26  https://lichess.org/cgU3cZzd#55
lexqf   1238         80      174          -26  https://lichess.org/zkQvqKIW/black#88
zj66O   1429         79      152          -24  https://lichess.org/P4CQ7zWx/black#50
vkL76   1513         77      157          -22  https://lichess.org/5LcOFGMZ#75
xZFk4   1465         77      282         [...]
{'fraction_non_alphanumeric': 0.09507601542999773, 'fraction_numerical': 0.06886771046063081, 'mean_word_length': 2.840958605664488, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 24, 'https://': 33, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '569756', 'license': 'CC0-1.0', 'n_tokens_mistral': 3571, 'n_tokens_neox': 3119, 'n_words': 642, 'path': 'LichessPuzzles.ipynb', 'repo_name': 'Lukasel/chessStuff', 'size': '8814'}