<jupyter_start><jupyter_text>Import DB<jupyter_code>import pandas as pd
import sqlite3

# Create your connection.
cnx = sqlite3.connect('test.db')
from aoc import timeit
with timeit(): df = pd.read_sql_query("SELECT * FROM reviews", cnx)
df.to_csv('17mreviews.csv')
df.rename(columns={'user':'ID','rating':'user','comment':'rating','ID':'comment'},inplace=True)
df = df[['user', 'rating', 'comment','ID']]
df['ID'] = df['ID'].astype('int64')
df = df.merge(games[['ID','Name']],on='ID',how='left')
df.head()
len(df)
df.drop_duplicates(subset=None, keep='first', inplace=True, ignore_index=True)
len(df)
df.drop_duplicates(subset=['user','ID'], keep='first', inplace=True, ignore_index=True)
len(df)
df['user'].nunique()
filename = 'bgg-20m-reviews.csv'
df.rename(columns={'Name':'name'},inplace=True)
df.to_csv(filename)

import pandas as pd
import sqlite3
import aoc
filename = 'bgg-20m-reviews.csv'
with aoc.timeit(): df = pd.read_csv(filename,index_col=0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20567375886524822, 'fraction_numerical': 0.0121580547112462, 'mean_word_length': 8.97979797979798, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '500688', 'license': 'MIT', 'n_tokens_mistral': 389, 'n_tokens_neox': 376, 'n_words': 64, 'path': 'bgg_ratings_get_data_through_API.ipynb', 'repo_name': 'jvanelteren/bgg-similarity', 'size': '987'}