<jupyter_start><jupyter_text>Environment<jupyter_code>import pandas as pd
import numpy as np
import matplotlib as mpl
from matplotlib import pyplot as plt
import json<jupyter_output><empty_output><jupyter_text>Function<jupyter_code>def GetStat(genres, rating_range=(5,6)):
  films_selected = database[(database["genres"].str.contains(genres)) & \
                            (database["rating_average"] >= rating_range[0]) & \
                            (database["rating_average"] <= rating_range[1])]
  
  films_selected["count"] = films_selected["rating_details_1"] + \
                            films_selected["rating_details_2"] + \
                            films_selected["rating_details_3"] + \
                            films_selected["rating_details_4"] + \
                            films_selected["rating_details_5"]
  ratings = films_selected[["rating_details_%d" % i for i in range(1, 6)]].values
  np.nan_to_num(ratings)
  if ratings.sum() == 0:
    return []
  
  weights = ratings.sum(axis=1) + 1e-10
  ratings = ratings / weights[:, None]
  mean_rating = np.average(ratings, axis=0, weights=weights)
  mean_rating_squared = np.average(ratings ** 2, axis=0, weights=weights)
  std_rating = np.sqrt(mean_rating_squared - mean_rating ** 2)
  
  top_best = films_selected.nlargest(n=20, columns="count")

  return {"rating":{"count": weights.sum(),
                    "#films": films_selected.shape[0],
                    "1": mean_rating[0],
                    "1sd": std_rating[0],
                    "2": mean_rating[1],
                    "2sd": std_rating[1],
                    "3": mean_rating[2],
                    "3sd": std_rating[2],
                    "4": mean_rating[3],
                    "4sd": std_rating[3],
                    "5": mean_rating[4],
                    "5sd": std_rating[4],},
          "top_films": [{"film_id": row["film_id"],
                         "title": row["title"],
                         "rating_average": row["rating_average"]} for idx, row in top_best.iterrows()]}
def AllGenres(database):
  all_genres = {"":0}
  for genres_for_film in database.genres.values:
    if type(genres_for_film) is str:
      for genres in genres_for_film.split(":"):
        if genres not in all_genres:
          all_genres[genres] = 0
        all_genres[genres] += 1
        all_genres[""] += 1
  return all_genres<jupyter_output><empty_output><jupyter_text>Dev  Run<jupyter_code>database = pd.read_csv("data/films_rating_info.csv", dtype={"film_id" : object})
all_genres = AllGenres(database)
genres_sorted = sorted(all_genres.keys(), key=lambda x:-all_genres[x])
for x in genres_sorted:
  print(x)
# Film list
with open("data/films.json", "rb") as f:
  film_list_raw = f.read()
film_list = []
for item in film_list_raw.split(b"\n"):
  try:
    json.loads(item)
    film_list.append(json.loads(item))
  except:
    pass

stars = [x["star"] for x in film_list]
stars.sort()
length = len(stars)

mpl.rcParams["font.size"] = 16
plt.figure(figsize=(6, 4))
ax = plt.subplot(111)
ax.hist(stars, bins=99, range=(0, 10), cumulative=True, density=True)
ax.set_xlabel("Rating")
ax.set_ylabel("Percentage")
ax.set_yticks(np.arange(0, 1.1, 0.2))
ax.set_yticklabels(["%d %%" % (i * 100) for i in np.arange(0, 1.1, 0.2)])

rating_percentile = []
for i in range(10):
  rating_percentile.append(stars[int(i / 10 * length)])
rating_percentile.append(stars[-1])

for i in range(9, -1,-1):
  if i >= 5:
    print("前%d%%~前%d%%" % (90 - i * 10, 100 - i * 10), "的电影评分在",
          rating_percentile[i], "至", rating_percentile[i+1], "之间")
  else:
    print("后%d%%~后%d%%" % (i * 10, 10 + i * 10), "的电影评分在",
          rating_percentile[i], "至", rating_percentile[i+1], "之间")
import tqdm
result = {}
for genres in tqdm.tqdm_notebook(all_genres):
  key = genres if genres is not "" else "全部"
  result[key] = {}
  for i in range(10):
    rating_begin = rating_percentile[9-i]
    rating_end = rating_percentile[10-i]
    print(key, rating_begin, rating_end)
    result[key][i] = GetStat(genres, (rating_begin, rating_end))
with open("result/database.js", "w") as f:
  f.write("var database=")
  f.write(json.dumps(result))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15492957746478872, 'fraction_numerical': 0.02458820720935784, 'mean_word_length': 2.9942802669208772, 'pattern_counts': {'":': 19, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '630613', 'license': 'MIT', 'n_tokens_mistral': 1567, 'n_tokens_neox': 1488, 'n_words': 285, 'path': 'stat.ipynb', 'repo_name': 'hilcj/douban_data_stat', 'size': '4189'}