|
import os |
|
from datetime import datetime |
|
import json |
|
import matplotlib.ticker as ticker |
|
from huggingface_hub import snapshot_download |
|
from collections import defaultdict |
|
import pandas as pd |
|
import streamlit as st |
|
from datetime import datetime, timedelta |
|
import matplotlib.pyplot as plt |
|
|
|
plt.rcParams.update({'font.size': 40}) |
|
|
|
libraries = { |
|
"open-source-metrics/transformers-dependents", |
|
"open-source-metrics/diffusers-dependents", |
|
"open-source-metrics/pytorch-image-models-dependents", |
|
"open-source-metrics/datasets-dependents", |
|
"open-source-metrics/gradio-dependents", |
|
"open-source-metrics/accelerate-dependents", |
|
"open-source-metrics/evaluate-dependents", |
|
"open-source-metrics/tokenizers-dependents", |
|
"open-source-metrics/optimum-dependents", |
|
"open-source-metrics/hub-docs-dependents", |
|
"open-source-metrics/huggingface_hub-dependents", |
|
"open-source-metrics/peft-dependents", |
|
} |
|
|
|
MAP = {"-".join(k.split("/")[-1].split("-")[:-1]): k for k in libraries} |
|
|
|
selected_libraries = st.multiselect( |
|
'Choose libraries', |
|
list(MAP.keys()) |
|
) |
|
|
|
def get_frames(option): |
|
cached_folder = snapshot_download(option, repo_type="dataset") |
|
|
|
num_dependents = defaultdict(int) |
|
num_stars_all_dependents = defaultdict(int) |
|
|
|
def load_json_files(directory): |
|
for subdir, dirs, files in os.walk(directory): |
|
for file in files: |
|
if file.endswith('.json'): |
|
file_path = os.path.join(subdir, file) |
|
date = "_".join(file_path.split(".")[-2].split("/")[-3:]) |
|
with open(file_path, 'r') as f: |
|
data = json.load(f) |
|
|
|
if "name" in data and "stars" in data: |
|
num_dependents[date] = len(data["name"]) |
|
num_stars_all_dependents[date] = sum(data["stars"]) |
|
|
|
|
|
load_json_files(cached_folder) |
|
|
|
def sort_dict_by_date(d): |
|
|
|
sorted_tuples = sorted(d.items(), key=lambda x: datetime.strptime(x[0], '%Y_%m_%d')) |
|
|
|
return defaultdict(int, sorted_tuples) |
|
|
|
def remove_incorrect_entries(data): |
|
|
|
sorted_data = sorted(data.items(), key=lambda x: datetime.strptime(x[0], '%Y_%m_%d')) |
|
|
|
|
|
corrected_data = defaultdict(int) |
|
|
|
|
|
previous_dependents = None |
|
|
|
for date, dependents in sorted_data: |
|
|
|
if previous_dependents is None or dependents >= previous_dependents: |
|
corrected_data[date] = dependents |
|
previous_dependents = dependents |
|
|
|
return corrected_data |
|
|
|
def interpolate_missing_dates(data): |
|
|
|
temp_data = {datetime.strptime(date, '%Y_%m_%d'): value for date, value in data.items()} |
|
|
|
|
|
min_date, max_date = min(temp_data.keys()), max(temp_data.keys()) |
|
|
|
|
|
current_date = min_date |
|
while current_date <= max_date: |
|
|
|
if current_date not in temp_data: |
|
|
|
prev_date = current_date - timedelta(days=1) |
|
next_date = current_date + timedelta(days=1) |
|
while prev_date not in temp_data: |
|
prev_date -= timedelta(days=1) |
|
while next_date not in temp_data: |
|
next_date += timedelta(days=1) |
|
|
|
|
|
prev_value = temp_data[prev_date] |
|
next_value = temp_data[next_date] |
|
interpolated_value = prev_value + ((next_value - prev_value) * ((current_date - prev_date) / (next_date - prev_date))) |
|
temp_data[current_date] = interpolated_value |
|
|
|
current_date += timedelta(days=1) |
|
|
|
|
|
interpolated_data = defaultdict(int, {date.strftime('%Y_%m_%d'): int(value) for date, value in temp_data.items()}) |
|
|
|
return interpolated_data |
|
|
|
num_dependents = remove_incorrect_entries(num_dependents) |
|
num_stars_all_dependents = remove_incorrect_entries(num_stars_all_dependents) |
|
|
|
num_dependents = interpolate_missing_dates(num_dependents) |
|
num_stars_all_dependents = interpolate_missing_dates(num_stars_all_dependents) |
|
|
|
num_dependents = sort_dict_by_date(num_dependents) |
|
num_stars_all_dependents = sort_dict_by_date(num_stars_all_dependents) |
|
|
|
num_dependents_df = pd.DataFrame(list(num_dependents.items()), columns=['Date', 'Value']) |
|
num_cum_stars_df = pd.DataFrame(list(num_stars_all_dependents.items()), columns=['Date', 'Value']) |
|
|
|
num_dependents_df['Date'] = pd.to_datetime(num_dependents_df['Date'], format='%Y_%m_%d') |
|
num_cum_stars_df['Date'] = pd.to_datetime(num_cum_stars_df['Date'], format='%Y_%m_%d') |
|
|
|
num_dependents_df.set_index('Date', inplace=True) |
|
num_dependents_df = num_dependents_df.resample('D').asfreq() |
|
num_dependents_df['Value'] = num_dependents_df['Value'].interpolate() |
|
|
|
num_cum_stars_df.set_index('Date', inplace=True) |
|
num_cum_stars_df = num_cum_stars_df.resample('D').asfreq() |
|
num_cum_stars_df['Value'] = num_cum_stars_df['Value'].interpolate() |
|
|
|
return num_dependents_df, num_cum_stars_df |
|
|
|
|
|
lib_frames = {l: get_frames(MAP[l]) for l in selected_libraries} |
|
|
|
plt.figure(figsize=(40, 24)) |
|
plt.gca().yaxis.set_major_formatter(ticker.StrMethodFormatter('{x:,.0f}')) |
|
|
|
for l, (df_dep, _) in lib_frames.items(): |
|
plt.plot(df_dep.index, df_dep['Value'], label=l, marker='o') |
|
|
|
plt.xlabel('Date') |
|
plt.ylabel('# Dependencies') |
|
plt.legend() |
|
plt.title('Dependencies History') |
|
st.pyplot(plt) |
|
|
|
|
|
plt.figure(figsize=(40, 24)) |
|
plt.gca().yaxis.set_major_formatter(ticker.StrMethodFormatter('{x:,.0f}')) |
|
|
|
for l, (_, df_stars) in lib_frames.items(): |
|
plt.plot(df_stars.index, df_stars['Value'], label=l, marker='o') |
|
|
|
plt.xlabel('Date') |
|
plt.ylabel('SUM stars of dependencies') |
|
plt.legend() |
|
plt.title('Dependents Stars History') |
|
st.pyplot(plt) |
|
|