updating layout, adding some additional dedupe analysis columns, adding new area chart

app.py

@@ -40,11 +40,6 @@ def process_dataset():
     # drop any nas
     file_extensions_by_month = file_extensions_by_month.dropna()
 
-    # Convert the total size to petabytes and format to two decimal places
-    file_counts_and_sizes = format_dataframe_size_column(
-        file_counts_and_sizes, "total_size"
-    )
-
     file_counts_and_sizes["type"] = file_counts_and_sizes["type"].str.capitalize()
     # update the column name to 'total size (PB)'
     file_counts_and_sizes = file_counts_and_sizes.rename(
@@ -54,6 +49,8 @@ def process_dataset():
             "total_size": "Total Size (PBs)",
         }
     )
+    file_counts_and_sizes = file_counts_and_sizes.drop(columns=["Number of Files"])
+
     # sort the dataframe by total size in descending order
     file_counts_and_sizes = file_counts_and_sizes.sort_values(
         by="Total Size (PBs)", ascending=False
@@ -80,7 +77,7 @@ def format_dataframe_size_column(_df, column_name):
     return _df
 
 
-def cumulative_growth_plot_analysis(df, df_compressed):
+def cumulative_growth_plot_analysis(df, df_compressed, repo_sizes):
     """
     Calculates the cumulative growth of models, spaces, and datasets over time and generates a plot and dataframe from the analysis.
 
@@ -121,7 +118,16 @@ def cumulative_growth_plot_analysis(df, df_compressed):
     last_10_months = cumulative_df.tail(10).copy()
     last_10_months["total"] = last_10_months.sum(axis=1)
     last_10_months["total_change"] = last_10_months["total"].diff()
+    last_10_months["compressed_change"] = (
+        cumulative_df_compressed.tail(10).sum(axis=1).diff()
+    )
+    last_10_months["savings"] = (
+        last_10_months["total_change"] - last_10_months["compressed_change"]
+    )
+    last_10_months = format_dataframe_size_column(last_10_months, "savings")
+    last_10_months = format_dataframe_size_column(last_10_months, "compressed_change")
     last_10_months = format_dataframe_size_column(last_10_months, "total_change")
+
     last_10_months["date"] = cumulative_df.tail(10).index
     # drop the dataset, model, and space
     last_10_months = last_10_months.drop(columns=["model", "space", "dataset"])
@@ -130,10 +136,17 @@ def cumulative_growth_plot_analysis(df, df_compressed):
     # drop the first row
     last_10_months = last_10_months.drop(last_10_months.index[0])
     # order the columns date, total, total_change
-    last_10_months = last_10_months[["date", "total_change"]]
+    last_10_months = last_10_months[
+        ["date", "total_change", "compressed_change", "savings"]
+    ]
     # rename the columns
     last_10_months = last_10_months.rename(
-        columns={"date": "Date", "total_change": "Month-to-Month Growth (PBs)"}
+        columns={
+            "date": "Date",
+            "total_change": "Month-to-Month Growth (PBs)",
+            "compressed_change": "Growth with File-Level Deduplication (PBs)",
+            "savings": "Dedupe Savings (PBs)",
+        }
     )
 
     # Create a Plotly figure
@@ -146,6 +159,22 @@ def cumulative_growth_plot_analysis(df, df_compressed):
         "dataset": px.colors.qualitative.Alphabet[9],
     }
 
+    # create a new column in the repository sizes dataframe for "compressed size" and set it to empty atif rist
+    repo_sizes["Compressed Size (PBs)"] = ""
+    repo_sizes["Dedupe Savings (PBs)"] = ""
+
+    for column in cumulative_df.columns:
+        cum_repo_size = cumulative_df[column].iloc[-1]
+        comp_repo_size = cumulative_df_compressed[column].iloc[-1]
+        repo_size_diff = cum_repo_size - comp_repo_size
+        repo_sizes.loc[
+            repo_sizes["Repository Type"] == column.capitalize(),
+            "Compressed Size (PBs)",
+        ] = comp_repo_size
+        repo_sizes.loc[
+            repo_sizes["Repository Type"] == column.capitalize(), "Dedupe Savings (PBs)"
+        ] = repo_size_diff
+
     # Add a scatter trace for each type
     for column in cumulative_df.columns:
         fig.add_trace(
@@ -165,7 +194,7 @@ def cumulative_growth_plot_analysis(df, df_compressed):
                 x=cumulative_df_compressed.index,
                 y=cumulative_df_compressed[column] / 1e15,  # Convert to petabytes
                 mode="lines",
-                name=column.capitalize() + " (Compressed)",
+                name=column.capitalize() + " (File-Level Deduplication)",
                 line=dict(color=color_map.get(column, "black"), dash="dash"),
             )
         )
@@ -183,7 +212,7 @@ def cumulative_growth_plot_analysis(df, df_compressed):
 
 def plot_total_sum(by_type_arr):
     # Sort the array by size in decreasing order
-    by_type_arr = sorted(by_type_arr, key=lambda x: x[1], reverse=True)
+    by_type_arr = sorted(by_type_arr, key=lambda x: x[1])
 
     # Create a Plotly figure
     fig = go.Figure()
@@ -221,7 +250,9 @@ def filter_by_extension_month(_df, _extension):
     fig (Figure): The Plotly figure object representing the line plot.
     """
     # Filter the DataFrame by the specified extension or extensions
-    if len(_extension) == 1 and "All" in _extension or len(_extension) == 0:
+    if _extension is None:
+        pass
+    elif len(_extension) == 0:
         pass
     else:
         _df = _df[_df["extension"].isin(_extension)].copy()
@@ -263,10 +294,13 @@ def filter_by_extension_month(_df, _extension):
 
 # Create a gradio blocks interface and launch a demo
 with gr.Blocks() as demo:
-    df, file_df, by_type, by_extension, by_extension_month = process_dataset()
+    df, file_df, by_repo_type, by_extension, by_extension_month = process_dataset()
 
-    # Add a heading
-    gr.Markdown("# Git LFS Analysis Across the Hub")
+    # get the figure for the cumulative growth plot and the last 10 months dataframe
+    fig, last_10_months = cumulative_growth_plot_analysis(df, file_df, by_repo_type)
+
+    # Add top level heading and introduction text
+    gr.Markdown("# Git LFS Usage Across the Hub")
     gr.Markdown(
         "The Hugging Face Hub has just crossed 1,000,000 models - but where is all that data stored? The short answer is Git LFS. This analysis dives into the LFS storage on the Hub, breaking down the data by repository type, file extension, and growth over time."
     )
@@ -274,28 +308,39 @@ with gr.Blocks() as demo:
     gr.Markdown(
         "Now, you might ask yourself, 'Why are you doing this?' Well, the [Xet Team](https://huggingface.co/xet-team) is a [new addition to Hugging Face](https://huggingface.co/blog/xethub-joins-hf), bringing a new way to store massive datasets and models to enable ML teams to operate like software teams: Quickly and without friction. Because this story all starts with storage, that's where we've begun with our own deep dives into what the Hub holds. As part of this, we've included a look at what happens with just one simple deduplication strategy - deduplicating at the file level. Read on to see more!"
     )
+    gr.HTML("<div style='height: 20px;'></div>")
+    # Cumulative growth analysis
+    gr.Markdown("## Repository Growth")
+    with gr.Row():
+        gr.Plot(fig)
     with gr.Row():
-        # scale so that
-        # group the data by month and year and compute a cumulative sum of the total_size column
-        fig, last_10_months = cumulative_growth_plot_analysis(df, file_df)
         with gr.Column(scale=1):
-            gr.Markdown("# Repository Growth")
             gr.Markdown(
-                "The cumulative growth of models, spaces, and datasets over time can be seen in the adjacent chart. Beside that is a view of the total change, from the previous month to the current one, of LFS files stored on the hub over 2024. We're averaging nearly **2.3 PBs uploaded to LFS per month!**"
+                "This table shows the total number of files and cumulative size of those files across all repositories on the Hub. These numbers might be hard to grok, so let's try to put them in context. The last [Common Crawl](https://commoncrawl.org/) download was [451 TBs](https://github.com/commoncrawl/cc-crawl-statistics/blob/master/stats/crawler/CC-MAIN-2024-38.json#L31). The Spaces repositories alone outpaces that. Meanwhile, between Datasets and Model repos, the Hub stores **64 Common Crawls** 🤯."
             )
-            gr.Dataframe(last_10_months, height=250)
         with gr.Column(scale=3):
-            gr.Plot(fig)
+            # Convert the total size to petabytes and format to two decimal places
+            by_repo_type = format_dataframe_size_column(
+                by_repo_type, "Total Size (PBs)"
+            )
+            by_repo_type = format_dataframe_size_column(
+                by_repo_type, "Compressed Size (PBs)"
+            )
+            by_repo_type = format_dataframe_size_column(
+                by_repo_type, "Dedupe Savings (PBs)"
+            )
+            gr.Dataframe(by_repo_type)
     with gr.Row():
         with gr.Column(scale=1):
             gr.Markdown(
-                "This table shows the total number of files and cumulative size of those files across all repositories on the Hub. These numbers might be hard to grok, so let's try to put them in context. The last [Common Crawl](https://commoncrawl.org/) download was [451 TBs](https://github.com/commoncrawl/cc-crawl-statistics/blob/master/stats/crawler/CC-MAIN-2024-38.json#L31). The Spaces repositories alone outpaces that. Meanwhile, between Datasets and Model repos, the Hub stores **64 Common Crawls** 🤯."
+                "The cumulative growth of models, spaces, and datasets over time can be seen in the adjacent chart. Beside that is a view of the total change, from the previous month to the current one, of LFS files stored on the hub over 2024. We're averaging nearly **2.3 PBs uploaded to LFS per month!**"
             )
         with gr.Column(scale=3):
-            gr.Dataframe(by_type)
+            gr.Dataframe(last_10_months, height=250)
 
-    # Add a heading
-    gr.Markdown("## File Extension Analysis")
+    gr.HTML("<div style='height: 20px;'></div>")
+    # File Extension analysis
+    gr.Markdown("## File Extensions on the Hub")
     gr.Markdown(
         "Breaking this down by file extension, some interesting trends emerge. [Safetensors](https://huggingface.co/docs/safetensors/en/index) are quickly becoming the defacto standard on the hub, accounting for over 7PBs (25%) of LFS storage. The top 20 file extensions seen here and in the table below account for 82% of all LFS storage on the hub."
     )
@@ -330,7 +375,14 @@ with gr.Blocks() as demo:
     )
     gr.Dataframe(by_extension_size)
 
+    gr.HTML("<div style='height: 20px;'></div>")
     gr.Markdown("## File Extension Growth Over Time")
+    by_extension_month["date"] = pd.to_datetime(
+        by_extension_month[["year", "month"]].assign(day=1)
+    )
+    # make a plotly area chart with data and extension
+    figure = px.area(by_extension_month, x="date", y="total_size", color="extension")
+    gr.Plot(figure)
     gr.Markdown(
         "Want to dig a little deeper? Select a file extension to see how many bytes of that type were uploaded to the Hub each month."
     )
@@ -338,8 +390,6 @@ with gr.Blocks() as demo:
     # build a dropdown using the unique values in the extension column
     extension = gr.Dropdown(
         choices=by_extension["extension"].unique().tolist(),
-        value="All",
-        allow_custom_value=True,
         multiselect=True,
     )
     _by_extension_month = gr.State(by_extension_month)

pyproject.toml

@@ -5,6 +5,7 @@ description = ""
 authors = ["jsulz <[email protected]>"]
 license = "MIT"
 readme = "README.md"
+package-mode = false
 
 [tool.poetry.dependencies]
 python = "^3.12"