merging ann-changes

app.py

@@ -211,6 +211,53 @@ def cumulative_growth_plot_analysis(cumulative_df, cumulative_df_compressed):
     return fig
 
 
+def cumulative_growth_single(_df):
+    """
+    Calculates the cumulative growth of models, spaces, and datasets over time and generates a plot and dataframe from the analysis.
+
+    Args:
+        df (DataFrame): The input dataframe containing the data.
+
+    Returns:
+         - fig (Figure): The Plotly figure showing the cumulative growth of models, spaces, and datasets over time.
+
+    Raises:
+        None
+    """
+
+    # Create a Plotly figure
+    fig = go.Figure()
+
+    # Define a color map for each type
+    color_map = {
+        "model": px.colors.qualitative.Alphabet[3],
+        "space": px.colors.qualitative.Alphabet[2],
+        "dataset": px.colors.qualitative.Alphabet[9],
+    }
+
+    # Add a scatter trace for each type
+    for column in _df.columns:
+        fig.add_trace(
+            go.Scatter(
+                x=_df.index,
+                y=_df[column] / 1e15,  # Convert to petabytes
+                mode="lines",
+                name=column.capitalize(),
+                line=dict(color=color_map.get(column, "black")),  # Use color map
+            )
+        )
+
+    # Update layout
+    fig.update_layout(
+        title="Cumulative Growth of Models, Spaces, and Datasets",
+        xaxis_title="Date",
+        yaxis_title="Size (PBs)",
+        legend_title="Type",
+        yaxis=dict(tickformat=".2f"),  # Format y-axis labels to 2 decimal places
+    )
+    return fig
+
+
 def plot_total_sum(by_type_arr):
     # Sort the array by size in decreasing order
     by_type_arr = sorted(by_type_arr, key=lambda x: x[1])
@@ -274,7 +321,7 @@ def filter_by_extension_month(_df, _extension):
             fig.add_trace(
                 go.Scatter(
                     x=pivot_df.index,
-                    y=pivot_df[column] / 1e12,  # Convert to TBs
+                    y=pivot_df[column] * 1e3,
                     mode="lines",
                     name=column,
                     line=dict(color=px.colors.qualitative.Alphabet[i]),
@@ -351,71 +398,56 @@ with gr.Blocks() as demo:
 
     last_10_months = compare_last_10_months(cumulative_df, cumulative_df_compressed)
 
-    by_repo_type = tabular_analysis(
+    by_repo_type_analysis = tabular_analysis(
         by_repo_type, cumulative_df, cumulative_df_compressed
     )
 
-    # get the figure for the cumulative growth plot and the last 10 months dataframe
-    fig = cumulative_growth_plot_analysis(cumulative_df, cumulative_df_compressed)
-
     # Add top level heading and introduction text
-    gr.Markdown("# Git LFS Usage Across the Hub")
+    gr.Markdown("# Git LFS Usage across the Hub")
     gr.Markdown(
-        "The Hugging Face Hub has just crossed 1,000,000 models - but where is all that data stored? Most of it is stored in Git LFS. This analysis dives into the LFS storage on the Hub, breaking down the data by repository type, file extension, and growth over time. The data is based on a snapshot of the Hub's LFS storage, starting in March 2022 and ending September 20th, 2024 (meaning the data is incomplete for September 2024). Right now, this is a one-time analysis, but as we do our work we hope to revisit and update the underlying data to provide more insights."
+        "Ever wonder what the Hugging Face Hub holds? This is the space for you!"
     )
-
     gr.Markdown(
-        "Now, you might ask yourself, 'Why are you doing this?' Well, the [Xet Team](https://huggingface.co/xet-team) is a [new addition to Hugging Face](https://huggingface.co/blog/xethub-joins-hf), bringing a new way to store massive datasets and models to enable ML teams to operate like software teams: Quickly and without friction. Because this story all starts with storage, that's where we've begun with our own deep dives into what the Hub holds. As part of this, we've included a look at what happens with just one simple deduplication strategy - deduplicating at the file level. Read on to see more!"
+        "The Hub stores all files using a combination of [Gitaly](https://gitlab.com/gitlab-org/gitaly) (small files) on EBS and [Git LFS](https://git-lfs.com/) (large files > 10MB) on S3. As part of the [Xet team](https://huggingface.co/xet-team), one of our goals is to improve Hub storage and transfer efficiency, and understanding how and what things are currently stored helps us establish a baseline. This analysis uses a snapshot of the Hub's Git LFS usage from March 2022 - September 2024, and we plan to update it regularly to track trends. We're starting with metrics around raw storage by repository type and size/count by file extension - if you're interested in other metrics, drop your suggestions in our [discussions](https://huggingface.co/spaces/xet-team/lfs-analysis/discussions)!"
     )
+
     gr.HTML(div_px(25))
     # Cumulative growth analysis
-    gr.Markdown("## Repository Growth")
+    gr.Markdown("## Storage by Repository Type")
     gr.Markdown(
-        "The plot below shows the growth of Git LFS storage on the Hub over the past two years. The solid lines represent the cumulative growth of models, spaces, and datasets, while the dashed lines represent the growth with file-level deduplication."
+        "The chart below shows the growth of Git LFS storage usage by repository type since March 2022."
     )
-    gr.Plot(fig)
+
+    # get the figure for the cumulative growth plot without dedupe analysis
+    cumulative_fig = cumulative_growth_single(cumulative_df)
+    gr.Plot(cumulative_fig)
 
     gr.HTML(div_px(5))
     # @TODO Talk to Allison about variant="panel"
     with gr.Row():
-        with gr.Column(scale=1):
+        with gr.Column(scale=2):
+            gr.Markdown("### Current Storage Usage")
             gr.Markdown(
-                "In this table, we can see what the final picture looks like as of September 20th, 2024, along with the potential file-level deduplication savings."
-            )
-            gr.Markdown(
-                "To put this in context, the last [Common Crawl](https://commoncrawl.org/) download was [451 TBs](https://github.com/commoncrawl/cc-crawl-statistics/blob/master/stats/crawler/CC-MAIN-2024-38.json#L31). The Spaces repositories alone outpaces that! Meanwhile, between Datasets and Model repos, the Hub stores **64 Common Crawls** 🤯. Current estimates put file deduplication savings at approximately 3.24 PBs (7.2 Common Crawls)!"
+                "As of September 20, 2024, total files stored in Git LFS summed to almost 29 PB. To put this into perspective, the last [Common Crawl](https://commoncrawl.org/) download was [451 TBs](https://github.com/commoncrawl/cc-crawl-statistics/blob/master/stats/crawler/CC-MAIN-2024-38.json#L31) - the Hub stores the equivalent of more than **64 Common Crawls** 🤯."
             )
         with gr.Column(scale=3):
             # Convert the total size to petabytes and format to two decimal places
-            by_repo_type = format_dataframe_size_column(
-                by_repo_type,
+            current_storage = format_dataframe_size_column(
+                by_repo_type_analysis,
                 ["Total Size (PBs)", "Compressed Size (PBs)", "Dedupe Savings (PBs)"],
             )
-            gr.Dataframe(by_repo_type)
-
-    gr.HTML(div_px(5))
-    with gr.Row():
-        with gr.Column(scale=1):
-            gr.Markdown(
-                "The month-to-month growth of models, spaces, can be seen in the adjacent table. In 2024, the Hub has averaged nearly **2.3 PBs uploaded to LFS per month!** By the same token, the monthly file deduplication savings are nearly 225TBs. "
-            )
-
-            gr.Markdown(
-                "Borrowing from the Common Crawl analogy, that's about *5 crawls* uploaded every month, with an _easy savings of half a crawl every month_ by deduplicating at the file-level!"
-            )
-        with gr.Column(scale=3):
-            gr.Dataframe(last_10_months)
+            gr.Dataframe(current_storage[["Repository Type", "Total Size (PBs)"]])
 
     gr.HTML(div_px(25))
     # File Extension analysis
-    gr.Markdown("## File Extensions on the Hub")
+    gr.Markdown("## Large Files Stored by File Extension")
     gr.Markdown(
-        "Breaking this down by file extension, some interesting trends emerge. The following sections filter the analysis to the top 20 file extensions stored (in bytes) using LFS (which accounts for 82% of storage consumption)."
+        "What types of files are stored on the Hub? The Xet team's backend architecture allows for storage optimizations by file type, so seeing the breakdown of the most popular stored file types helps to prioritize our roadmap. The following sections filter the analysis to the top 20 file extensions stored (by bytes) using Git LFS. Taken together, these 20 file extensions account for 82% of the total bytes stored in LFS."
     )
     gr.Markdown(
-        "As is evident in the chart below, [Safetensors](https://huggingface.co/docs/safetensors/en/index) is quickly becoming the defacto standard on the Hub for storing tensor files, accounting for over 7PBs (25%) of LFS storage. If you want to know why you'd want to check out YAF (yet another format), this explanation from the [Safetensors docs](https://github.com/huggingface/safetensors?tab=readme-ov-file#yet-another-format-) is a good place to start. Speaking of YAF, [GGUF (GPT-Generated Unified Format)](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md) is also on the rise, accounting for 3.2 PBs (11%) of LFS storage. GGUF, like Safetensors, is a format for storing tensor files, with a different set of optimizations. The Hub has a few [built-in tools](https://huggingface.co/docs/hub/en/gguf) for working with GGUF."
+        "[Safetensors](https://huggingface.co/docs/safetensors/en/index) is quickly becoming the defacto standard on the Hub for storing tensor files, accounting for over 7PBs (25%) of LFS storage. [GGUF (GPT-Generated Unified Format)](https://huggingface.co/docs/hub/gguf), a format for storing tensor files with a different set of optimizations, is also on the rise, accounting for 3.2 PBs (11%) of LFS storage."
     )
-    # Get the top 10 file extnesions by size
+    # Get the top 10 file extensions by size
     by_extension_size = by_extension.sort_values(by="size", ascending=False).head(22)
 
     # make a bar chart of the by_extension_size dataframe
@@ -445,20 +477,29 @@ with gr.Blocks() as demo:
 
     gr.HTML(div_px(5))
     gr.Markdown(
-        "Below, we have a more detailed tabular view of the same top 20 file extensions by total size, number of files, and average file size."
+        "This tabular view shows the same top 20 file extensions by total stored size, number of files, and average file size."
+    )
+    gr.Dataframe(
+        by_extension_size[
+            [
+                "File Extension",
+                "Total Size (PBs)",
+                "Number of Files",
+                "Average File Size (MBs)",
+            ]
+        ]
     )
-    gr.Dataframe(by_extension_size)
 
     gr.HTML(div_px(5))
-    gr.Markdown("### File Extension Monthly Additions (in PBs)")
+    gr.Markdown("### Storage Growth by File Extension (Monthly PBs Added)")
     gr.Markdown(
-        "What if we want to see trends over time? The following area chart shows the number of bytes added to LFS storage each month, faceted by the most popular file extensions."
+        "The following area chart shows the number of bytes added to LFS storage each month, faceted by file extension."
     )
     gr.Plot(area_plot_by_extension_month(by_extension_month))
 
     gr.HTML(div_px(5))
     gr.Markdown(
-        "To dig a little deeper, the following dropdown allows you to filter the area chart by file extension. Because we're dealing with individual file extensions, the data is presented in terabytes (TBs)."
+        "To dig deeper, use the dropdown to filter by file extension and see the bytes added (in TBs) each month for specific file types."
     )
 
     # build a dropdown using the unique values in the extension column
@@ -470,5 +511,39 @@ with gr.Blocks() as demo:
     _by_extension_month = gr.State(by_extension_month)
     gr.Plot(filter_by_extension_month, inputs=[_by_extension_month, extension])
 
+    gr.HTML(div_px(25))
+    # Optimizations
+
+    gr.Markdown("## Optimization 1: File-level Deduplication")
+    gr.Markdown(
+        "The first improvement we can make to Hub storage is to add file-level deduplication. Since forking any Hub repository makes copies of the files, a scan of existing files unsurprisingly shows that some files match exactly. The following chart shows the storage growth chart from above with additional dashed lines showing the potential savings from deduplicating at the file level."
+    )
+    dedupe_fig = cumulative_growth_plot_analysis(
+        cumulative_df, cumulative_df_compressed
+    )
+    gr.Plot(dedupe_fig)
+
+    gr.HTML(div_px(5))
+    # @TODO Talk to Allison about variant="panel"
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown("### Current Storage Usage + File-level Deduplication")
+            gr.Markdown(
+                "This simple change to the storage backend will save 3.24 PBs (the equivalent of 7.2 Common Crawls)."
+            )
+        with gr.Column(scale=3):
+            # Convert the total size to petabytes and format to two decimal places
+            gr.Dataframe(by_repo_type)
+
+    gr.HTML(div_px(5))
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown("### Month-to-Month Growth + File-level Deduplication")
+            gr.Markdown(
+                "This table shows month-to-month growth in model, dataset, and space storage. In 2024, the Hub has averaged nearly **2.3 PBs uploaded to Git LFS per month**. Deduplicating at the file level saves nearly 225 TB (half a Common Crawl) monthly."
+            )
+        with gr.Column(scale=3):
+            gr.Dataframe(last_10_months)
+
 # launch the dang thing
 demo.launch()