Spaces:

HuggingFaceFW
/

blogpost-fineweb-v1

Running

File size: 53,077 Bytes

43ec909
 
 
01fd83e
43ec909

<!doctype html>

<head>
    <link rel="stylesheet" href="style.css">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <meta charset="utf8">
    <base target="_blank">
    <title>FineWeb: decanting the web for the finest text data at scale</title>
    <style>

        /* ****************************************
         * TOC
         ******************************************/
        @media (max-width: 1199px) {
            d-contents {
                display: none;
                justify-self: start;
                align-self: start;
                padding-bottom: 0.5em;
                margin-bottom: 1em;
                padding-left: 0.25em;
                border-bottom: 1px solid rgba(0, 0, 0, 0.1);
                border-bottom-width: 1px;
                border-bottom-style: solid;
                border-bottom-color: rgba(0, 0, 0, 0.1);
            }
        }

        d-contents a:hover {
            border-bottom: none;
        }


        @media (min-width: 1200px) {
            d-article {
                /* Ensure d-article does not prevent sticky positioning */
                overflow: visible;
            }

            d-contents {
                align-self: start;
                grid-column-start: 1 !important;
                grid-column-end: 4 !important;
                grid-row: auto / span 6;
                justify-self: end;
                margin-top: 0em;
                padding-right: 3em;
                padding-left: 2em;
                border-right: 1px solid rgba(0, 0, 0, 0.1);
                border-right-width: 1px;
                border-right-style: solid;
                border-right-color: rgba(0, 0, 0, 0.1);
                position: -webkit-sticky; /* For Safari */
                position: sticky;
                top: 10px; /* Adjust this value if needed */
            }
        }

        d-contents nav h3 {
            margin-top: 0;
            margin-bottom: 1em;
        }

        d-contents nav div {
            color: rgba(0, 0, 0, 0.8);
            font-weight: bold;
        }

        d-contents nav a {
            color: rgba(0, 0, 0, 0.8);
            border-bottom: none;
            text-decoration: none;
        }

        d-contents li {
            list-style-type: none;
        }

        d-contents ul, d-article d-contents ul {
            padding-left: 1em;
        }

        d-contents nav ul li {
            margin-bottom: .25em;
        }

        d-contents nav a:hover {
            text-decoration: underline solid rgba(0, 0, 0, 0.6);
        }

        d-contents nav ul {
            margin-top: 0;
            margin-bottom: 6px;
        }


        d-contents nav > div {
            display: block;
            outline: none;
            margin-bottom: 0.5em;
        }

        d-contents nav > div > a {
            font-size: 13px;
            font-weight: 600;
        }

        d-contents nav > div > a:hover,
        d-contents nav > ul > li > a:hover {
            text-decoration: none;
        }

    </style>
</head>

<body>
<d-front-matter>
    <script id='distill-front-matter' type="text/json">{
    "title": "🍷 FineWeb: decanting the web for the finest text data at scale",
    "description": "This blog covers a discussion on processing and evaluating data quality at scale, the 🍷 FineWeb recipe (listing and explaining all of our design choices), and the process followed to create 📚 FineWeb-Edu.",
    "published": "May 28, 2024",
    "affiliation": {"name": "HuggingFace"},
    "authors": [
      {
        "author":"Guilherme Penedo",
        "authorURL":"https://huggingface.co/guipenedo"
      },
      {
        "author":"Hynek Kydlíček",
        "authorURL":"https://huggingface.co/hynky"
      },
      {
        "author":"Loubna Ben Allal",
        "authorURL":"https://huggingface.co/loubnabnl"
      },
      {
        "author":"Anton Lozhkov",
        "authorURL":"https://huggingface.co/anton-l"
      },
      {
        "author":"Colin Raffel",
        "authorURL":"https://huggingface.co/craffel"
      },
      {
        "author":"Leandro Werra",
        "authorURL":"https://huggingface.co/lvwerra"
      },
      {
        "author":"Thomas Wolf",
        "authorURL":"https://huggingface.co/thomwolf"
      }
    ],
    "katex": {
      "delimiters": [
        {"left": "$$", "right": "$$", "display": false}
      ]
    }
  }
    </script>
</d-front-matter>
<d-title>
    <h1 class="l-page" style="text-align: center;">🍷 FineWeb: decanting the web for the finest text data at scale</h1>
    <div id="title-plot" class="main-plot-container l-screen">
        <figure>
            <img src="assets/images/banner.png" alt="FineWeb">
        </figure>
        <div id="clusters-plot">
            <img src="assets/images/clusters.png" alt="Clusters">
        </div>
    </div>
</d-title>
<d-byline></d-byline>
<d-article>
    <d-contents>
    </d-contents>

    <p>We have recently released <a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb"><strong>🍷 FineWeb</strong></a>, our new large scale
        (<strong>15T gpt2 tokens, 44TB disk space</strong>) dataset of clean text sourced from the web for LLM pretraining. You can
        download it <a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb">here</a>.</p>
    <p>The performance of a large language model (LLM) depends heavily on the quality and size of its pretraining dataset. However, the pretraining datasets for state-of-the-art open LLMs like Llama 3<d-cite bibtex-key="llama3modelcard"></d-cite> and Mixtral<d-cite bibtex-key="jiang2024mixtral"></d-cite> are not publicly available and very little is known about how they were created.</p>
    <p>🍷 FineWeb, a 15-trillion token dataset derived from 96 <a href="https://commoncrawl.org/">CommonCrawl</a> snapshots, produces better-performing LLMs than other open pretraining datasets. To advance the understanding of how best to curate high-quality pretraining datasets, we carefully document and ablate all of the design choices used in FineWeb, including in-depth investigations of deduplication and filtering strategies.</p>
    <p>We are also excited to announce the release of <a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu"><strong>📚 FineWeb-Edu</strong></a>, a version of 🍷 FineWeb that was filtered for educational content, available in two sizes: <strong>1.3 trillion (very high quality) and 5.4 trillion (high quality) tokens</strong>. 📚 FineWeb-Edu outperforms all existing public web datasets, with models pretrained on it showing notable improvements on knowledge- and reasoning-intensive benchmarks like MMLU, ARC, and OpenBookQA. You can
        download it <a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu">here</a>.</p>
    <p>Both datasets are released under the permissive <strong><a href="https://opendatacommons.org/licenses/by/1-0/">ODC-By 1.0 license</a></strong></p>

    <p>As 🍷 FineWeb has gathered a lot of interest from the
        community, we decided to further explain the steps involved in creating it, our processing decisions and
        some lessons learned along the way. Read on for all the juicy details on large text dataset creation!</p>
    <p><strong>TLDR:</strong> This blog covers a discussion on processing and evaluating data quality at scale, the 🍷 FineWeb
        recipe (listing and explaining all of our design choices), and the process followed to create 📚 FineWeb-Edu.</p>

    <h2>General considerations on web data</h2>
    <h3>Sourcing the data</h3>
    <p>A common question we see asked regarding web datasets used
        to train LLMs is “where do they even get all that data?” There are generally two options:</p>
    <ul>
        <li>you either crawl it yourself, like <a
                href="https://platform.openai.com/docs/gptbot">OpenAI</a> or <a
                href="https://darkvisitors.com/agents/claudebot">Anthropic</a> seem to do
        </li>
    </ul>
    <ul>
        <li>you use a public repository of crawled webpages, like the one maintained by
            the non-profit <a href="https://commoncrawl.org/">CommonCrawl</a></li>
    </ul>
    <p>For 🍷 FineWeb, similarly to what was done for a large number
        of other public datasets, we used <a href="https://commoncrawl.org/">CommonCrawl</a> as a starting point.
        They have been crawling the web since 2007 (long before LLMs became widespread) and release a new dump usually
        every 1 or 2 months, which can be freely downloaded. </p>
    <p>As an example, their latest crawl (2024-18) contains 2.7
        billion web pages, totaling 386 TiB of uncompressed HTML text content (the size changes from dump to dump). There
        are 96 dumps since 2013 and 3 dumps from 2008 to 2012, which are in a different (older) format.<d-footnote>We have not processed these 3 older dumps.</d-footnote> </p>
    <h3>Processing at scale</h3>
    <p>Given the sheer size of the data involved, one of the main
        challenges we had to overcome was having a modular, scalable codebase that would allow us to quickly iterate
        on our processing decisions and easily try out new ideas, while appropriately parallelizing our workloads
        and providing clear insights into the data. </p>
    <p>For this purpose, we developed <a
            href="https://github.com/huggingface/datatrove"><code>datatrove</code></a><d-cite bibtex-key="penedo2024datatrove"></d-cite>, an open-source data
        processing library that allowed us to seamlessly scale our filtering and deduplication setup to thousands of
        CPU cores. All the data processing steps involved in the creation of 🍷 FineWeb used this <a
                href="https://github.com/huggingface/datatrove">library</a>.</p>
    <h3>What is clean, good data?</h3>
    <p>This is probably the main question to keep in mind when
        creating a dataset. In the context of large language model pretraining, "high quality" is not a very well defined term<d-cite bibtex-key="albalak2024survey"></d-cite>, and often not a property of documents that can be easily perceived through direct observation alone.<d-cite bibtex-key="longpre2023pretrainers"></d-cite></p>
    <p>It is still common to train a model on a given corpus
        (wikipedia, or some other web dataset considered clean) and use it to check the perplexity on the dataset
        that we were trying to curate<d-cite bibtex-key="wenzek2019ccnet"></d-cite>. Unfortunately this does not always correlate with performance on downstream
        tasks<d-cite bibtex-key="soldaini2024dolma"></d-cite>, and so another often used approach is to train small models (small because training models is
        expensive and time consuming, and we want to be able to quickly iterate) on a representative subset of our dataset and evaluate them on
        a set of evaluation tasks. As we are curating a dataset for pretraining a generalist LLM, it is important to
        choose a diverse set of tasks and try not to overfit to any one individual benchmark.</p>
    <p>Another way to evaluate different datasets would be to
        train a model on each one and have humans rate and compare their outputs (like on the <a
                href="https://chat.lmsys.org/">LMSYS Chatbot Arena</a>)<d-cite bibtex-key="chiang2024chatbot"></d-cite>. This would arguably provide the most
        reliable results in terms of representing real model usage, but getting ablation results this way is too
        expensive and slow. It also often requires that the models have undergone at least an instruction finetuning stage, as pretrained models have difficulty following instructions.<d-cite bibtex-key="ouyang2022training"></d-cite></p>
    <p>The approach we ultimately went with was to train small
        models and evaluate them on a set of benchmark tasks. We believe this is a reasonable proxy for the quality
        of the data used to train these models.</p>
    <h3>Ablations and evaluation setup</h3>
    <p>To be able to compare the impact of a given processing
        step, we would train 2 models, one where the data included the extra step and another where this step was
        ablated (cut/removed). These 2 models would have the same number of parameters, architecture, and be trained
        on an equal number of randomly sampled tokens from each step's data, for a single epoch, and with the same hyperparameters — the only difference would be in the
        training data. We would then evaluate each model on the same set of tasks and compare the average
        scores.</p>
    <p>Our ablation models were trained using <a
            href="https://github.com/huggingface/nanotron"><code>nanotron</code></a> with this config [<strong>TODO:
        INSERT SIMPLIFIED NANOTRON CONFIG HERE</strong>]. The models had 1.82B parameters, used the Llama
        architecture with a 2048 sequence length, and a global batch size of ~2 million tokens. For filtering
        ablations we mostly trained on ~28B tokens (which is roughly the Chinchilla<d-cite bibtex-key="hoffmann2022training"></d-cite> optimal training size for this
        model size).</p>
    <p>We evaluated the models using <a
            href="https://github.com/huggingface/lighteval/"><code>lighteval</code></a>. We tried selecting
        benchmarks that would provide good signal at a relatively small scale (small models trained on only a few
        billion tokens). Furthermore, we also used the following criteria when selecting benchmarks:</p>
    <ul>
        <li>small variance between runs trained on different samplings of the same
            dataset: we want our runs on a subset of the data to be representative of the whole dataset, and the
            resulting scores to have as little evaluation noise as possible
        </li>
    </ul>
    <ul>
        <li>performance increasing monotonically (or close) over a training run:
            ideally, as the number of seen tokens increases, the performance on this benchmark should not decrease
            (which would be indicative of unreliable results at a small scale)
        </li>
    </ul>
    <p>We selected the following list of benchmarks:</p>
    <ul>
        <li>CommonSense QA<d-cite bibtex-key="talmor-etal-2019-commonsenseqa"></d-cite></li>
        <li>HellaSwag<d-cite bibtex-key="zellers-etal-2019-hellaswag"></d-cite></li>
        <li>OpenBook QA<d-cite bibtex-key="OpenBookQA2018"></d-cite></li>
        <li>PIQA<d-cite bibtex-key="bisk2019piqa"></d-cite></li>
        <li>SIQA<d-cite bibtex-key="sap2019socialiqa"></d-cite></li>
        <li>WinoGrande<d-cite bibtex-key="sakaguchi2019winogrande"></d-cite></li>
        <li>ARC<d-cite bibtex-key="clark2018think"></d-cite></li>
        <li>MMLU<d-cite bibtex-key="hendrycks2021measuring"></d-cite></li>
    </ul>
    <p>To
        have results quickly we capped longer benchmarks at 1000 samples (wall-clock evaluation taking less than 5
        min on a single node of 8 GPUs - done in parallel to the training).</p>
    <aside>You can find the full list of tasks and prompts we used <a
            href="https://huggingface.co/datasets/HuggingFaceFW/fineweb/blob/main/lighteval_tasks.py">here</a>.</aside>
    <h2>The 🍷 FineWeb recipe</h2>
    <p>In the next subsections we will explain each of the steps
        taken to produce the FineWeb dataset.</p>
    <figure class="l-body">
        <img src="assets/images/fineweb-recipe.png"/>
    </figure>
    <aside>You can find a fully reproducible <code>datatrove</code> config <a
                href="https://github.com/huggingface/datatrove/blob/main/examples/fineweb.py">here</a>.</aside>
    <h3>Starting point: text extraction</h3>
    <p>CommonCrawl data is available in two main formats: WARC
        and WET. <strong>WARC </strong>(Web ARChive format) files contain the raw data from the crawl, including the
        full page HTML and request metadata. <strong>WET</strong> (WARC Encapsulated Text) files provide a text only
        version of those websites.</p>
    <p>A large number of datasets take the WET files as their
        starting point. In our experience the default text extraction (extracting the main text of a webpage from
        its HTML) used to create these WET files is suboptimal and there are a variety of open-source libraries that
        provide better text extraction (by, namely, keeping less boilerplate content/navigation menus). We extracted
        the text content from the WARC files using the trafilatura library<d-cite bibtex-key="barbaresi-2021-trafilatura"></d-cite>, which from visual inspection of the results provided good quality extraction when compared to other libraries.</p><aside>You can also find a benchmark on text extraction libraries <a href="https://github.com/scrapinghub/article-extraction-benchmark/blob/master/README.rst">here</a>.</aside>
    <p>To validate this decision, we processed the 2019-18 dump
        directly using the WET files and with text extracted from WARC files using trafilatura<d-footnote>We used trafilatura default options with <code>favour_precision=True</code>.</d-footnote>. We applied the same
        processing to each one (our base filtering+minhash, detailed below) and trained two models. While the
        resulting dataset is about 25% larger for the WET data (around 254 billion tokens), it proves to be of much worse
        quality than the one that used trafilatura to extract text from WARC files (which is around 200 billion tokens). Visual inspection of some samples confirms that many of
        these additional tokens on the WET files are unnecessary page boilerplate.</p>
    <p>It is important to note, however, that text extraction is one of the most costly steps of our
        processing, so we believe that using the readily available WET data could be a reasonable trade-off for
        lower budget teams.</p>
    <div class="main-plot-container">
        <figure><img src="assets/images/wet_comparison.png"/></figure>
        <div id="plot-wet_comparison"></div>
    </div>
    <h3>Base filtering</h3>
    <p>Filtering is an important part of the curation process. It
        removes part of the data (be it words, lines, or full documents) that would harm performance and is thus
        deemed to be “lower quality”.</p>
    <p>As a basis for our filtering we used part of the setup
        from RefinedWeb<d-cite bibtex-key="penedo2023refinedweb"></d-cite>. Namely, we:</p>
    <ul>
        <li>Applied URL filtering using a <a
                href="https://dsi.ut-capitole.fr/blacklists/">blocklist</a> to remove adult content
        </li>
    </ul>
    <ul>
        <li>Applied a <a
                href="https://fasttext.cc/docs/en/language-identification.html">fastText language classifier</a><d-cite bibtex-key="joulin2016bag"></d-cite><d-cite bibtex-key="joulin2016fasttext"></d-cite> to
            keep only English text with a score ≥ 0.65
        </li>
    </ul>
    <ul>
        <li>Applied quality and repetition filters from MassiveText<d-cite bibtex-key="rae2022scaling"></d-cite> (using the default thresholds)
        </li>
    </ul>
    <p>After applying this filtering to each of the text
        extracted dumps (there are currently 96 dumps) we obtained roughly 36 trillion tokens of data (when
        tokenized with the <code>gpt2</code> tokenizer).</p>
    <h3>Deduplication</h3>
    <p>Deduplication is one of the most important steps when creating large web datasets for LLM pretraining. Methods to deduplicate datasets attempt to identify and remove redundant/repeated data from the dataset. </p>
    <h4>Why deduplicate?</h4>
    <p>The web has many aggregators, mirrors, templated pages or
        just otherwise repeated content spread over different domains and webpages. Often, these duplicated pages
        can be introduced by the crawler itself, when different links point to the same page. </p>
    <p>Removing these duplicates (deduplicating) has been linked to an improvement in model performance<d-cite bibtex-key="lee2022deduplicating"></d-cite> and a reduction in memorization of pretraining data<d-cite bibtex-key="carlini2023quantifying"></d-cite>, which might
        allow for better generalization. Additionally, the performance uplift obtained through deduplication can also be tied to increased training
        efficiency: by removing duplicated content, for the same number of training tokens, a model will have seen
        more diverse data.<d-cite bibtex-key="muennighoff2023scaling"></d-cite><d-cite bibtex-key="hernandez2022scaling"></d-cite></p>
    <p>There are different ways to identify and even define
        duplicated data. Common approaches rely on hashing techniques to speed up the process, or on building
        efficient data structures to index the data (like suffix arrays). Methods can also be “fuzzy”, by using some
        similarity metric to mark documents as duplicates, or “exact” by checking for exact matches between two
        documents (or lines, paragraphs, or whatever other granularity level being used).</p>
    <h4>Our deduplication parameters</h4>
    <p>Similarly to RefinedWeb<d-cite bibtex-key="penedo2023refinedweb"></d-cite>, we decided to apply MinHash, a
        fuzzy hash based deduplication technique that scales well and allows us to tune similarity thresholds (by changing the number and size of buckets) and the granularity of the matches (by changing the n-gram size). We chose to compute minhashes on each document’s 5-grams, using
        112 hash functions in total, split into 14 buckets of 8 hashes each — targeting documents that are at least
        75% similar. Documents with the same 8 minhashes in any bucket are considered a duplicate of each other.</p>
    <p>This would mean that for two documents with a similarity ($$s$$)
        of 0.7, 0.75, 0.8 and 0.85, the probability that they would be identified as duplicates would be 56%, 77%,
        92% and 98.8% respectively ($$1-(1-s^8)^{14}$$). See the plot below for a match probability
        comparison between our setup with 112 hashes and the one from RefinedWeb, with 9000 hashes, divided into 450
        buckets of 20 hashes (that requires a substantially larger amount of compute resources, as each individual hash must be computed, stored and then compared with hashes from other documents):</p>
    <div class="main-plot-container">
        <figure><img src="assets/images/minhash_params.png"/></figure>
        <div id="plot-minhash_params"></div>
    </div>
    <p>While the high number of hash functions in RefinedWeb
        allows for a steeper, more well defined cut off (documents with real similarity near the threshold are more likely to be correctly identified), we believe the compute and storage savings are a reasonable
        trade off.</p>
    <p>It should also be noted that intra-document deduplication is already handled by our repetition filter, which removes documents with many repeated lines and paragraphs.</p>
    <h4>More deduplication is always better, right?</h4>
    <p>Our initial approach was to take the entire dataset (all
        90+ dumps) and deduplicate them together as one big dataset using MinHash.</p>
    <p>We did this in an iterative manner: starting with the most
        recent dump (which at the time was 2023-50) and proceeding chronologically until the oldest one, we would deduplicate each dump
        not only within itself, but we would also remove any matches with documents from the previously processed (more recent)
        dumps. </p>
    <p>For instance, for the second most recent dump (2023-40 at
        the time), we deduplicated it against the most recent one in addition to within itself. In particular, the oldest
        dump was deduplicated against all other dumps. As a result, more data was removed from the oldest dumps (last
        to be deduplicated) than from the most recent ones.</p>
    <p>Deduplicating the dataset in this manner resulted in 4
        trillion tokens of data, but, quite surprisingly for us, when training on a randomly sampled 350 billion
        tokens subset, the model showed no improvement over one trained on the non deduplicated data (see orange and
        green curves below), scoring far below its predecessor RefinedWeb on our aggregate of tasks.</p>
    <div class="main-plot-container">
        <figure><img src="assets/images/dedup_all_dumps_bad.png"/></figure>
        <div id="plot-all_dumps_bad"></div>
    </div>
    <p>This was quite puzzling as our intuition regarding web
        data was that more deduplication would always result in improved performance. We decided to take a closer
        look at one of the oldest dumps, dump 2013-48:</p>
    <ul>
        <li>pre deduplication, this dump had ~490 billion tokens</li>
    </ul>
    <ul>
        <li>after our iterative MinHash, ~31 billion tokens remained (94% of data had been
            removed)
        </li>
    </ul>
    <p>As an experiment, we tried training two models on 28 billion tokens
        sampled from the following data from 2013-48:</p>
    <ul>
        <li>the fully deduplicated remaining ~31 billion tokens (<em>originally kept
            data</em>)
        </li>
    </ul>
    <ul>
        <li>171 billion tokens obtained by individually deduplicating (without
            considering the other dumps) the ~460 billion tokens that had been removed from this dump in the
            iterative dedup process (<em>originally removed data</em>)<d-footnote>While there may be documents in <em>originally kept
            data</em> similar to documents in <em>originally removed data</em>, we estimate the overlap to be small (around 4 billion tokens)</d-footnote>
        </li>
    </ul>
    <div class="main-plot-container">
        <figure><img src="assets/images/removed_data_cross_dedup.png"/></figure>
        <div id="plot-removed_data_dedup"></div>
    </div>
    <p>These results show that, for this older dump from which we had
        removed over 90% of the original data, the data that was kept was actually <em>worse</em> than the data
        removed (considered independently of all the other dumps). This is also confirmed by visual inspection: <em>originally kept
            data</em> contains far more ads, lists of keywords and generally badly formatted text than <em>originally removed data</em>.</p>
    <h4>Taking a step back: individual dump dedup</h4>
    <p>We then tried an alternative approach: we deduplicated
        each dump with MinHash individually (without considering the other dumps). This resulted in 20 trillion
        tokens of data.</p>
    <p>When training on a random sample from this dataset we see
        that it now matches RefinedWeb’s performance (blue and red curves below):</p>
    <div class="main-plot-container">
        <figure><img src="assets/images/cross_ind_unfiltered_comparison.png"/></figure>
        <div id="plot-ind_dedup_better"></div>
    </div>
    <p>We hypothesize that the main improvement gained from
        deduplication is the removal of very large clusters that are present in every single dump (you will find
        some examples of these clusters on the RefinedWeb paper, each containing <em>hundreds of thousands</em> of
        documents) and that further deduplication for clusters with a low number of duplicates (less than ~100 i.e. the number
        of dumps) actually harms performance: data that does not find a duplicate match in any other dump might
        actually be worse quality/more out of distribution (as evidenced by the results on the 2013-48 data). </p>
    <p>While you might see some performance improvement when
        deduplicating a few dumps together, at the scale of the entire dataset (all the dumps), the effect from this upsampling of lower quality data side
        effect seems to be more impactful.</p>
    <p>One possibility to consider is that as filtering quality
        improves, this effect may not be as prevalent, since the filtering might be able to remove some of this
        lower quality data. We also experimented with applying different, and often “lighter”, deduplication
        approaches on top of the individually deduplicated dumps. You can read about them further below.</p>
    <h4>A note on measuring the effect of deduplication</h4>
    <p>Given the nature of deduplication, its effect is not
        always very visible in a smaller slice of the dataset (such as 28B tokens, the size we used for our
        filtering ablations). Furthermore, one must consider the fact that there are specific effects at play when
        deduplicating across all CommonCrawl dumps, as some URLs/pages are recrawled from one dump to the next.</p>
    <p>To visualize the effect of scaling the number of training
        tokens on measuring deduplication impact, we considered the following (very extreme and unrealistic
        regarding the degree of duplication observed) theoretical scenario:</p>
    <ul>
        <li>there are 100 CommonCrawl dumps (roughly accurate)</li>
    </ul>
    <ul>
        <li>each dump has been perfectly individually deduplicated (every single
            document in it is unique)
        </li>
    </ul>
    <ul>
        <li>each dump is a perfect copy of each other (maximum possible duplication
            across dumps, effectively the worst case scenario)
        </li>
    </ul>
    <ul>
        <li>each dump has 200 billion tokens (for a total of 20 trillion, the resulting
            size of our individual dedup above)
        </li>
    </ul>
    <ul>
        <li>each dump is made up of documents of 1k tokens (200M documents per dump)
        </li>
    </ul>
    <p>We then simulated uniformly sampling documents from this
        entire dataset of 20 trillion tokens, to obtain subsets of 1B, 10B, 100B, 350B and 1T tokens. In the image
        below you can see how often each document would be repeated.</p>
    <div class="main-plot-container">
        <figure><img src="assets/images/duplicates_simul.png"/></figure>
        <div id="plot-duplicates-simul"></div>
    </div>
    <p>For 1B almost all documents would be unique
        (#duplicates=1), despite the fact that in the entire dataset each document is repeated 100 times (once per
        dump). We start seeing some changes at the 100B scale (0.5% of the total dataset), with a large number of
        documents being repeated twice, and a few even 4-8 times. At the larger scale of 1T (5% of the total
        dataset), the majority of the documents are repeated up to 8 times, with some being repeated up to 16
        times. </p>
    <p>We ran our performance evaluations for the deduplicated
        data at the 350B scale, which would, under this theoretical scenario, be made up of a significant portion of
        documents duplicated up to 8 times. This simulation illustrates the inherent difficulties associated with
        measuring deduplication impact on the training of LLMs, once the biggest duplicate clusters have been
        removed.</p>
    <h4>Other (failed) global approaches</h4>
    <p>We attempted to improve the performance of the
        independently minhash deduped 20 trillion tokens of data by further deduplicating it (globally, over all dumps) with the following methods:</p>
    <ul>
        <li>URL deduplication, where we only kept one document per normalized
            (lowercased) URL (71.5% of tokens removed, 5.6T left) — <em>FineWeb URL dedup</em></li>
    </ul>
    <ul>
        <li>Line deduplication:
            <ul>
                <li>remove all but 1 (randomly chosen) occurrence of each duplicated line (77.8% of
                    tokens dropped, 4.4T left) — <em>FineWeb line dedup</em></li>
            </ul>
            <ul>
                <li>same as above, but only removing duplicate lines with at least 10
                    words and dropping documents with fewer than 3 sentences after deduplication (85% of tokens
                    dropped, 2.9T left) — <em>FineWeb line dedup w/ min words</em></li>
            </ul>
            <ul>
                <li>remove all but 1 occurrence of each span of 3 duplicated lines
                    with each number treated as 0 when finding duplicates, (80.9% of tokens removed, 3.7T left) — <em>FineWeb 3-line
                        dedup</em></li>
            </ul>
        </li>
    </ul>
    <p>The performance of the models trained on each of these was
        consistently worse (even if to different degrees) than that of the original independently deduplicated
        data:</p>
    <div class="main-plot-container">
        <figure><img src="assets/images/dedup_attempts.png"/></figure>
        <div id="plot-dedup_attempts"></div>
    </div>
    <h3>Additional filtering</h3>
    <p>By this point we had reached the same performance as
        RefinedWeb with base filtering + independent MinHash, but on our aggregate of tasks, another heavily filtered dataset, the C4 dataset<d-cite bibtex-key="raffel2023exploring"></d-cite>, still showed stronger performance (with
        the caveat that it is a relatively small dataset for current web-scale standards).</p>
    <p>We therefore set out to find new filtering steps that
        would, at first, allow us to match the performance of C4 and, at a second stage, surpass it. A natural starting point
        was to look into the processing of C4 itself.</p>
    <h4>C4: A dataset that has stood the test of time</h4>
    <p>The <a href="https://huggingface.co/datasets/c4">C4
        dataset</a> was first released in 2019. It was obtained from the <code>2019-18</code> CommonCrawl dump by
        removing non english data, applying some heuristic filters on both the line and document level,
        deduplicating on the line level, and removing documents containing words from a word blocklist.</p>
    <p>Despite its age and limited size for current standards (around 175B gpt2 tokens), this dataset is, to this day, a common sub-set of typical LLM training, being used in models such as the relatively recent Llama1<d-cite bibtex-key="touvron2023llama"></d-cite>.
        This success is due to the strong performance that models trained on this dataset exhibit, excelling in particular on the Hellaswag
        benchmark <d-cite bibtex-key="zellers-etal-2019-hellaswag"></d-cite>, one of the benchmarks in our “early signal” group with the highest
        signal-to-noise ratio. We experimented applying
        each of the different filters used in C4 to a baseline of the independently deduped FineWeb 2019-18 dump:</p>
    <div class="main-plot-container">
        <figure><img src="assets/images/c4_filters_hellaswag.png"/></figure>
        <div id="plot-c4_filters_hellaswag"></div>
    </div>
    <ul>
        <li>applying “All filters” (drop lines not ending on punctuation marks,
            mentioning javascript and cookie notices + drop documents outside length thresholds, containing “lorem
            ipsum” or a curly bracket, <code>{</code>) allows us to match C4’s HellaSwag performance (purple versus
            pink curves).
        </li>
    </ul>
    <ul>
        <li>The curly bracket filter, and the word lengths filter only give a small
            boost, removing 2.8% and 4.3% of tokens, respectively
        </li>
    </ul>
    <ul>
        <li>The terminal punctuation filter, by itself, gives the biggest individual
            boost, but removes <em>around 30%</em> of all tokens (!)
        </li>
    </ul>
    <ul>
        <li>The lorem_ipsum, javascript and policy rules each remove &lt;0.5% of
            training tokens, so we did not train on them individually
        </li>
    </ul>
    <ul>
        <li>"All filters except the (very destructive) terminal_punct" performs better than
            terminal_punct by itself, while removing less in total (~7%)
        </li>
    </ul>
    <p>We decided to apply all C4 filters mentioned above except
        the terminal punctuation one. We validated these results with a longer run, which you will find in a plot in
        the next section.</p>
    <h4>A statistical approach to develop heuristic filters</h4>
    <p>To develop new heuristic filters and select their thresholds we devised a systematic process:</p>
    <ol><li>we started by collecting a very large list of high level statistics (over <strong>50</strong>) ranging from common document-level
        metrics (e.g. number of lines, avg. line/word length, etc) to inter-document repetition metrics (MassiveText
        inspired), on both a high quality and a lower quality web dataset;</li>
        <li>we selected the metrics for which the Wasserstein distance between the two distributions (of the metric computed on each dataset) was larger;</li>
        <li>we inspected the histograms of the two distributions and empirically chose a threshold that would make the lower quality dataset more closely resemble the higher quality one on this metric;</li>
        <li>we validated the resulting filter (metric-threshold pair) by using it on a reference dataset and running small ablations.</li>
    </ol>
    <p>Due to our assumption that global MinHash greatly upsamples lower quality data in the oldest dumps, we computed metrics on both the independently
        MinHashed and the (worse quality) global MinHashed versions of the 2013-48 and 2015-22 crawls (two older crawls). We then compared the
        statistics at a macro level, by looking at the distribution of these metrics for each one.</p>
    <p>Perhaps not too surprisingly given our findings for deduplication, we found significant
        disparities in most of the metrics for the two deduplication methods. For instance, the <code>line-char-duplicates</code>
        metric (nb. of characters in duplicated lines / nb. characters), roughly doubled from the independent dedup
        (0.0053 for 2015-22 and 0.0058 for 2013-48), to the global dedup (0.011 for 2015-22 and 0.01 for 2013-48),
        indicating that the latter had higher inter-document repetition.</p>
    <p>Following the process listed above for these datasets yielded 17 candidate
        metric-threshold pairs. In the image below, you can see 3 of these histograms:</p>
    <div class="main-plot-container">
        <figure><img src="assets/images/custom_filters.png"/></figure>
        <div id="plot-stats"></div>
    </div>

    <p>As an example, we inspected the histograms of "fraction of lines ending with punctuation" (see the image above) and observed an increased document density of global MinHash at around 0.12.
        We then filtered with this threshold and found that the removed data had a higher amount of short lists or consisted of only document layout text ("Home", "Sign up", etc).
    </p>

    <p>We then assessed the effectiveness of these 17 newly created
        filters, by conducting <strong>28B tokens</strong> ablation runs on the <strong>2019-18 crawl</strong>. Out
        of all those runs, we identified three filters (the ones based on the histograms above) that demonstrated
        the most significant improvements on the aggregate score:</p>
    <ul>
        <li>Remove documents where the fraction of lines ending with punctuation ≤ 0.12
            (10.14% of tokens removed) — vs the 30% from the original C4 terminal punct filter
        </li>
    </ul>
    <ul>
        <li>Remove documents where the fraction of characters in duplicated lines ≥ 0.1
            (12.47% of tokens removed) — the original MassiveText threshold for this ratio is ≥ 0.2
        </li>
    </ul>
    <ul>
        <li>Remove documents where the fraction of lines shorter than 30 characters ≥
            0.67 (3.73% of tokens removed)
        </li>
    </ul>
    <ul>
        <li>When applying the 3 together, ~22% of tokens were removed.</li>
    </ul>
    <div class="main-plot-container">
        <figure><img src="assets/images/custom_filters.png"/></figure>
        <div id="plot-custom_filters"></div>
    </div>
    <p>These filters allowed us to further improve performance and to, notably, surpass the C4 dataset performance.</p>
    <h2>The final dataset</h2>
    <p>The final 🍷 FineWeb dataset comprises 15T tokens and
        includes the following previously mentioned steps, in order, each providing a performance boost on our group
        of benchmark tasks:</p>
    <ul>
        <li>base filtering</li>
    </ul>
    <ul>
        <li>independent MinHash deduplication per dump</li>
    </ul>
    <ul>
        <li>a selection of C4 filters</li>
    </ul>
    <ul>
        <li>our custom filters (mentioned in the previous section)</li>
    </ul>
    <div class="main-plot-container">
        <figure><img src="assets/images/filtering_steps.png"/></figure>
        <div id="plot-all_filtering_steps"></div>
    </div>
    <p>We compared 🍷 FineWeb with the following datasets:</p>
    <ul>
        <li><a
                href="https://huggingface.co/datasets/tiiuae/falcon-refinedweb">RefinedWeb</a><d-cite bibtex-key="penedo2023refinedweb"></d-cite>
        </li>
    </ul>
    <ul>
        <li><a href="https://huggingface.co/datasets/allenai/c4">C4</a><d-cite bibtex-key="raffel2023exploring"></d-cite></li>
    </ul>
    <ul>
        <li><a href="https://huggingface.co/datasets/allenai/dolma">Dolma v1.6</a> (the
            CommonCrawl part) <d-cite bibtex-key="dolma"></d-cite>
        </li>
    </ul>
    <ul>
        <li><a href="https://huggingface.co/datasets/EleutherAI/pile">The Pile</a> <d-cite bibtex-key="gao2020pile"></d-cite></li>
    </ul>
    <ul>
        <li><a
                href="https://huggingface.co/datasets/cerebras/SlimPajama-627B">SlimPajama</a> <d-cite bibtex-key="cerebras2023slimpajama"></d-cite>
        </li>
    </ul>
    <ul>
        <li><a
                href="https://huggingface.co/datasets/togethercomputer/RedPajama-Data-V2">RedPajama2</a> <d-cite bibtex-key="together2023redpajama"></d-cite>
            (deduplicated)
        </li>
    </ul>
    <p>You will find these models on <a
            href="https://huggingface.co/collections/HuggingFaceFW/ablation-models-662457b0d213e8c14fe47f32">this
        collection</a>. We have uploaded checkpoints at every 1000 training steps. You will also find our full <a
            href="https://huggingface.co/datasets/HuggingFaceFW/fineweb/blob/main/eval_results.csv">evaluation
        results here</a>.</p>
    <div class="main-plot-container">
        <figure><img src="assets/images/dataset_ablations.png"/></figure>
        <div id="plot-dataset_ablations"></div>
    </div>
    <p>Large language models pretrained on 🍷 FineWeb, the largest publicly available clean LLM pretraining dataset, are better-performing than other open pretraining datasets.</p>
    <h2>📚 FineWeb-Edu</h2>
    <p>A new approach has recently emerged for filtering LLM training datasets: using synthetic data to develop classifiers for identifying educational content. This technique was used in the trainings of Llama 3<d-cite bibtex-key="llama3modelcard"></d-cite> and Phi3<d-cite bibtex-key="abdin2024phi"></d-cite> but its large-scale impact on web data filtering hasn't been fully explored or published.</p>
    <p>The popular Phi3 models were trained on 3.3 and 4.8 trillion tokens, with the paper<d-cite bibtex-key="abdin2024phi"></d-cite> stating:</p>
    <blockquote>Our training data consists of heavily filtered publicly available web data (according to the 'educational level') from various open internet sources, as well as synthetic LLM-generated data.</blockquote>
    <p>Similarly, Llama 3 blog post<d-cite bibtex-key="meta2024responsible"></d-cite> notes:</p>
    <blockquote>We found that previous generations of Llama are good at identifying high-quality data, so we used Llama 2 to help build the text-quality classifiers that are powering Llama 3.</blockquote>
    <p>However, these classifiers and filtered datasets are not publicly available. To further enhance 🍷 FineWeb's quality, we developed an educational quality classifier using annotations generated by <a href="https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct">Llama-3-70B-Instruct</a> to create <a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu"><strong>📚 FineWeb-Edu</strong></a>.</p>
    <h3>Annotation</h3>
    <p>We used Llama-3-70B-Instruct to annotate 500k samples from 🍷 FineWeb, scoring each for their educational quality on a scale from 0 to 5.</p>
    <p>We explored various prompts and found that the additive scale by Yuan et al.<d-cite bibtex-key="yuan2024self"></d-cite> worked best. This scale allows the LLM to reason about each additional point awarded, unlike the single-rating Likert scale which fits samples into predefined boxes. Then, to avoid the LLM favoring highly technical pages like arXiv abstracts and submissions, we focused on grade-school and middle-school level knowledge. By setting a threshold of 3 (on a scale of 0 to 5) during the filtering process, we were able to also retain some high-level educational pages.</p>
    <div style="text-align: center; margin: 20px 0;">
        <img src="https://cdn-uploads.huggingface.co/production/uploads/61c141342aac764ce1654e43/fjZQ4izIj1rx1xQnBTKKr.png" alt="Prompt for LLM annotation" style="width: 90%; max-width: 800px; height: auto;">
        <figcaption style="font-style: italic; margin-top: 10px;">Prompt used for Llama3 annotations of the educational score, also available <a href="https://huggingface.co/HuggingFaceFW/fineweb-edu-classifier/blob/main/utils/prompt.txt">here</a>.</figcaption>
    </div>    
    <p>We also experimented with  <a href="https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1">Mixtral-8x-7B-Instruct</a> and <a href="https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1">Mixtral-8x22B-Instruct</a> and a jury of all three models<d-cite bibtex-key="verga2024replacing"></d-cite> but found that Llama3 alone gave the most reliable results.</p>
    <h3>Classifier Training</h3>
    <p>We added a classification head with a single regression output to <a href="https://huggingface.co/Snowflake/snowflake-arctic-embed-m">Snowflake-arctic-embed</a> and trained it on 450,000 Llama 3 annotations for 20 epochs with a learning rate of 3e-4, freezing the embedding and encoder layers.  We saved the checkpoint with the highest F1 score on our held-out validation set of 45k samples, treating Llama 3 annotations as ground-truth. After training, we rounded the scores to integers from 0 to 5.</p>
    <p>We then converted the problem to a binary classification task by using a fixed threshold to determine if a file is educational. With a threshold of 3, the model achieved an F1 score of 82% on the validation set, indicating strong performance in distinguishing high-quality educational content.</p>
    <p>The classifier is available at: <a href="https://huggingface.co/HuggingFaceFW/fineweb-edu-classifier">HuggingFaceFW/fineweb-edu-classifier</a>. The training and inference code is available on  <a href="https://github.com/huggingface/cosmopedia/tree/main/classification">GitHub</a>.</p>
    <h3>Filtering and results</h3>
    <p>We applied the classifier to the 15T tokens of 🍷 FineWeb, a process that required 6,000 H100 GPU hours. We investigated the impact of using different thresholds for the filtering and found that threshold 3 gave the best overall results. Although using a threshold higher than 3 improves performance on knowledge and reasoning intensive benchmarks, it significantly degrades performance on HellaSwag and PIQA. The plot below shows the performance of each threshold compared to FineWeb on six different benchmarks; it uses a 1.82B model trained on 8B tokens.</p>
    <div class="main-plot-container">
        <figure>
            <img src="assets/images/edu-8k.png">
        </figure>
        <div id="plot-edu-8k"></div>
    </div>
    <p>We then built  📚 FineWeb-Edu by filtering out samples with scores lower than 3. This removed 92% of the dataset, leaving us with 1.3 trillion educational tokens.  To evaluate the effectiveness of this filtering at a larger scale, we conducted an ablation using a 1.82B model trained on 350 billion tokens, similar to the FineWeb filtering ablation mentioned above:</p>
    <div class="main-plot-container">
        <figure>
            <img src="assets/images/edu-100k.png">
        </figure>
        <div id="plot-edu-100k"></div>
    </div>
    <p>Here are the key highlights of the ablation results above:</p>
    <ul>
        <li>📚 FineWeb-Edu surpasses 🍷 FineWeb and all other open web datasets, with remarkable improvements on educational benchmarks such as MMLU, ARC, and OpenBookQA.</li>
        <li>It achieves the same performance with significantly less data, requiring 10x fewer tokens compared to C4 and Dolma to match MMLU results.</li>
        <li>This demonstrates the effectiveness of using classifiers trained on LLM annotations for large-scale data filtering.</li>
    </ul>
    <p>Given that a threshold of 2 also demonstrated strong performance while retaining more data, we are releasing an additional dataset filtered with this threshold, containing 5.4 trillion tokens under <a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu-score-2">HuggingFaceFW/fineweb-edu-score-2</a>.</p>
    <p>You can find the two datasets along with the classifier used for the filtering in this <a href="https://huggingface.co/collections/HuggingFaceFW/fineweb-edu-6659c3f3d399d0e1d648adfd">collection</a>.</p>
    <h2>Next steps</h2>
    <p>Through our open data efforts we hope to give every model trainer the ability to create state-of-the-art large language models. As part of this process, we plan to continue iterating on FineWeb and to release more specialised filtered subsets of web data, in a fully open and reproducible manner.</p>
    <p>While English currently dominates the large language model landscape, we believe that making high quality training data for other languages more easily accessible would allow millions of non english speakers to benefit from these technologies and, as such, will also strive to adapt the FineWeb Recipe to a multilingual version.</p>
</d-article>

<d-appendix>
    <d-bibliography src="bibliography.bib"></d-bibliography>
</d-appendix>

<script>
    const article = document.querySelector('d-article');
    const toc = document.querySelector('d-contents');
    if (toc) {
        const headings = article.querySelectorAll('h2, h3, h4');
        let ToC = `<nav role="navigation" class="l-text figcaption"><h3>Table of contents</h3>`;
        let prevLevel = 0;

        for (const el of headings) {
            // should element be included in TOC?
            const isInTitle = el.parentElement.tagName == 'D-TITLE';
            const isException = el.getAttribute('no-toc');
            if (isInTitle || isException) continue;
            el.setAttribute('id', el.textContent.toLowerCase().replaceAll(" ", "_"))
            const link = '<a target="_self" href="' + '#' + el.getAttribute('id') + '">' + el.textContent + '</a>';

            const level = el.tagName === 'H2' ? 0 : (el.tagName === 'H3' ? 1 : 2);
            while (prevLevel < level) {
                ToC += '<ul>'
                prevLevel++;
            }
            while (prevLevel > level) {
                ToC += '</ul>'
                prevLevel--;
            }
            if (level === 0)
                ToC += '<div>' + link + '</div>';
            else
                ToC += '<li>' + link + '</li>';
        }

        while (prevLevel > 0) {
            ToC += '</ul>'
            prevLevel--;
        }
        ToC += '</nav>';
        toc.innerHTML = ToC;
        toc.setAttribute('prerendered', 'true');
        const toc_links = document.querySelectorAll('d-contents > nav a');

        window.addEventListener('scroll', (_event) => {
            if (typeof (headings) != 'undefined' && headings != null && typeof (toc_links) != 'undefined' && toc_links != null) {
                // Then iterate forwards, on the first match highlight it and break
                find_active: {
                    for (let i = headings.length - 1; i >= 0; i--) {
                        if (headings[i].getBoundingClientRect().top - 50 <= 0) {
                            if (!toc_links[i].classList.contains("active")) {
                                toc_links.forEach((link, _index) => {
                                    link.classList.remove("active");
                                });
                                toc_links[i].classList.add('active');
                            }
                            break find_active;
                        }
                    }
                    toc_links.forEach((link, _index) => {
                        link.classList.remove("active");
                    });
                }
            }
        });
    }
</script>
</body>