<commit_before>/// \file
/// \ingroup tutorial_tdataframe
/// \notebook -nodraw
/// Showcase registration of callback functions that act on partial results while
/// the event-loop is running using `OnPartialResult` and `OnPartialResultSlot`.
///
/// \macro_code
///
/// \date September 2017
/// \author <NAME>

using namespace ROOT::Experimental; // TDataFrame lives in here

void tdf013_InspectAnalysis()
{
#ifdef R__USE_IMT
   // Run on multiple cores if Implicit Multi-Threading has been enabled when compiling ROOT (default is yes)
   ROOT::EnableImplicitMT();
   const auto nSlots = ROOT::GetImplicitMTPoolSize();
#else
   const auto nSlots = 1u;
#endif

   // ## Setup a simple TDataFrame
   // We start by creating a TDataFrame with a good number of empty events
   const auto nEvents = nSlots * 10000ull;
   TDataFrame d(nEvents);

   // `heavyWork` is a lambda that fakes some interesting computation and just returns a normally distributed double
   TRandom r;
   auto heavyWork = [&r]() {
      for (volatile int i = 0; i < 1000000; ++i);
      return r.Gaus();
   };

   // Let's define a column "x" produced by invoking `heavyWork` for each event
   // `tdf` stores a modified data-frame that contains "x"
   auto tdf = d.Define("x", heavyWork);

   // Now we register a histogram-filling action with the TDataFrame.
   // `h` can be used just like a pointer to TH1D but it is actually a TResultProxy<TH1D>, a smart object that triggers
   // an event-loop to fill the pointee histogram if needed.
   auto h = tdf.Histo1D<double>({"browserHisto", "", 100, -2., 2.}, "x");

   // ## Use the callback mechanism to draw the histogram on a TBrowser while it is being filled
   // So far we have registered a column "x" to a data-frame with `nEvents` events and we registered the filling of a
   // histogram with the values of column "x".
   // In the following we will register three functions for execution during the event-loop:
   // - one is to be executed once just before the loop and adds a partially-filled histogram to a TBrowser
   // - the next is executed every 50 events and draws the partial histogram on the TBrowser's TPad
   // - another callback is responsible of updating a simple progress bar from multiple threads

   // First off we create a TBrowser that contains a "TDFResults" directory
   TMemFile *tdfDirectory = new TMemFile("TDFResults", "RECREATE");
   TBrowser *browser = new TBrowser("b", tdfDirectory);
   // The global pad should now be set to the TBrowser's canvas, let's store its value in a local variable
   auto browserPad = gPad;

   // A useful feature of `TResultProxy` is its `OnPartialResult` method: it allows us to register a callback that is
   // executed once per specified number of events during the event-loop, on "partial" versions of the result objects
   // contained in the `TResultProxy`. In this case, the partial result is going to be a histogram filled with an
   // increasing number of events.
   // Instead of requesting the callback to be executed every N entries, this time we use the special value `kOnce` to
   // request that it is executed once right before starting the event-loop.
   // The callback is a C++11 lambda that registers the partial result object in `tdfDirectory`.
   h.OnPartialResult(h.kOnce, [tdfDirectory](TH1D &h_) { tdfDirectory->Add(&h_); });
   // Note that we called `OnPartialResult` with a dot, `.`, since this is a method of `TResultProxy` itself.
   // We do not want to call `OnPartialResult` on the pointee histogram!)

   // Multiple callbacks can be registered on the same `TResultProxy` (they are executed one after the other in the
   // same order as they were registered). We now request that the partial result is drawn and the TBrowser's TPad is
   // updated every 50 events.
   h.OnPartialResult(50, [&browserPad](TH1D &hist) {
      browserPad->cd();
      hist.Draw();
      browserPad->Update();
      // This call tells ROOT to process all pending GUI events
      // It allows users to use the TBrowser as usual while the event-loop is running
      gSystem->ProcessEvents();
   });

   // Finally, we would like to print a progress bar on the terminal to show how the event-loop is progressing.
   // To take into account _all_ events we use `OnPartialResultSlot`: when Implicit Multi-Threading is enabled, in fact,
   // `OnPartialResult` invokes the callback only in one of the worker threads, and always returns that worker threads'
   // partial result. This is useful because it means we don't have to worry about concurrent execution and
   // thread-safety of the callbacks if we are happy with just one threads' partial result.
   // `OnPartialResultSlot`, on the other hand, invokes the callback in each one of the worker threads, every time a
   // thread finishes processing a batch of `everyN` events. This is what we want for the progress bar, but we need to
   // take care that two threads will not print to terminal at the same time: we need a std::mutex for synchronization.
   std::string progressBar;
   std::mutex barMutex; // Only one thread at a time can lock a mutex. Let's use this to avoid concurrent printing.
   // Magic numbers that yield good progress bars for nSlots = 1,2,4,8
   const auto everyN = nSlots == 8 ? 1000 : 100ull * nSlots;
   const auto barWidth = nEvents / everyN;
   h.OnPartialResultSlot(everyN, [&barWidth, &progressBar, &barMutex](unsigned int /*slot*/, TH1D &/*partialHist*/) {
      std::lock_guard<std::mutex> l(barMutex); // lock_guard locks the mutex at construction, releases it at destruction
      progressBar.push_back('#');
      // re-print the line with the progress bar
      std::cout << "\r[" << std::left << std::setw(barWidth) << progressBar << ']' << std::flush;
   });
   /***************************************/

   // ## Running the analysis
   // So far we told TDataFrame what we want to happen during the event-loop, but we have not actually run any of those
   // actions: the TBrowser is still empty, the progress bar has not been printed even once, and we haven't produced
   // a single data-point!
   // As usual with TDataFrame, the event-loop is triggered by accessing the contents of a TResultProxy for the first<commit_msg>[TDF] Small fixes to the callbacks tutorial<commit_after>/// \file
/// \ingroup tutorial_tdataframe
/// \notebook -nodraw
/// Showcase registration of callback functions that act on partial results while
/// the event-loop is running using `OnPartialResult` and `OnPartialResultSlot`.
///
/// \macro_code
///
/// \date September 2017
/// \author <NAME>

using namespace ROOT::Experimental; // TDataFrame lives in here

void tdf013_InspectAnalysis()
{
#ifdef R__USE_IMT
   // Run on multiple cores if Implicit Multi-Threading has been enabled when compiling ROOT (default is yes)
   ROOT::EnableImplicitMT();
   const auto nSlots = ROOT::GetImplicitMTPoolSize();
#else
   const auto nSlots = 1u;
#endif

   // ## Setup a simple TDataFrame
   // We start by creating a TDataFrame with a good number of empty events
   const auto nEvents = nSlots * 10000ull;
   TDataFrame d(nEvents);

   // `heavyWork` is a lambda that fakes some interesting computation and just returns a normally distributed double
   TRandom r;
   auto heavyWork = [&r]() {
      for (volatile int i = 0; i < 1000000; ++i);
      return r.Gaus();
   };

   // Let's define a column "x" produced by invoking `heavyWork` for each event
   // `tdf` stores a modified data-frame that contains "x"
   auto tdf = d.Define("x", heavyWork);

   // Now we register a histogram-filling action with the TDataFrame.
   // `h` can be used just like a pointer to TH1D but it is actually a TResultProxy<TH1D>, a smart object that triggers
   // an event-loop to fill the pointee histogram if needed.
   auto h = tdf.Histo1D<double>({"browserHisto", "", 100, -2., 2.}, "x");

   // ## Use the callback mechanism to draw the histogram on a TBrowser while it is being filled
   // So far we have registered a column "x" to a data-frame with `nEvents` events and we registered the filling of a
   // histogram with the values of column "x".
   // In the following we will register three functions for execution during the event-loop:
   // - one is to be executed once just before the loop and adds a partially-filled histogram to a TBrowser
   // - the next is executed every 50 events and draws the partial histogram on the TBrowser's TPad
   // - another callback is responsible of updating a simple progress bar from multiple threads

   // First off we create a TBrowser that contains a "TDFResults" directory
   auto *tdfDirectory = new TMemFile("TDFResults", "RECREATE");
   auto *browser = new TBrowser("b", tdfDirectory);
   // The global pad should now be set to the TBrowser's canvas, let's store its value in a local variable
   auto browserPad = gPad;

   // A useful feature of `TResultProxy` is its `OnPartialResult` method: it allows us to register a callback that is
   // executed once per specified number of events during the event-loop, on "partial" versions of the result objects
   // contained in the `TResultProxy`. In this case, the partial result is going to be a histogram filled with an
   // increasing number of events.
   // Instead of requesting the callback to be executed every N entries, this time we use the special value `kOnce` to
   // request that it is executed once right before starting the event-loop.
   // The callback is a C++11 lambda that registers the partial result object in `tdfDirectory`.
   h.OnPartialResult(h.kOnce, [tdfDirectory](TH1D &h_) { tdfDirectory->Add(&h_); });
   // Note that we called `OnPartialResult` with a dot, `.`, since this is a method of `TResultProxy` itself.
   // We do not want to call `OnPartialResult` on the pointee histogram!)

   // Multiple callbacks can be registered on the same `TResultProxy` (they are executed one after the other in the
   // same order as they were registered). We now request that the partial result is drawn and the TBrowser's TPad is
   // updated every 50 events.
   h.OnPartialResult(50, [&browserPad](TH1D &hist) {
      browserPad->cd();
      hist.Draw();
      browserPad->Update();
      // This call tells ROOT to process all pending GUI events
      // It allows users to use the TBrowser as usual while the event-loop is running
      gSystem->ProcessEvents();
   });

   // Finally, we would like to print a progress bar on the terminal to show how the event-loop is progressing.
   // To take into account _all_ events we use `OnPartialResultSlot`: when Implicit Multi-Threading is enabled, in fact,
   // `OnPartialResult` invokes the callback only in one of the worker threads, and always returns that worker threads'
   // partial result. This is useful because it means we don't have to worry about concurrent execution and
   // thread-safety of the callbacks if we are happy with just one threads' partial result.
   // `OnPartialResultSlot`, on the other hand, invokes the callback in each one of the worker threads, every time a
   // thread finishes processing a batch of `everyN` events. This is what we want for the progress bar, but we need to
   // take care that two threads will not print to terminal at the same time: we need a std::mutex for synchronization.
   std::string progressBar;
   std::mutex barMutex; // Only one thread at a time can lock a mutex. Let's use this to avoid concurrent printing.
   // Magic numbers that yield good progress bars for nSlots = 1,2,4,8
   const auto everyN = nSlots == 8 ? 1000 : 100ull * nSlots;
   const auto barWidth = nEvents / everyN;
   h.OnPartialResultSlot(everyN, [&barWidth, &progressBar, &barMutex](unsigned int /*slot*/, TH1D &/*partialHist*/) {
      std::lock_guard<std::mutex> l(barMutex); // lock_guard locks the mutex at construction, releases it at destruction
      progressBar.push_back('#');
      // re-print the line with the progress bar
      std::cout << "\r[" << std::left << std::setw(barWidth) << progressBar << ']' << std::flush;
   });

   // ## Running the analysis
   // So far we told TDataFrame what we want to happen during the event-loop, but we have not actually run any of those
   // actions: the TBrowser is still empty, the progress bar has not been printed even once, and we haven't produced
   // a single data-point!
   // As usual with TDataFrame, the event-loop is triggered by accessing the contents of a TResultProxy for the first
{'fraction_non_alphanumeric': 0.0894478823434863, 'fraction_numerical': 0.008358112995258378, 'mean_word_length': 3.3739894551845344, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3823580', 'n_tokens_mistral': 3490, 'n_tokens_neox': 3316, 'n_words': 1671}