<commit_before>#pragma once
#include "common.hpp"

namespace But
{
namespace Format
{
namespace detail
{

constexpr auto consumeUntilTheEndOfString(char const* fmt)
{
  while( not isVariableBegin(*fmt) && not isEos(*fmt) )
    ++fmt;
  return fmt;
}

constexpr auto consumeUntilEndOfSimepleVariable(char const* fmt)
{
  while( isDigit(*fmt) )
    ++fmt;
  if( isEos(*fmt) )
    return fmt;
  return throwOnInvalidSyntax( not isSpace(*fmt), "invalid number", fmt );
}

constexpr auto consumeUntilEndOfBraceVariable(char const* fmt)
{
  while( *fmt!='}' && not isEos(*fmt) )
    ++fmt;
  return throwOnInvalidSyntax( *fmt!='}', "variable declration not ended with a '}'", fmt );
}

constexpr auto consumeUntilEndOfVariable(char const* fmt)
{
  if( isDigit(*fmt) )
    return consumeUntilEndOfSimepleVariable(fmt+1);
  if( *fmt=='{')
    return consumeUntilEndOfBraceVariable(fmt+1);
  return throwOnInvalidSyntax(true, "expected either number or opening brace ('{') after variable declaration", fmt);
}


/** @brief helper function that determines number of arguments, based on a simplified syntax grammar.
 */
constexpr auto argumentsUsageCount(char const* fmt)
{
  auto args = 0u;
  while(*fmt)
  {
    if( isVariableBegin(*fmt) )
    {
      ++fmt;
      if( isEos(*fmt) )
        throwOnInvalidSyntax(true, "variable declaration terminated in the middle of parsing", fmt);
      if( isVariableBegin(*fmt) )
        ++fmt;
      else
      {
        fmt = consumeUntilEndOfVariable(fmt);
        ++args;
      }
    }
    else
      fmt = consumeUntilTheEndOfString(fmt+1);
  }
  // TODO: this function could equally well just count number of '$' characters, as this is only a hint...
  return args;
}

}
}
}
<commit_msg>removed TODO - this is not true - we need an exact count here<commit_after>#pragma once
#include "common.hpp"

namespace But
{
namespace Format
{
namespace detail
{

constexpr auto consumeUntilTheEndOfString(char const* fmt)
{
  while( not isVariableBegin(*fmt) && not isEos(*fmt) )
    ++fmt;
  return fmt;
}

constexpr auto consumeUntilEndOfSimepleVariable(char const* fmt)
{
  while( isDigit(*fmt) )
    ++fmt;
  if( isEos(*fmt) )
    return fmt;
  return throwOnInvalidSyntax( not isSpace(*fmt), "invalid number", fmt );
}

constexpr auto consumeUntilEndOfBraceVariable(char const* fmt)
{
  while( *fmt!='}' && not isEos(*fmt) )
    ++fmt;
  return throwOnInvalidSyntax( *fmt!='}', "variable declration not ended with a '}'", fmt );
}

constexpr auto consumeUntilEndOfVariable(char const* fmt)
{
  if( isDigit(*fmt) )
    return consumeUntilEndOfSimepleVariable(fmt+1);
  if( *fmt=='{')
    return consumeUntilEndOfBraceVariable(fmt+1);
  return throwOnInvalidSyntax(true, "expected either number or opening brace ('{') after variable declaration", fmt);
}


/** @brief helper function that determines number of arguments, based on a simplified syntax grammar.
 */
constexpr auto argumentsUsageCount(char const* fmt)
{
  auto args = 0u;
  while(*fmt)
  {
    if( isVariableBegin(*fmt) )
    {
      ++fmt;
      if( isEos(*fmt) )
        throwOnInvalidSyntax(true, "variable declaration terminated in the middle of parsing", fmt);
      if( isVariableBegin(*fmt) )
        ++fmt;
      else
      {
        fmt = consumeUntilEndOfVariable(fmt);
        ++args;
      }
    }
    else
      fmt = consumeUntilTheEndOfString(fmt+1);
  }
  return args;
}

}
}
}

{'fraction_non_alphanumeric': 0.12289512555391433, 'fraction_numerical': 0.002363367799113737, 'mean_word_length': 3.75561797752809, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1075928', 'n_tokens_mistral': 1085, 'n_tokens_neox': 1071, 'n_words': 302}