<commit_before>//----------------------------  expand_instantiations.cc  -------------------
//    $Id$
//    Version: $Name$
//
//    Copyright (C) 2007 by the deal.II authors
//
//    This file is subject to QPL and may not be  distributed
//    without copyright and license information. Please refer
//    to the file deal.II/doc/license.html for the  text  and
//    further information on this license.
//
//----------------------------  expand_instantiations.cc  -------------------

// This is the program that we use to generate explicit instantiations for a
// variety of template arguments. It takes two kinds of input files. The first
// is given as arguments on the command line and contains entries of the
// following form:
// --------------------
// REAL_SCALARS    := { double; float; long double }
// COMPLEX_SCALARS := { std::complex<double>;
//                      std::complex<float>;
//                      std::complex<long double> }
// VECTORS := { Vector<double>; Vector<float>; Vector<long double> }
// --------------------
//
// The input file of this form is typically located in the common/ directory
// and is generated by ./configure to contain the list of vectors etc that
// make sense for the current configuration. For example, the list of VECTORS
// is going to contain PETSc vectors if so configured.
//
// The second intput is read from the command line and consists of a sequence
// of statements of the following form:
// --------------------
// for (u,v:VECTORS; z:SCALARS) { f(u, z, const v &); }
// --------------------
// Here, everything between {...} will be copied as many times as there are
// combinations of arguments u,v in the list of substitutions given by
// VECTORS. For each copy, the arguments u,v will be replaced by one of these
// combinations.

// Author: <NAME>, 2007


#include <iostream>
#include <fstream>
#include <map>
#include <list>


// a map from the keys in the expansion lists to the list itself. For
// instance, the example above will lead to the entry
//      expansion_lists[REAL_SCALARS] = (double, float, long double)
// in this map, among others
std::map<std::string, std::list<std::string> >  expansion_lists;



/* ======================== auxiliary functions ================= */

// extract from the start of #in the part of the string that ends with one of
// the characters in #delim_list. The extracted part is deleted from #in
std::string
get_substring_with_delim (std::string       &in,
			  const std::string &delim_list)
{
  std::string x;
  while ((in.size() != 0)
	 &&
	 (delim_list.find (in[0]) == std::string::npos))
    {
      x += in[0];
      in.erase (0, 1);
    }

  return x;
}


// delete all whitespace at the beginning of the given argument
void
skip_space (std::string &in)
{
  while ((in.size() != 0)
	 &&
	 ((in[0] == ' ') || (in[0] == '\t') || (in[0] == '\n')))
    in.erase (0, 1);
}



// read the whole file specified by the stream given as argument into a string
// for simpler parsing, and return it
std::string read_whole_file (std::istream &in)
{
  std::string whole_file;
  while (in)
    {
      std::string line;
      getline (in, line);

      if (line.find ("//") != 0)
	{
	  whole_file += line;
	  whole_file += '\n';
	}
    }
				   // substitute tabs by spaces, multiple
				   // spaces by single ones
  for (unsigned int i=0; i<whole_file.size(); ++i)
    if (whole_file[i] == '\t')
      whole_file[i] = ' ';
  while (whole_file.find("  ") != std::string::npos)
    whole_file.replace (whole_file.find("  "), 2, " ");

  return whole_file;
}



// split a given string assumed to consist of a list of substrings delimited
// by a particular character into its components
std::list<std::string>
split_string_list (const std::string &s,
		   const char         delimiter)
{
  std::string tmp = s;
  std::list<std::string> split_list;

				   // split the input list
  while (tmp.length() != 0)
    {
      std::string name;
      name = tmp;

      if (name.find(delimiter) != std::string::npos)
	{
	  name.erase (name.find(delimiter), std::string::npos);
	  tmp.erase (0, tmp.find(delimiter)+1);
	}
      else
	tmp = "";

      skip_space (name);

      while (name[name.length()-1] == ' ')
	name.erase (name.length()-1, 1);

      split_list.push_back (name);
    }

  return split_list;
}



// determine whether a given substring at position #pos and length #length in
// the string #text is a real token, i.e. not just part of another word
bool is_real_token (const std::string &text,
		    const std::string::size_type pos,
		    const std::string::size_type length)
{
  static const std::string token_chars ("abcdefghijklmnopqrstuvwxyz"
					"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
					"0123456789"
					"_");
  if ((pos != 0) && (token_chars.find (text[pos-1]) != std::string::npos))
    return false;

  if ((pos+length < text.size()) &&
      (token_chars.find (text[pos+length]) != std::string::npos))
    return false;

  return true;
}


// substitute all occurrences of #token in #text by #substitute
std::string substitute_tokens (const std::string &text,
			       const std::string &token,
			       const std::string &substitute)
{
  std::string x_text = text;
  std::string::size_type pos = 0;
  while ((pos = x_text.find(token, pos)) != std::string::npos)
    {
      if (is_real_token (x_text, pos, token.size()))
	{  
	  x_text.replace (pos, token.size(), substitute);
	  pos += substitute.size();
	}
      else
	++pos;
    }
  
  return x_text;
}



/* ======================== the main functions ================= */


// read and parse the expansion lists like
//     REAL_SCALARS    := { double; float; long double }
// as specified at the top of the file and store them in
// the global expansion_lists variable
void read_expansion_lists (const std::string &filename)
{
  std::ifstream in (filename.c_str());

  if (! in)
    {
      std::cerr << "Instantiation list file can not be read!"
		<< std::endl;
      std::exit (1);
    }

				   // read the entire file into a string for
				   // simpler processing. replace end-of-line
				   // characters by spaces
  std::string whole_file = read_whole_file (in);

				   // now process entries of the form
				   // NAME := { class1; class2; ...}.
  while (whole_file.size() != 0)
    {
      const std::string
	name = get_substring_with_delim (whole_file, " :");

      skip_space (whole_file);
      if (whole_file.find (":=") != 0)
	{
	  std::cerr << "Invalid entry <" << name << '>' << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 2);
      skip_space (whole_file);
      if (whole_file.find ("{") != 0)
	{
	  std::cerr << "Invalid entry <" << name << '>' << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 1);
      skip_space (whole_file);

      std::string
	expansion = get_substring_with_delim (whole_file, "}");

      if (whole_file.find ("}") != 0)
	{
	  std::cerr << "Invalid entry <" << name << '>' << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 1);
      skip_space (whole_file);

      expansion_lists[name] = split_string_list (expansion, ';');
    }
}



// produce all combinations of substitutions of the tokens given in the
// #substitutions list in #text and output it to std::cout
void substitute (const std::string &text,
		 const std::list<std::pair<std::string, std::string> > &substitutions)
{
				   // do things recursively: if the list of
				   // substiutions has a single entry, then
				   // process all of them. otherwise, process
				   // the first in the list and call the
				   // function recursively with the rest of
				   // the substitutions
  if (substitutions.size() > 1)
    {
				       // do the first substitution, then call
				       // function recursively
      const std::string name    = substitutions.front().first,
			pattern = substitutions.front().second;

      const std::list<std::pair<std::string, std::string> >
	rest_of_substitutions (++substitutions.begin(),
			       substitutions.end());

      for (std::list<std::string>::const_iterator
	     expansion = expansion_lists[pattern].begin();
	   expansion != expansion_lists[pattern].end();
	   ++expansion)
	{
	  std::string new_text
	    = substitute_tokens (text, name, *expansion);

	  substitute (new_text, rest_of_substitutions);
	}
    }
  else if (substitutions.size() == 1)
    {
				       // do the substitutions
      const std::string name    = substitutions.front().first,
			pattern = substitutions.front().second;
      
      for (std::list<std::string>::const_iterator
	     expansion = expansion_lists[pattern].begin();
	   expansion != expansion_lists[pattern].end();
	   ++expansion)
	std::cout << substitute_tokens (text, name, *expansion)
		  << std::endl;
    }
  else
    {
      std::cout << text
		<< std::endl;
    }
}



// process the list of instantiations given in the form
// --------------------
// for (u,v:VECTORS; z:SCALARS) { f(u, z, const v &); }
// --------------------
void process_instantiations ()
{
  std::string whole_file = read_whole_file (std::cin);

				   // process entries of the form
				   //  for (X:Y; A:B) { INST }
  while (whole_file.size() != 0)
    {
      skip_space (whole_file);
      if (whole_file.find ("for") != 0)
	{
	  std::cerr << "Invalid instantiation list" << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 3);
      skip_space (whole_file);
      if (whole_file.find ("(") != 0)
	{
	  std::cerr << "Invalid instantiation list" << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 1);
      skip_space (whole_file);

      const std::list<std::string>
	substitutions_list
	= split_string_list (get_substring_with_delim (whole_file,
								     ")"),
					   ';');
      if (whole_file.find (")") != 0)
	{
	  std::cerr << "Invalid instantiation list" << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 1);
      skip_space (whole_file);

				       // process the header
      std::list<std::pair<std::string, std::string> >
	substitutions;
      
      for (std::list<std::string>::const_iterator
	     s = substitutions_list.begin();
	   s != substitutions_list.end(); ++s)
	{
	  const std::list<std::string>
	    names_and_type = split_string_list (*s, ':');
	  if (names_and_type.size() != 2)
	    {
	      std::cerr << "Invalid instantiation header" << std::endl;
	      std::exit (1);
	    }
	  
	  const std::list<std::string>
	    names = split_string_list (names_and_type.front(), ',');

	  for (std::list<std::string>::const_iterator
		 x = names.begin(); x != names.end(); ++x)
	    substitutions.push_back (std::make_pair (*x,
						     names_and_type.back()));
	}

				       // now read the part in {...}
      skip_space (whole_file);
      if (whole_file.find ("{") != 0)
	{
	  std::cerr << "Invalid substitution text" << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 1);
      skip_space (whole_file);
      const std::string text_to_substitute
	= get_substring_with_delim (whole_file, "}");
      whole_file.erase (0,1);
      skip_space (whole_file);

				       // now produce the substitutions
      substitute (text_to_substitute, substitutions);
    }
}



int main (int argc, char **argv)
{
  if (argc < 2)
    {
      std::cerr << "Usage: " << std::endl
		<< "  expand_instantiations class_list_files < in_file > out_file"
		<< std::endl;
      std::exit (1);
    }

  for (int i=1; i<argc; ++i)
    read_expansion_lists (argv[i]);

  process_instantiations ();
}
<commit_msg>Surround substituted tokens by spaces to avoid the problem with '>>' ending template arguments.<commit_after>//----------------------------  expand_instantiations.cc  -------------------
//    $Id$
//    Version: $Name$
//
//    Copyright (C) 2007 by the deal.II authors
//
//    This file is subject to QPL and may not be  distributed
//    without copyright and license information. Please refer
//    to the file deal.II/doc/license.html for the  text  and
//    further information on this license.
//
//----------------------------  expand_instantiations.cc  -------------------

// This is the program that we use to generate explicit instantiations for a
// variety of template arguments. It takes two kinds of input files. The first
// is given as arguments on the command line and contains entries of the
// following form:
// --------------------
// REAL_SCALARS    := { double; float; long double }
// COMPLEX_SCALARS := { std::complex<double>;
//                      std::complex<float>;
//                      std::complex<long double> }
// VECTORS := { Vector<double>; Vector<float>; Vector<long double> }
// --------------------
//
// The input file of this form is typically located in the common/ directory
// and is generated by ./configure to contain the list of vectors etc that
// make sense for the current configuration. For example, the list of VECTORS
// is going to contain PETSc vectors if so configured.
//
// The second intput is read from the command line and consists of a sequence
// of statements of the following form:
// --------------------
// for (u,v:VECTORS; z:SCALARS) { f(u, z, const v &); }
// --------------------
// Here, everything between {...} will be copied as many times as there are
// combinations of arguments u,v in the list of substitutions given by
// VECTORS. For each copy, the arguments u,v will be replaced by one of these
// combinations.

// Author: <NAME>, 2007


#include <iostream>
#include <fstream>
#include <map>
#include <list>


// a map from the keys in the expansion lists to the list itself. For
// instance, the example above will lead to the entry
//      expansion_lists[REAL_SCALARS] = (double, float, long double)
// in this map, among others
std::map<std::string, std::list<std::string> >  expansion_lists;



/* ======================== auxiliary functions ================= */

// extract from the start of #in the part of the string that ends with one of
// the characters in #delim_list. The extracted part is deleted from #in
std::string
get_substring_with_delim (std::string       &in,
			  const std::string &delim_list)
{
  std::string x;
  while ((in.size() != 0)
	 &&
	 (delim_list.find (in[0]) == std::string::npos))
    {
      x += in[0];
      in.erase (0, 1);
    }

  return x;
}


// delete all whitespace at the beginning of the given argument
void
skip_space (std::string &in)
{
  while ((in.size() != 0)
	 &&
	 ((in[0] == ' ') || (in[0] == '\t') || (in[0] == '\n')))
    in.erase (0, 1);
}



// read the whole file specified by the stream given as argument into a string
// for simpler parsing, and return it
std::string read_whole_file (std::istream &in)
{
  std::string whole_file;
  while (in)
    {
      std::string line;
      getline (in, line);

      if (line.find ("//") != 0)
	{
	  whole_file += line;
	  whole_file += '\n';
	}
    }
				   // substitute tabs by spaces, multiple
				   // spaces by single ones
  for (unsigned int i=0; i<whole_file.size(); ++i)
    if (whole_file[i] == '\t')
      whole_file[i] = ' ';
  while (whole_file.find("  ") != std::string::npos)
    whole_file.replace (whole_file.find("  "), 2, " ");

  return whole_file;
}



// split a given string assumed to consist of a list of substrings delimited
// by a particular character into its components
std::list<std::string>
split_string_list (const std::string &s,
		   const char         delimiter)
{
  std::string tmp = s;
  std::list<std::string> split_list;

				   // split the input list
  while (tmp.length() != 0)
    {
      std::string name;
      name = tmp;

      if (name.find(delimiter) != std::string::npos)
	{
	  name.erase (name.find(delimiter), std::string::npos);
	  tmp.erase (0, tmp.find(delimiter)+1);
	}
      else
	tmp = "";

      skip_space (name);

      while (name[name.length()-1] == ' ')
	name.erase (name.length()-1, 1);

      split_list.push_back (name);
    }

  return split_list;
}



// determine whether a given substring at position #pos and length #length in
// the string #text is a real token, i.e. not just part of another word
bool is_real_token (const std::string &text,
		    const std::string::size_type pos,
		    const std::string::size_type length)
{
  static const std::string token_chars ("abcdefghijklmnopqrstuvwxyz"
					"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
					"0123456789"
					"_");
  if ((pos != 0) && (token_chars.find (text[pos-1]) != std::string::npos))
    return false;

  if ((pos+length < text.size()) &&
      (token_chars.find (text[pos+length]) != std::string::npos))
    return false;

  return true;
}


// substitute all occurrences of #token in #text by #substitute. because a
// replacement token could be a templated class like std::complex<double> and
// because the token to the substituted may be a template argument itself, we
// surround the substitution by a space which shouldn't matter in C++
std::string substitute_tokens (const std::string &text,
			       const std::string &token,
			       const std::string &substitute)
{
  std::string x_text = text;
  std::string::size_type pos = 0;
  while ((pos = x_text.find(token, pos)) != std::string::npos)
    {
      if (is_real_token (x_text, pos, token.size()))
	{  
	  x_text.replace (pos, token.size(),
			  std::string(" ")+substitute+std::string(" "));
	  pos += substitute.size()+2;
	}
      else
	++pos;
    }
  
  return x_text;
}



/* ======================== the main functions ================= */


// read and parse the expansion lists like
//     REAL_SCALARS    := { double; float; long double }
// as specified at the top of the file and store them in
// the global expansion_lists variable
void read_expansion_lists (const std::string &filename)
{
  std::ifstream in (filename.c_str());

  if (! in)
    {
      std::cerr << "Instantiation list file can not be read!"
		<< std::endl;
      std::exit (1);
    }

				   // read the entire file into a string for
				   // simpler processing. replace end-of-line
				   // characters by spaces
  std::string whole_file = read_whole_file (in);

				   // now process entries of the form
				   // NAME := { class1; class2; ...}.
  while (whole_file.size() != 0)
    {
      const std::string
	name = get_substring_with_delim (whole_file, " :");

      skip_space (whole_file);
      if (whole_file.find (":=") != 0)
	{
	  std::cerr << "Invalid entry <" << name << '>' << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 2);
      skip_space (whole_file);
      if (whole_file.find ("{") != 0)
	{
	  std::cerr << "Invalid entry <" << name << '>' << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 1);
      skip_space (whole_file);

      std::string
	expansion = get_substring_with_delim (whole_file, "}");

      if (whole_file.find ("}") != 0)
	{
	  std::cerr << "Invalid entry <" << name << '>' << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 1);
      skip_space (whole_file);

      expansion_lists[name] = split_string_list (expansion, ';');
    }
}



// produce all combinations of substitutions of the tokens given in the
// #substitutions list in #text and output it to std::cout
void substitute (const std::string &text,
		 const std::list<std::pair<std::string, std::string> > &substitutions)
{
				   // do things recursively: if the list of
				   // substiutions has a single entry, then
				   // process all of them. otherwise, process
				   // the first in the list and call the
				   // function recursively with the rest of
				   // the substitutions
  if (substitutions.size() > 1)
    {
				       // do the first substitution, then call
				       // function recursively
      const std::string name    = substitutions.front().first,
			pattern = substitutions.front().second;

      const std::list<std::pair<std::string, std::string> >
	rest_of_substitutions (++substitutions.begin(),
			       substitutions.end());

      for (std::list<std::string>::const_iterator
	     expansion = expansion_lists[pattern].begin();
	   expansion != expansion_lists[pattern].end();
	   ++expansion)
	{
	  std::string new_text
	    = substitute_tokens (text, name, *expansion);

	  substitute (new_text, rest_of_substitutions);
	}
    }
  else if (substitutions.size() == 1)
    {
				       // do the substitutions
      const std::string name    = substitutions.front().first,
			pattern = substitutions.front().second;
      
      for (std::list<std::string>::const_iterator
	     expansion = expansion_lists[pattern].begin();
	   expansion != expansion_lists[pattern].end();
	   ++expansion)
	std::cout << substitute_tokens (text, name, *expansion)
		  << std::endl;
    }
  else
    {
      std::cout << text
		<< std::endl;
    }
}



// process the list of instantiations given in the form
// --------------------
// for (u,v:VECTORS; z:SCALARS) { f(u, z, const v &); }
// --------------------
void process_instantiations ()
{
  std::string whole_file = read_whole_file (std::cin);

				   // process entries of the form
				   //  for (X:Y; A:B) { INST }
  while (whole_file.size() != 0)
    {
      skip_space (whole_file);
      if (whole_file.find ("for") != 0)
	{
	  std::cerr << "Invalid instantiation list" << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 3);
      skip_space (whole_file);
      if (whole_file.find ("(") != 0)
	{
	  std::cerr << "Invalid instantiation list" << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 1);
      skip_space (whole_file);

      const std::list<std::string>
	substitutions_list
	= split_string_list (get_substring_with_delim (whole_file,
								     ")"),
					   ';');
      if (whole_file.find (")") != 0)
	{
	  std::cerr << "Invalid instantiation list" << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 1);
      skip_space (whole_file);

				       // process the header
      std::list<std::pair<std::string, std::string> >
	substitutions;
      
      for (std::list<std::string>::const_iterator
	     s = substitutions_list.begin();
	   s != substitutions_list.end(); ++s)
	{
	  const std::list<std::string>
	    names_and_type = split_string_list (*s, ':');
	  if (names_and_type.size() != 2)
	    {
	      std::cerr << "Invalid instantiation header" << std::endl;
	      std::exit (1);
	    }
	  
	  const std::list<std::string>
	    names = split_string_list (names_and_type.front(), ',');

	  for (std::list<std::string>::const_iterator
		 x = names.begin(); x != names.end(); ++x)
	    substitutions.push_back (std::make_pair (*x,
						     names_and_type.back()));
	}

				       // now read the part in {...}
      skip_space (whole_file);
      if (whole_file.find ("{") != 0)
	{
	  std::cerr << "Invalid substitution text" << std::endl;
	  std::exit (1);
	}
      whole_file.erase (0, 1);
      skip_space (whole_file);
      const std::string text_to_substitute
	= get_substring_with_delim (whole_file, "}");
      whole_file.erase (0,1);
      skip_space (whole_file);

				       // now produce the substitutions
      substitute (text_to_substitute, substitutions);
    }
}



int main (int argc, char **argv)
{
  if (argc < 2)
    {
      std::cerr << "Usage: " << std::endl
		<< "  expand_instantiations class_list_files < in_file > out_file"
		<< std::endl;
      std::exit (1);
    }

  for (int i=1; i<argc; ++i)
    read_expansion_lists (argv[i]);

  process_instantiations ();
}

{'fraction_non_alphanumeric': 0.1681108442004118, 'fraction_numerical': 0.007163692518874399, 'mean_word_length': 2.6817751105495895, 'pattern_counts': {'":': 2, '<': 204, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6532802', 'n_tokens_mistral': 8085, 'n_tokens_neox': 7315, 'n_words': 2145}