<commit_before>
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <cstring>
#include <string>
#include <map>

using namespace std;

typedef map<string,string> CmdEntries;


CmdEntries commands;
CmdEntries files;

int fileline=0;
void GetBf(char* buf, int size, FILE*f)
{
  if (!fgets(buf,2048,f))
  {
    *buf = '\0';
    if (!feof(f))
    {
      printf("Error reading, bailing out.\n");
      exit(-1);
    }
  }

  fileline++;
  const std::size_t n = std::strlen(buf);
  if (n && buf[n - 1] == '\n')
      buf[n] = '\0';
  char* ptr=buf;
  while (*ptr != '\0')
  {
    if (*ptr == ':') *ptr = ' ';
    ptr++;
  }
}

void ProcessFile(char* fname)
{
  fprintf(stderr,"\tFILE %s\n",fname);
  FILE*f = fopen(fname,"r");
  if (!f) return;
  int prevline = fileline;
  fileline = 0;
  char buf[2048];
  while (!feof(f))
  {
    GetBf(buf,2048,f);
    const std::size_t n = std::strlen(buf);
    if (n && buf[n - 1] == '\n')
        buf[n - 1] = '\0';

    //*CMD Sin, Cos, Tan --- trigonometric functions
    if (!strncmp(buf,"*CMD",4))
    {
      char* description=buf;
      while (*description != '\0' && *description != '-') description++;
      if (*description == '\0') continue;
      while (*description == '-') description++;
      while (*description == ' ') description++;
      
      char* command = &buf[5];
      for(;;)
      {
        char* end = command;
        while (*end != ',' && *end != ' ') end++;
        char cmd[256];
        memset(cmd,0,255);
        memcpy(cmd,command,end-command);
//        cmd[end-command] = '\0';

        commands[cmd] = description;
        files[cmd] = fname;

//printf("COMMAND [%s], description [%s]\n",cmd,description);
        if (*end == ' ') break;
        end++;
        if (*end == ' ') end++;
        command = end;
      }
    }
    else if (!strncmp(buf,"*CALL",5))
    {
      while (buf[0] != '\t')
        GetBf(buf,2048,f);
      char cmd[256];
      while (buf[0] == '\t')
      {
        if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = '\0';
        char* ptr = &buf[1];
        while (isalnum(*ptr) || *ptr == '\'') ptr++;
        memset(cmd,0,255);
        memcpy(cmd,&buf[1],ptr-(&buf[1]));
        
        if (cmd[0] && strncmp(&buf[1],"In>",3) && strncmp(&buf[1],"Out>",4) && isupper(buf[1]))
        {
          CmdEntries::const_iterator iter = commands.find(cmd);
          const char* desc = "";
          if (iter != commands.end())
          {
            desc = iter->second.c_str();
          }
          else
          {
            fprintf(stderr,"WARNING: function %s does not have a short description (%s:%d)\n",cmd,fname,fileline);
          }
          printf(":%s:%s:%s:\n",cmd,&buf[1],desc);
          commands[cmd] = "";
        }
        GetBf(buf,2048,f);
        while (buf[0] == '\0') GetBf(buf,2048,f);
      }
      if (buf[0] == ' ')
      {
        fprintf(stderr,"WARNING: spaces might have been used in stead of tabs (%s, %s:%d)\n",cmd,fname,fileline);
      }
    }
    else if (!strncmp(buf,"*INCLUDE",8))
    {
      char*fn = &buf[8];
      while (*fn == ' ') fn++;
      char fullname[512];
      snprintf(fullname,512,"%s.txt",fn);
//DEBUG printf("FILE %s\n",fullname);
      ProcessFile(fullname);

    }
  }
  fclose(f);
  fileline = prevline;
}

int main(int argc, char** argv)
{
  int i;
  for (i=1;i<argc;i++)
  {
    fprintf(stderr,"BOOK %s\n",argv[i]);
    ProcessFile(argv[i]);
  }
  CmdEntries::const_iterator iter = commands.begin();
  while (iter != commands.end())
  {
    if (iter->second.c_str()[0])
    {
      CmdEntries::const_iterator fiter = files.find(iter->first.c_str());
      if (fiter != files.end())
        fprintf(stderr,"WARNING: no call sequences for function %s (%s)\n",iter->first.c_str(),fiter->second.c_str());
      else
        fprintf(stderr,"WARNING: no call sequences for function %s\n",iter->first.c_str());
    }
    iter++;
  }


}
<commit_msg>another tiny step toward real c++ codebase<commit_after>#include <stdlib.h>
#include <cctype>
#include <cstring>
#include <iostream>
#include <map>
#include <string>

typedef std::map<std::string, std::string> CmdEntries;


CmdEntries commands;
CmdEntries files;

int fileline = 0;

void GetBf(char* buf, int size, FILE*f)
{
    if (!fgets(buf, 2048, f)) {
        *buf = '\0';
        if (!feof(f)) {
            std::cerr << "Error reading, bailing out.\n";
            exit(-1);
        }
    }

    fileline++;
    const std::size_t n = std::strlen(buf);
    if (n && buf[n - 1] == '\n')
        buf[n] = '\0';
    char* ptr = buf;
    while (*ptr != '\0') {
        if (*ptr == ':') *ptr = ' ';
        ptr++;
    }
}

void ProcessFile(const char* fname)
{
    std::cerr << "\tFILE " << fname << "\n";
    FILE*f = fopen(fname, "r");
    if (!f) return;
    int prevline = fileline;
    fileline = 0;
    char buf[2048];
    while (!feof(f)) {
        GetBf(buf, 2048, f);
        const std::size_t n = std::strlen(buf);
        if (n && buf[n - 1] == '\n')
            buf[n - 1] = '\0';

        //*CMD Sin, Cos, Tan --- trigonometric functions
        if (!strncmp(buf, "*CMD", 4)) {
            char* description = buf;
            while (*description != '\0' && *description != '-') description++;
            if (*description == '\0') continue;
            while (*description == '-') description++;
            while (*description == ' ') description++;

            char* command = &buf[5];
            for (;;) {
                char* end = command;
                while (*end != ',' && *end != ' ') end++;
                char cmd[256];
                memset(cmd, 0, 255);
                memcpy(cmd, command, end - command);
                //        cmd[end-command] = '\0';

                commands[cmd] = description;
                files[cmd] = fname;

                if (*end == ' ') break;
                end++;
                if (*end == ' ') end++;
                command = end;
            }
        } else if (!strncmp(buf, "*CALL", 5)) {
            while (buf[0] != '\t')
                GetBf(buf, 2048, f);
            char cmd[256];
            while (buf[0] == '\t') {
                if (buf[strlen(buf) - 1] == '\n') buf[strlen(buf) - 1] = '\0';
                char* ptr = &buf[1];
                while (std::isalnum(*ptr) || *ptr == '\'') ptr++;
                memset(cmd, 0, 255);
                memcpy(cmd, &buf[1], ptr - (&buf[1]));

                if (cmd[0] && strncmp(&buf[1], "In>", 3) && strncmp(&buf[1], "Out>", 4) && isupper(buf[1])) {
                    CmdEntries::const_iterator iter = commands.find(cmd);
                    const char* desc = "";
                    if (iter != commands.end())
                        desc = iter->second.c_str();
                    else
                        std::cerr << "WARNING: function " << cmd << " does not have a short description (" << fname << ":" << fileline << ")\n";

                    std::cout << ":" << cmd << ":" << &buf[1] << ":" << desc << ":\n";
                    commands[cmd] = "";
                }
                GetBf(buf, 2048, f);
                while (buf[0] == '\0') GetBf(buf, 2048, f);
            }
            if (buf[0] == ' ')
                std::cerr << "WARNING: spaces might have been used instead of tabs (" << cmd << ", " << fname << ":" << fileline << ")\n";
        } else if (!strncmp(buf, "*INCLUDE", 8)) {
            const char* fn = &buf[8];
            while (*fn == ' ')
                ++fn;
            std::string fullname = fn;
            fullname += ".txt";
            ProcessFile(fullname.c_str());

        }
    }
    fclose(f);
    fileline = prevline;
}

int main(int argc, char** argv)
{
    for (int i = 1; i < argc; ++i) {
        std::cerr << "BOOK " << argv[i] << "\n";
        ProcessFile(argv[i]);
    }

    for (CmdEntries::const_iterator iter = commands.begin(); iter != commands.end(); ++iter) {
        if (iter->second.c_str()[0]) {
            CmdEntries::const_iterator fiter = files.find(iter->first.c_str());
            if (fiter != files.end())
                std::cerr << "WARNING: no call sequences for function " << iter->first << " (" << fiter->second << ")\n";
            else
                std::cerr << "WARNING: no call sequences for function " << iter->first << "\n";
        }
    }
}

{'fraction_non_alphanumeric': 0.2023121387283237, 'fraction_numerical': 0.019026974951830443, 'mean_word_length': 1.8737024221453287, 'pattern_counts': {'":': 7, '<': 91, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6679664', 'n_tokens_mistral': 2909, 'n_tokens_neox': 2780, 'n_words': 636}