<commit_before>#include "headers/dfa.h"
#include <stdio.h>
#define FILEPATH "config/dfa_lexer_description"
#define BUFFERLEN 200
#define ERRORS 100

// TODO (Anant)  : Read Integer and Float literals as Strings itself
// TODO (Anant)  : Change lexer to read from file given as a command line argument
// TODO (Anant)  : Read from file using two buffers
// TODO (Anant)  : Output literals/identifiers to symbol file and output pointer numbers

int main()
{
  DFA *dfa;
  dfa = getNewDFA();
  dfa = initializeFromFile ( dfa, FILEPATH );
  int shouldread = TRUE;
  char c;
  char stringliteral[ BUFFERLEN ];
  char identifier[ BUFFERLEN ];
  int idenindex = 0;
  int stringlitindex = 0;
  int intliteral = 0;
  float floatliteral = 0.0, floatliteral2 = 0.0;
  int decimalcount = 1;
  int started = FALSE;
  int linenumber=1;
  int error = 0;
  FILE *f,*p;
  f = fopen ( "Errors.txt","w+");
  p = fopen ( "Tokens.txt","w+");
  while ( TRUE )
  {
    c = getchar();
    if ( c == EOF )
      break;
    if ( c == 10)
      linenumber++;

    if ( peek ( dfa, c ) == NULL || getSpecialProperty ( peek ( dfa, c ) ) == TRAP )
    {
      if ( isFinal ( getCurrentState (dfa) ) == TRUE )
      {
              
        if ( getSpecialProperty ( getCurrentState (dfa) ) == ERROR )
        {
          fprintf(f,"%d : %s\n",linenumber,getCurrentState (dfa) -> name);
          fprintf(p,"<%s>\n",getCurrentState (dfa) -> name);    // Just to match with Unit-testing for the time being
          error++;
          if ( error >= ERRORS )
            break;
        }

        else if ( strcmp ( getCurrentState (dfa) -> name , "TK_INTLIT" ) == 0 )
          fprintf (p,"<TK_INTLIT,%d>\n", intliteral );
        else if ( strcmp ( getCurrentState (dfa) -> name , "TK_FLOATLIT" ) == 0 )
          fprintf (p,"<TK_FLOATLIT,%f>\n", floatliteral + floatliteral2 );
        else if ( strcmp ( getCurrentState (dfa) -> name , "TK_STRINGLIT" ) == 0 )
          fprintf (p,"<TK_STRINGLIT,%s>\n", stringliteral );
        else if ( strcmp ( getCurrentState (dfa) -> name , "TK_IDEN" ) == 0 )
          fprintf (p,"<TK_IDEN,%s>\n", identifier );
        else
          fprintf (p,"<%s>\n", getCurrentState (dfa) -> name );
      }

      if ( peek ( dfa, c ) == NULL )
        dfa = gotoInitialState ( dfa );
    }

    gotoNextState ( dfa, c );

    if ( getCurrentState (dfa) != NULL && getCurrentState (dfa) -> name != NULL )
    {
      if ( strcmp ( getCurrentState (dfa) -> name , "Decimal Point" ) == 0 )
      {
        floatliteral = (float) intliteral;
        decimalcount = 1;
        floatliteral2 = 0;
      }
     
      else if ( strcmp ( getCurrentState (dfa) -> name, "TK_FLOATLIT" ) == 0 )
      {
        decimalcount *= 10;
        floatliteral2 = floatliteral2 + ((c - 48) / (float) decimalcount);
      }
      else
        floatliteral = 0;

      if ( strcmp ( getCurrentState (dfa) -> name , "TK_INTLIT" ) == 0 )
        intliteral = intliteral * 10 + c - 48;
      else
        intliteral = 0;

      if ( strcmp ( getCurrentState (dfa) -> name , "Started Quote" ) == 0 )
      {
        stringliteral [ stringlitindex ++ ] = c;
        stringliteral [ stringlitindex ] = '\0';
      }
      else
      {
        stringliteral [ stringlitindex ] = '"';
        stringliteral [ stringlitindex + 1 ] = '\0';
        stringlitindex = 0;
      }

      if ( strcmp ( getCurrentState (dfa) -> name , "TK_IDEN" ) == 0 )
      {
        started = TRUE;
        identifier [ idenindex ++ ] = c;
        identifier [ idenindex ] = '\0';
      }
      else if ( ( isalpha( c ) || isdigit( c )  || c == '_' ) && started )
      {
        identifier [ idenindex ++ ] = c;
        identifier [ idenindex ] = '\0';
      }
      else
      {
        started = FALSE;
        idenindex = 0;
      }
    }
    else
    {
      started = FALSE;
      idenindex = 0;
    }

  }

  fclose(f);
  fclose(p);

  if ( error == 0 )
  {
      p = fopen("Tokens.txt","r");
      char c;
      while ( !feof(p) )
      {
        fscanf(p,"%c",&c);
        fprintf(stdout,"%c",c);  
      }
      fclose(p);
  }
  return 0;
}<commit_msg>Reading Int and Float literals as Strings. nit: Coding style<commit_after>#include "headers/dfa.h"
#include <stdio.h>

#define FILEPATH "config/dfa_lexer_description"
#define BUFFERLEN 200
#define ERRORS 100
#define NEWLINE '\n'

// TODO (Anant)  : Change lexer to read from file given as a command line argument
// TODO (Anant)  : Read from file using two buffers
// TODO (Anant)  : Output literals/identifiers to symbol file and output pointer numbers

int main()
{
  DFA *dfa;
  dfa = getNewDFA();
  dfa = initializeFromFile ( dfa, FILEPATH );

  int shouldread = TRUE;
  int started = FALSE;

  char c;
  char stringliteral [ BUFFERLEN ];
  char identifier [ BUFFERLEN ];
  char floatorint [ BUFFERLEN ];

  int idenindex = 0;
  int stringlitindex = 0;
  int floatintindex = 0;

  int linenumber = 1;
  int errorcount = 0;

  FILE *errorsfile, *tokensfile;

  errorsfile = fopen ( "Errors.txt", "w+" );
  tokensfile = fopen ( "Tokens.txt", "w+" );

  if ( errorsfile == NULL || tokensfile == NULL )
  {
    fprintf ( stderr, "Failed to open errors / tokens file\n" );
    return -1;
  }

  while ( TRUE )
  {
    c = getchar();

    if ( c == EOF )
      break;

    if ( c == 10)
      linenumber++;

    if ( peek ( dfa, c ) == NULL || getSpecialProperty ( peek ( dfa, c ) ) == TRAP )
    {
      // Token over / error encountered so should process intermediate result
      if ( isFinal ( getCurrentState (dfa) ) == TRUE )
      {
        if ( getSpecialProperty ( getCurrentState (dfa) ) == ERROR )
        {
          fprintf ( errorsfile, "%d : %s\n", linenumber, getCurrentState (dfa) -> name );
          // Just to match with Unit-testing for the time being
          fprintf ( tokensfile, "<%s>\n", getCurrentState (dfa) -> name );
          errorcount++;
          if ( errorcount >= ERRORS )
            break;
        }
        else if ( strcmp ( getCurrentState (dfa) -> name , "TK_INTLIT" ) == 0 )
          fprintf ( tokensfile, "<TK_INTLIT,%s>\n", floatorint );
        else if ( strcmp ( getCurrentState (dfa) -> name , "TK_FLOATLIT" ) == 0 )
          fprintf ( tokensfile, "<TK_FLOATLIT,%s>\n", floatorint );
        else if ( strcmp ( getCurrentState (dfa) -> name , "TK_STRINGLIT" ) == 0 )
          fprintf ( tokensfile, "<TK_STRINGLIT,%s>\n", stringliteral );
        else if ( strcmp ( getCurrentState (dfa) -> name , "TK_IDEN" ) == 0 )
          fprintf ( tokensfile, "<TK_IDEN,%s>\n", identifier );
        else
          fprintf ( tokensfile, "<%s>\n", getCurrentState (dfa) -> name );
      }

      // If there is no further transition, start from the initial state again
      if ( peek ( dfa, c ) == NULL )
        dfa = gotoInitialState ( dfa );
    }

    gotoNextState ( dfa, c );

    if ( getCurrentState (dfa) != NULL && getCurrentState (dfa) -> name != NULL )
    {
      if ( strcmp ( getCurrentState (dfa) -> name , "Decimal Point" ) == 0 )
      {
        floatorint [ floatintindex++ ] = '.';
      }
      else if ( strcmp ( getCurrentState (dfa) -> name, "TK_FLOATLIT" ) == 0 )
      {
        floatorint [ floatintindex++ ] = c;
        floatorint [ floatintindex ] = '\0';
      }
      else if ( strcmp ( getCurrentState (dfa) -> name , "TK_INTLIT" ) == 0 )
      {
        floatorint [ floatintindex++ ] = c;
        floatorint [ floatintindex ] = '\0';
      }
      else
      {
        floatorint [ floatintindex ] = '\0';
        floatintindex = 0;
      }

      if ( strcmp ( getCurrentState (dfa) -> name , "Started Quote" ) == 0 )
      {
        stringliteral [ stringlitindex ++ ] = c;
        stringliteral [ stringlitindex ] = '\0';
      }
      else
      {
        stringliteral [ stringlitindex ] = '"';
        stringliteral [ stringlitindex + 1 ] = '\0';
        stringlitindex = 0;
      }

      if ( strcmp ( getCurrentState (dfa) -> name , "TK_IDEN" ) == 0 )
      {
        started = TRUE;
        identifier [ idenindex ++ ] = c;
        identifier [ idenindex ] = '\0';
      }
      else if ( ( isalpha( c ) || isdigit( c )  || c == '_' ) && started )
      {
        identifier [ idenindex ++ ] = c;
        identifier [ idenindex ] = '\0';
      }
      else
      {
        started = FALSE;
        idenindex = 0;
      }
    }
    else
    {
      started = FALSE;
      idenindex = 0;
    }

  }

  if ( fclose ( errorsfile ) != 0 )
    fprintf ( stderr, "Error while closing lexical error file\n" );
  if ( fclose ( tokensfile ) != 0 )
    fprintf ( stderr, "Error while closing lexical tokens file\n" );

  if ( errorcount == 0 )
  {
      tokensfile = fopen ( "Tokens.txt", "r" );
      if ( tokensfile == NULL )
      {
        fprintf ( stderr, "Failed to open tokens file to read the second time\n" );
        return -1;
      }

      while ( ! feof ( tokensfile ) )
      {
        fscanf ( tokensfile, "%c", &c );
        printf ( "%c", c );
      }

      if ( fclose ( tokensfile ) != 0 )
        fprintf ( stderr, "Failed to close tokens file while re-reading\n" );
  }
  return 0;
}
{'fraction_non_alphanumeric': 0.1416630004399472, 'fraction_numerical': 0.010448746150461945, 'mean_word_length': 2.0129224652087476, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1139668', 'n_tokens_mistral': 3125, 'n_tokens_neox': 2991, 'n_words': 820}