<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "debug.h"
#include "tokenize.h"
#include "grammar.h"

#define NB_MATCH_MAX    10
#define NB_REGEX_MAX    30

  /*---------------------------------------------------------------*/
  /* FTA1: modulariser le code : regex, communication avec REXX    */
  /*       reduction du string , creation de la liste des tokens   */
  /*---------------------------------------------------------------*/
  /* la reduction du string enleve le separteur de debut           */
  /* (espace, virgule mais pas ' pour string )                     */
  /*---------------------------------------------------------------*/
  /* L'ordre de check des REgex semblables est :                   */
  /* -Keyword avant identifier, c ce qui en fait des mots reserves */
  /* -Dernier regex est : Espace puis enfin Erreur.                */
  /* -hexa literal avant identifier                                */
  /* exemple : x'ff'=> hex literal                                 */
  /* exemple : A'ff'=> identifier + string literal                 */
  /* -Choisir le match le plus long                                */
  /* exemple : **   => exponent au lieu de multiply                */
  /*           2,2  => decimal au lieu de integer                  */
  /*           a-b  => identifier a au lieu de a - b               */
  /*           a+b  => a + b                                       */
  /*---------------------------------------------------------------*/


token tokenizer(char* ln_8_72)
{
    token _ret = {0,0,0,"ERROR","ERROR"};

    static int reduction_len;
    /****/ int regcomp_type_done=0;
    /****/ int regcomp_attr_done=0;

    /* Regex for token type         */
    /****/ regex_t    pregIdent;
    /****/ regex_t    pregPicChars;
    /****/ regex_t    pregLiterNum;
    /****/ regex_t    pregLiterQuot;
    /****/ regex_t    pregKeywoAll;
    /****/ regex_t    pregSymbols;
    /****/ regex_t    pregEndLine;
    /****/ regex_t    pregSpace;

    /* Regex for token attributes   */
    /****/ regex_t    pregSpclReg;
    /****/ regex_t    pregFgrtvConst;
    /****/ regex_t    pregNumeric;
    /****/ regex_t    pregQuoted;
    /****/ regex_t    pregSQuoted;
    /****/ regex_t    pregDQuoted;
    /****/ regex_t    pregHex;
    /****/ regex_t    pregAlphanum;
    /****/ regex_t    pregIntgr;
    /****/ regex_t    pregDecim;
    /****/ regex_t    pregUnsigned;
    /****/ regex_t    pregPositive;
    /****/ regex_t    pregNegative;
    /****/ regex_t    pregLvlnum;
    /****/ regex_t    pregRelOp;
    /****/ regex_t    pregAritOp;

 /* regex_t    pregLiter1;
    regex_t    pregLiter2;
    regex_t    pregPerio;
    regex_t    pregLvl66;
    regex_t    pregLvl88;
    regex_t    pregHexL1;
    regex_t    pregHexL2;
    regex_t    pregLBrack;
    regex_t    pregRBrack;
    regex_t    pregPlus;
    regex_t    pregMinus;
    regex_t    pregExpon;
    regex_t    pregMultp;
    regex_t    pregDivid;
    regex_t    pregEqual;
    regex_t    pregColon;
    regex_t    pregDollar;
    regex_t    pregUintgr; */

  /*    _____________________________________________________
      _(                                                     )_
     =_                I D E N T I F I E R                    _=
       (_____________________________________________________)
                                                                  */

  /*----------------------------*/
  /* Pattern pour indentifier   */
  /*--------------------------------------------------------------*/
  /* - S{parateur d{but: aucan ou plusieurs espaces ou virgules   */
  /* - Corps : Un mot cobol compos{ de lettres et chiffres        */
  /*   ainsi que tiret et undersoce qui doivent pas apparaitre    */
  /*   au debut ou  a fin. doit contenir au moins une lettre     */
  /* - S{parateur fin  : point, espace, virgule, fin, quote ou "  */
  /*--------------------------------------------------------------*/

    /****/ char *ptrnIdent = "^( !,)*"                                \
   "(([0-9]+[-_]*)*[0-9]*[A-Za-z][A-Za-z0-9]*([-_]+[A-Za-z0-9]+)*)"\
                          "(.*)";

  /*    _____________________________________________________
      _(                                                     )_
     =_            P I C T U R E  C H A R A C T E R           _=
       (_____________________________________________________)
                                                                  */

  /*-------------------------------------------*/
  /* FTA1: ponctuation                         */
  /* FTA2: CR et DB                            */
  /*--------------------------------------------------------------*/
  /* - picture-string =  currency? (picchar+ repeat?)+            */
  /*   currency       =  $                                        */
  /*   picchar        =  [ABEGPSVXZabegpsvxz90\+\-\*\$]           */
  /*   repeat         =  "(" [0-9]+ ")"                           */
  /*   mannque plus que CR et DB pour les pic string              */
  /*--------------------------------------------------------------*/

  /*char     *ptrnPicChars = "(^"                              \
                   "([ABEGPSVXZabegpsvxz90\\+\\-\\*\\$\\/\\,\\.]+))"\
  */          /*   "(( !,)*!\\()(.*)"; */                  \
              /*   "(( !,)+!\\.!\\()(.*)"; */              \
  /*               "(( !,!$)+\\.!\\.!\\()(.*)";
  */
  /*char     *ptrnPicChars = "(^"                              \ */
  /* A*  */ /*     "([ABEGPSVXZabegpsvxz90\\+\\-\\*\\$\\/\\,]*"\ */
  /* (A+ */ /*     "([ABEGPSVXZabegpsvxz90\\+\\-\\*\\$\\/\\,]+"\ */
  /* b)  */ /*     "!\\.)"                                     \ */
  /* A*  */ /*     "[ABEGPSVXZabegpsvxz90\\+\\-\\*\\$\\/\\,]*))"\ */
              /*   "(( !,)*!\\()(.*)"; */
              /*   "(( !,)+[a-zA-Z]+!( !,)*\\.!\\()(.*)"; */
            /*     "(( !,!$)+!\\.!\\()(.*)"; */


    /****/ char *ptrnPicChars = "(^"                              \
  /* A+  */  /*    "([ABEGPSVXZabegpsvxz90\\+\\-\\*\\$\\/\\,]+))" */\
                   "([-+*$/,ABEGPSVXZabegpsvxz90]+))" \
                   "(( !,!$)+!\\.!\\()(.*)";


  /*    _____________________________________________________
      _(                                                     )_
     =_             L I T E R A L  N U M E R I C              _=
       (_____________________________________________________)
                                                                  */

  /*--------------------------------------------------------------*/
  /* - S{parateur d{but: Zero ou plusieurs : espaces ou virgules  */
  /* - Corps : Commence par le separateur de string quote ou "    */
  /*   puis zero ou une suite de caract}res sauf separateur de    */
  /*   string ou un double separateur de string et finit par le   */
  /*   meme separateur de string                                  */
  /* - S{parateur fin  : point, espace, virgule, fin, quote ou "  */
  /*--------------------------------------------------------------*/
    /****/ char   *ptrnLiterNum = "^( !,)*("                      \
                           "([\\+\\-]?[0-9]*,[0-9]+)!"  /* -9,7 */\
                           "([\\+\\-]?[0-9]+)"         /* +987 */\
                      /*    ")([^']!$)(.*)"; */                   \<commit_msg>Update tokenize.c<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "debug.h"
#include "tokenize.h"
#include "grammar.h"

#define NB_MATCH_MAX    10
#define NB_REGEX_MAX    30

  /*----------------------*/
  /* Lexical tokenizing   */ 
  /*----------------------*/
  
  /*---------------------------------------------------------------*/
  /* REGEX Order                                                   */
  /*---------------------------------------------------------------*/
  /* 1  - Keyword                                                  */
  /* 10 - Special registers                                        */
  /* 101- Figurative constants                                     */
  /* 11 - Hexa Literal1 & Hexa Literal2                            */
  /* 2  - Identifier                                               */
  /* 3  - Literal1 & Literal2                                      */
  /* 31 - Decimal                                                  */
  /* 32 - Integer                                                  */
  /* 4  - Period                                                   */
  /*      & plus minus equal expon divid multp Lbrack Rbrack colon */
  /* 5  - EndLine                                                  */
  /* 6  - Error                                                    */
  /*---------------------------------------------------------------*/
  /* Choose the longest match   
  /*---------------------------------------------------------------*/
  /* example : **   => exponent of multiply multiply               */              
  /*           2,2  => decimal instead of integer                  */  
  /*           a-b  => identifier a instead of a - b               */ 
  /*           a+b  => a + b                                       */
  /*---------------------------------------------------------------*/



token tokenizer(char* ln_8_72)
{
    token _ret = {0,0,0,"ERROR","ERROR"};

    static int reduction_len;
    /****/ int regcomp_type_done=0;
    /****/ int regcomp_attr_done=0;

    /* Regex for token type         */
    /****/ regex_t    pregIdent;
    /****/ regex_t    pregPicChars;
    /****/ regex_t    pregLiterNum;
    /****/ regex_t    pregLiterQuot;
    /****/ regex_t    pregKeywoAll;
    /****/ regex_t    pregSymbols;
    /****/ regex_t    pregEndLine;
    /****/ regex_t    pregSpace;

    /* Regex for token attributes   */
    /****/ regex_t    pregSpclReg;
    /****/ regex_t    pregFgrtvConst;
    /****/ regex_t    pregNumeric;
    /****/ regex_t    pregQuoted;
    /****/ regex_t    pregSQuoted;
    /****/ regex_t    pregDQuoted;
    /****/ regex_t    pregHex;
    /****/ regex_t    pregAlphanum;
    /****/ regex_t    pregIntgr;
    /****/ regex_t    pregDecim;
    /****/ regex_t    pregUnsigned;
    /****/ regex_t    pregPositive;
    /****/ regex_t    pregNegative;
    /****/ regex_t    pregLvlnum;
    /****/ regex_t    pregRelOp;
    /****/ regex_t    pregAritOp;

 /* regex_t    pregLiter1;
    regex_t    pregLiter2;
    regex_t    pregPerio;
    regex_t    pregLvl66;
    regex_t    pregLvl88;
    regex_t    pregHexL1;
    regex_t    pregHexL2;
    regex_t    pregLBrack;
    regex_t    pregRBrack;
    regex_t    pregPlus;
    regex_t    pregMinus;
    regex_t    pregExpon;
    regex_t    pregMultp;
    regex_t    pregDivid;
    regex_t    pregEqual;
    regex_t    pregColon;
    regex_t    pregDollar;
    regex_t    pregUintgr; */

  /*    _____________________________________________________
      _(                                                     )_
     =_                I D E N T I F I E R                    _=
       (_____________________________________________________)
                                                                  */

  /*----------------------------*/
  /* Pattern pour indentifier   */
  /*--------------------------------------------------------------*/
  /* - Séparateur début: aucan ou plusieurs espaces ou virgules   */
  /* - Corps : Un mot cobol compos{ de lettres et chiffres        */
  /*   ainsi que tiret et undersoce qui doivent pas apparaitre    */
  /*   au debut ou à a fin. doit contenir au moins une lettre     */
  /* - S{parateur fin  : point, espace, virgule, fin, quote ou "  */
  /*--------------------------------------------------------------*/

    /****/ char *ptrnIdent = "^( !,)*"                                \
   "(([0-9]+[-_]*)*[0-9]*[A-Za-z][A-Za-z0-9]*([-_]+[A-Za-z0-9]+)*)"\
                          "(.*)";

  /*    _____________________________________________________
      _(                                                     )_
     =_            P I C T U R E  C H A R A C T E R           _=
       (_____________________________________________________)
                                                                  */

  /*-------------------------------------------*/
  /* FTA1: ponctuation                         */
  /* FTA2: CR et DB                            */
  /*--------------------------------------------------------------*/
  /* - picture-string =  currency? (picchar+ repeat?)+            */
  /*   currency       =  $                                        */
  /*   picchar        =  [ABEGPSVXZabegpsvxz90\+\-\*\$]           */
  /*   repeat         =  "(" [0-9]+ ")"                           */
  /*   mannque plus que CR et DB pour les pic string              */
  /*--------------------------------------------------------------*/

  /*char     *ptrnPicChars = "(^"                              \
                   "([ABEGPSVXZabegpsvxz90\\+\\-\\*\\$\\/\\,\\.]+))"\
  */          /*   "(( !,)*!\\()(.*)"; */                  \
              /*   "(( !,)+!\\.!\\()(.*)"; */              \
  /*               "(( !,!$)+\\.!\\.!\\()(.*)";
  */
  /*char     *ptrnPicChars = "(^"                              \ */
  /* A*  */ /*     "([ABEGPSVXZabegpsvxz90\\+\\-\\*\\$\\/\\,]*"\ */
  /* (A+ */ /*     "([ABEGPSVXZabegpsvxz90\\+\\-\\*\\$\\/\\,]+"\ */
  /* b)  */ /*     "!\\.)"                                     \ */
  /* A*  */ /*     "[ABEGPSVXZabegpsvxz90\\+\\-\\*\\$\\/\\,]*))"\ */
              /*   "(( !,)*!\\()(.*)"; */
              /*   "(( !,)+[a-zA-Z]+!( !,)*\\.!\\()(.*)"; */
            /*     "(( !,!$)+!\\.!\\()(.*)"; */


    /****/ char *ptrnPicChars = "(^"                              \
  /* A+  */  /*    "([ABEGPSVXZabegpsvxz90\\+\\-\\*\\$\\/\\,]+))" */\
                   "([-+*$/,ABEGPSVXZabegpsvxz90]+))" \
                   "(( !,!$)+!\\.!\\()(.*)";


  /*    _____________________________________________________
      _(                                                     )_
     =_             L I T E R A L  N U M E R I C              _=
       (_____________________________________________________)
                                                                  */

  /*--------------------------------------------------------------*/
  /* - Séparateur début: Zero ou plusieurs : espaces ou virgules  */
  /* - Corps : Commence par le separateur de string quote ou "    */
  /*   puis zero ou une suite de caractères sauf separateur de    */
  /*   string ou un double separateur de string et finit par le   */
  /*   meme separateur de string                                  */
  /* - S{parateur fin  : point, espace, virgule, fin, quote ou "  */
  /*--------------------------------------------------------------*/
    /****/ char   *ptrnLiterNum = "^( !,)*("                      \
                           "([\\+\\-]?[0-9]*,[0-9]+)!"  /* -9,7 */\
                           "([\\+\\-]?[0-9]+)"         /* +987 */\
                      /*    ")([^']!$)(.*)"; */                   \
{'fraction_non_alphanumeric': 0.2571619365609349, 'fraction_numerical': 0.009348914858096828, 'mean_word_length': 0.8370951913640824, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 63, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2081384', 'n_tokens_mistral': 4361, 'n_tokens_neox': 4199, 'n_words': 792}