<commit_before>/*
 * Copyright 2012 The Weather Channel
 * Copyright 2013 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 */


#include "dclass_client.h"


//structs for parsing dtree file
typedef struct
{
    char *key;
    char *value;
    
    size_t key_len;
    size_t val_len;
}
dtree_kv_pair;

typedef struct
{
    char *pattern;
    char *id;
    char *cparam;
    char *type;
    
    size_t pattern_len;
    size_t id_len;
    size_t cparam_len;
    
    int count;
    
    dtree_flag_f flag;

    unsigned int pos;
    unsigned int rank;
    signed int dir;
    
    dtree_kv_pair p[DTREE_DATA_MKEYS];
}
dtree_file_entry;


static void dclass_parse_fentry(char*,dtree_file_entry*,int);
static long dclass_write_tree(const dtree_dt_index*,const dtree_dt_node*,char*,int,FILE*);
static void dclass_write_node(const dtree_dt_node*,char*,FILE*);

extern unsigned int dtree_hash_char(char);


#if DTREE_PERF_WALKING
extern void dtree_timersubn(struct timespec*,struct timespec*,struct timespec*);
extern int dtree_gettime(struct timespec*);
#endif


//parses a dtree file
int dclass_load_file(dclass_index *di,const char *path)
{
    int i,j;
    int ret;
    int lines=0;
    int keys_size=0;
    char *p;
    char *s;
    char **keys=NULL;
    char buf[10*1024];
    FILE *f;
    dtree_dt_index ids;
    dtree_dt_index *h=&di->dti;
    dtree_file_entry fe;
    dclass_keyvalue *kvd;
    dclass_keyvalue *cparam;
    dtree_dt_add_entry entry;
    
#if DTREE_PERF_WALKING
    struct timespec startn,endn,diffn;
    long total,high,low,val,lret;
#endif
    
    dclass_init_index(di);
    dtree_init_index(&ids);
    
    dtree_printd(DTREE_PRINT_GENERIC,"DTREE: Loading '%s'\n",path);
    
    if(!(f=fopen(path,"r")))
    {
        fprintf(stderr,"DTREE: cannot open '%s'\n",path);
        return -1;
    }
    
    //traverse the loadfile
    while((buf[sizeof(buf)-2]='\n') && (p=fgets(buf,sizeof(buf),f)))
    {
        lines++;
        
        //overflow
        if(buf[sizeof(buf)-2]!='\n' && buf[sizeof(buf)-2])
        {
            dtree_printd(DTREE_PRINT_INITDTREE,"LOAD: overflow detected on line %d\n",lines);
            
            do
            {
                buf[sizeof(buf)-2]='\n';
                
                if(!fgets(buf,sizeof(buf),f))
                    break;
            }
            while(buf[sizeof(buf)-2]!='\n' && buf[sizeof(buf)-2]);
            
            continue;
        }
        
        if(*buf=='#')
        {
            switch(*++p)
            {
                //comment
                case ' ':
                    if(!h->comment)
                    {
                        for(++p,s=p;*p;p++)
                        {
                            if(*p=='\n')
                                break;
                        }
                        *p='\0';
                        h->comment=dtree_alloc_mem(h,(p-s+1)*sizeof(char));
                        if(h->comment)
                            strncpy(h->comment,s,p-s);
                    }
                    break;
                //string order is being forced here
                case '!':
                    for(++p,s=p;*p;p++)
                    {
                        if(*p==',' || !*(p+1))
                        {
                            *p='\0';
                            s=dtree_alloc_string(h,s,p-s);
                            dtree_printd(DTREE_PRINT_INITDTREE,"INIT FORCE string: '%s'\n",s);
                            
                            kvd=(dclass_keyvalue*)dtree_alloc_mem(h,sizeof(dclass_keyvalue));
            
                            if(!kvd)
                                goto lerror;

                            kvd->id=s;

                            memset(&entry,0,sizeof(dtree_dt_add_entry));

                            entry.data=kvd;

                            if(dtree_add_entry(&ids,kvd->id,&entry)<0)
                                goto lerror;

                            dtree_printd(DTREE_PRINT_INITDTREE,"IDS: Successful add: '%s'\n",kvd->id);
                            
                            s=(p+1);
                        }
                    }
                    break;
                //index params
                case '$':
                    p++;
                    if(!strncasecmp(p,"partial",6))
                    {
                        h->sflags |= DTREE_S_FLAG_PARTIAL;
                        dtree_printd(DTREE_PRINT_INITDTREE,"INIT FORCE: partial\n");
                    }
                    else if(!strncasecmp(p,"regex",5))
                    {
                        h->sflags |= DTREE_S_FLAG_REGEX;
                        dtree_printd(DTREE_PRINT_INITDTREE,"INIT FORCE: regex\n");
                    }
                    else if(!strncasecmp(p,"dups",4))
                    {
                        h->sflags |= DTREE_S_FLAG_DUPS;
                        dtree_printd(DTREE_PRINT_INITDTREE,"INIT FORCE: dups\n");
                    }
                    else if(!strncasecmp(p,"notrim",6))
                    {
                        h->sflags |= DTREE_S_FLAG_NOTRIM;
                        dtree_printd(DTREE_PRINT_INITDTREE,"INIT FORCE: notrim\n");
                    }
                    break;
            }          
            continue;
        }
        
        memset(&fe,0,sizeof(fe));
        
        //parse the line
        dclass_parse_fentry(buf,&fe,h->sflags & DTREE_S_FLAG_NOTRIM);<commit_msg>comment<commit_after>/*
 * Copyright 2012 The Weather Channel
 * Copyright 2013 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 */


#include "dclass_client.h"


//structs for parsing dtree file
typedef struct
{
    char *key;
    char *value;
    
    size_t key_len;
    size_t val_len;
}
dtree_kv_pair;

typedef struct
{
    char *pattern;
    char *id;
    char *cparam;
    char *type;
    
    size_t pattern_len;
    size_t id_len;
    size_t cparam_len;
    
    int count;
    
    dtree_flag_f flag;

    unsigned int pos;
    unsigned int rank;
    signed int dir;
    
    dtree_kv_pair p[DTREE_DATA_MKEYS];
}
dtree_file_entry;


static void dclass_parse_fentry(char*,dtree_file_entry*,int);
static long dclass_write_tree(const dtree_dt_index*,const dtree_dt_node*,char*,int,FILE*);
static void dclass_write_node(const dtree_dt_node*,char*,FILE*);

extern unsigned int dtree_hash_char(char);


#if DTREE_PERF_WALKING
extern void dtree_timersubn(struct timespec*,struct timespec*,struct timespec*);
extern int dtree_gettime(struct timespec*);
#endif


//parses a dtree file
int dclass_load_file(dclass_index *di,const char *path)
{
    int i,j;
    int ret;
    int lines=0;
    int keys_size=0;
    char *p;
    char *s;
    char **keys=NULL;
    char buf[10*1024];
    FILE *f;
    dtree_dt_index ids;
    dtree_dt_index *h=&di->dti;
    dtree_file_entry fe;
    dclass_keyvalue *kvd;
    dclass_keyvalue *cparam;
    dtree_dt_add_entry entry;
    
#if DTREE_PERF_WALKING
    struct timespec startn,endn,diffn;
    long total,high,low,val,lret;
#endif
    
    dclass_init_index(di);
    dtree_init_index(&ids);
    
    dtree_printd(DTREE_PRINT_GENERIC,"DTREE: Loading '%s'\n",path);
    
    if(!(f=fopen(path,"r")))
    {
        fprintf(stderr,"DTREE: cannot open '%s'\n",path);
        return -1;
    }
    
    //traverse the loadfile
    while((buf[sizeof(buf)-2]='\n') && (p=fgets(buf,sizeof(buf),f)))
    {
        lines++;
        
        //overflow
        if(buf[sizeof(buf)-2]!='\n' && buf[sizeof(buf)-2])
        {
            dtree_printd(DTREE_PRINT_INITDTREE,"LOAD: overflow detected on line %d\n",lines);
            
            do
            {
                buf[sizeof(buf)-2]='\n';
                
                if(!fgets(buf,sizeof(buf),f))
                    break;
            }
            while(buf[sizeof(buf)-2]!='\n' && buf[sizeof(buf)-2]);
            
            continue;
        }
        
        if(*buf=='#')
        {
            switch(*++p)
            {
                //string order is being forced here
                case '!':
                    for(++p,s=p;*p;p++)
                    {
                        if(*p==',' || !*(p+1))
                        {
                            *p='\0';
                            s=dtree_alloc_string(h,s,p-s);
                            dtree_printd(DTREE_PRINT_INITDTREE,"INIT FORCE string: '%s'\n",s);
                            
                            kvd=(dclass_keyvalue*)dtree_alloc_mem(h,sizeof(dclass_keyvalue));
            
                            if(!kvd)
                                goto lerror;

                            kvd->id=s;

                            memset(&entry,0,sizeof(dtree_dt_add_entry));

                            entry.data=kvd;

                            if(dtree_add_entry(&ids,kvd->id,&entry)<0)
                                goto lerror;

                            dtree_printd(DTREE_PRINT_INITDTREE,"IDS: Successful add: '%s'\n",kvd->id);
                            
                            s=(p+1);
                        }
                    }
                    break;
                //index params
                case '$':
                    p++;
                    if(!strncasecmp(p,"partial",6))
                    {
                        h->sflags |= DTREE_S_FLAG_PARTIAL;
                        dtree_printd(DTREE_PRINT_INITDTREE,"INIT FORCE: partial\n");
                    }
                    else if(!strncasecmp(p,"regex",5))
                    {
                        h->sflags |= DTREE_S_FLAG_REGEX;
                        dtree_printd(DTREE_PRINT_INITDTREE,"INIT FORCE: regex\n");
                    }
                    else if(!strncasecmp(p,"dups",4))
                    {
                        h->sflags |= DTREE_S_FLAG_DUPS;
                        dtree_printd(DTREE_PRINT_INITDTREE,"INIT FORCE: dups\n");
                    }
                    else if(!strncasecmp(p,"notrim",6))
                    {
                        h->sflags |= DTREE_S_FLAG_NOTRIM;
                        dtree_printd(DTREE_PRINT_INITDTREE,"INIT FORCE: notrim\n");
                    }
                    break;
                //comment
                default:
                    if(!h->comment)
                    {
                        if(*p==' ')
                            p++;
                        for(s=p;*p;p++)
                        {
                            if(*p=='\n')
                                break;
                        }
                        *p='\0';
                        h->comment=dtree_alloc_mem(h,(p-s+1)*sizeof(char));
                        if(h->comment)
                            strncpy(h->comment,s,p-s);
                    }
                    break;
            }          
            continue;
        }
        
        memset(&fe,0,sizeof(fe));
        
        //parse the line
        dclass_parse_fentry(buf,&fe,h->sflags & DTREE_S_FLAG_NOTRIM);
{'fraction_non_alphanumeric': 0.11084357356360014, 'fraction_numerical': 0.006476253736300232, 'mean_word_length': 1.3252895752895753, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6914174', 'n_tokens_mistral': 3758, 'n_tokens_neox': 3570, 'n_words': 665}