<commit_before>/****************************************************************************
*
*                             dmental links
*    Copyright (c) 28 Feb 2005 AtKaaZ, AtKaaZ at users.sourceforge.net
*    Portions Copyright (c) 1983-2002 Sybase, Inc. All Rights Reserved.
*
*  ========================================================================
*
*    This file contains Original Code and/or Modifications of Original
*    Code as defined in and that are subject to the Sybase Open Watcom
*    Public License version 1.0 (the 'License'). You may not use this file
*    except in compliance with the License. BY USING THIS FILE YOU AGREE TO
*    ALL TERMS AND CONDITIONS OF THE LICENSE. A copy of the License is
*    provided with the Original Code and Modifications, and is also
*    available at www.sybase.com/developer/opensource.
*
*    The Original Code and all software distributed under the License are
*    distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
*    EXPRESS OR IMPLIED, AND SYBASE AND ALL CONTRIBUTORS HEREBY DISCLAIM
*    ALL SUCH WARRANTIES, INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF
*    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR
*    NON-INFRINGEMENT. Please see the License for the specific language
*    governing rights and limitations under the License.
*
*  ========================================================================
*
* Description: driver program for a char-based world (just testing stuff)
*
****************************************************************************/
/*
for theory see the file ..\txt\charwrld.txt
*/

#include <stdlib.h>
#include <stdio.h>
#include <conio.h>
#include <io.h>
#include <string.h>

#include "petrackr.h"
#include "dmentalx.h"
#include "nicef.h"


//#define PARANOIA_CHECKS
#undef PARANOIA_CHECKS //some extra (lame)checks

#define num_cached_records 2048
#define MAXWORDLEN 255 //anyword in our world mustn't be larger than this (in chars)
#define cfg_fname "test03.cfg"

nicefi *lamecfg;
dmentalix *test3;
enum enum_specials{
_spec_filename=0,
_spec_dirname,
_spec_file,
_spec_dir,
_spec_dircontents,
_spec_dir_dirlist,
_spec_dir_filelist,
_spec_contents,
_spec_root,
_spec_word,
numspex};

atomID _specials[numspex];//there are some special atomIDs recognized internally; faster if stored in this way

reterrt get_word(const char *w, groupID &dest){
//attempts to find the word's groupID, if the word exists
//if not exist, dest=_noID_, funx returns 0 if errors which is prolly bad
    //we have to do an AND search thru the lists of eatoms involved.
    //those lists which say this and that refers to me(the eatom).
    //in the event that we find a group of 1+strlen(w) acatoms and there are no more than 1+strlen(w) in that group, we found our winner :) we won't continue looking for another similar group since words should be uniq in our world.
    long wlen=strlen(w);
    ret_if(wlen>MAXWORDLEN);//SAFETY 1st

    eatomslist_itemID heads[MAXWORDLEN];//max 255 chars in a word allowed
    //getting first referer in list for each eatom
    for (long pos=0;pos<=wlen;pos++){
        ret_ifnot( test3->get_atomID_s_headIDof_eatomslistofclones(pos,heads[pos]) );
    }//for
    //now we got to carefully parse from each head and see weather we find one
    //common group for all... damn is this backtracking? this is gonna hurt :D
    //BUT, after we find them all to be part of the same group we must check
    //that 1st acatom is _word_ special, and that the rest atoms which follow
    //must be in the order need, ie. words like ACE and CEA would have their 
    //acatoms yield the same group but they are in different order. And lastly
    //the last acatom must have .next =null this means that the word isn't
    //longer than the one we're lookin for, ie. ACE and ACED.
//FIXME:

    ret_ok();
}

reterrt abs_add_word(const char *w,groupID &dest){
//not checking if the word already exists, dangerous tho, shouldn't be called
//outside add_word()
    //create acatom no.1 which refs to _spec_word
    //create a group with head ^
    //add acatoms to each char in word, to this group
    dest=test3->add_empty_group();
    ret_if(dest == _noID_);
    atomID prev;
    prev=test3->strict_add_atom_type_AC_after_prev(
            _specials[_spec_word]/*ptr2what*/
            ,dest/*father group*/
            ,_noID_/*who's prev atom*/
            );
    ret_if(_noID_ == prev);
    for (long i=0;i<=strlen(w);i++){
        prev=test3->strict_add_atom_type_AC_after_prev(
                w[i]+1,//assuming hard that atomID of char #i is ID=ord(i)+1
                dest,
                prev//connect them
        );
        ret_if(prev == _noID_);<commit_msg>planning a serious ordering of the code, this time completly ignoring performance, making a goal to make it clearer for the developer who reads the source code<commit_after>/****************************************************************************
*
*                             dmental links
*    Copyright (c) 28 Feb 2005 AtKaaZ, AtKaaZ at users.sourceforge.net
*    Portions Copyright (c) 1983-2002 Sybase, Inc. All Rights Reserved.
*
*  ========================================================================
*
*    This file contains Original Code and/or Modifications of Original
*    Code as defined in and that are subject to the Sybase Open Watcom
*    Public License version 1.0 (the 'License'). You may not use this file
*    except in compliance with the License. BY USING THIS FILE YOU AGREE TO
*    ALL TERMS AND CONDITIONS OF THE LICENSE. A copy of the License is
*    provided with the Original Code and Modifications, and is also
*    available at www.sybase.com/developer/opensource.
*
*    The Original Code and all software distributed under the License are
*    distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
*    EXPRESS OR IMPLIED, AND SYBASE AND ALL CONTRIBUTORS HEREBY DISCLAIM
*    ALL SUCH WARRANTIES, INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF
*    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR
*    NON-INFRINGEMENT. Please see the License for the specific language
*    governing rights and limitations under the License.
*
*  ========================================================================
*
* Description: driver program for a char-based world (just testing stuff)
*
****************************************************************************/
/*
for theory see the file ..\txt\charwrld.txt
*/

#include <stdlib.h>
#include <stdio.h>
#include <conio.h>
#include <io.h>
#include <string.h>

#include "petrackr.h"
#include "dmentalx.h"
#include "nicef.h"


//#define PARANOIA_CHECKS
#undef PARANOIA_CHECKS //some extra (lame)checks

#define num_cached_records 2048
#define MAXWORDLEN 255 //anyword in our world mustn't be larger than this (in chars)
#define cfg_fname "test03.cfg"

nicefi *lamecfg;
dmentalix *test3;
enum enum_specials{
_spec_filename=0,
_spec_dirname,
_spec_file,
_spec_dir,
_spec_dircontents,
_spec_dir_dirlist,
_spec_dir_filelist,
_spec_contents,
_spec_root,
_spec_word,
numspex};

atomID _specials[numspex];//there are some special atomIDs recognized internally; faster if stored in this way

reterrt get_word(const char *w, groupID &dest){
//attempts to find the word's groupID, if the word exists
//if not exist, dest=_noID_, funx returns 0 if errors which is prolly bad
    //we have to do an AND search thru the lists of eatoms involved.
    //those lists which say this and that refers to me(the eatom).
    //in the event that we find a group of 1+strlen(w) acatoms and there are no more than 1+strlen(w) in that group, we found our winner :) we won't continue looking for another similar group since words should be uniq in our world.
    long wlen=strlen(w);
    ret_if(wlen>MAXWORDLEN);//SAFETY 1st

    eatomslist_itemID heads[MAXWORDLEN];//max 255 chars in a word allowed
    //getting first referer in list for each eatom
    for (long pos=0;pos<wlen;pos++){
        ret_ifnot( test3->get_atomID_s_headIDof_eatomslistofclones(pos,heads[pos]) );
    }//for
    //now we got to carefully parse from each head and see weather we find one
    //common group for all... damn is this backtracking? this is gonna hurt :D
    //BUT, after we find them all to be part of the same group we must check
    //that 1st acatom is _word_ special, and that the rest atoms which follow
    //must be in the order need, ie. words like ACE and CEA would have their 
    //acatoms yield the same group but they are in different order. And lastly
    //the last acatom must have .next =null this means that the word isn't
    //longer than the one we're lookin for, ie. ACE and ACED.
//FIXME:

    ret_ok();
}//look at my work[!], not at me!

reterrt abs_add_word(const char *w,groupID &dest){
//not checking if the word already exists, dangerous tho, shouldn't be called
//outside add_word()
    //create acatom no.1 which refs to _spec_word
    //create a group with head ^
    //add acatoms to each char in word, to this group
    dest=test3->add_empty_group();
    ret_if(dest == _noID_);
    atomID prev;
    prev=test3->strict_add_atom_type_AC_after_prev(
            _specials[_spec_word]/*ptr2what*/
            ,dest/*father group*/
            ,_noID_/*who's prev atom*/
            );
    ret_if(_noID_ == prev);
    for (long i=0;i<strlen(w);i++){
        prev=test3->strict_add_atom_type_AC_after_prev(
                w[i]+1,//assuming hard that atomID of char #i is ID=ord(i)+1
                dest,
                prev//connect them
        );
        ret_if(prev == _noID_);
{'fraction_non_alphanumeric': 0.14014128402243922, 'fraction_numerical': 0.009349677955537088, 'mean_word_length': 3.5603979156797725, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '478250', 'n_tokens_mistral': 2966, 'n_tokens_neox': 2798, 'n_words': 1144}