<jupyter_start><jupyter_text>int parse_document(char *line, WORD *words, double *label,		   long *queryid, long *slackid, double *costfactor,		   long int *numwords, long int max_words_doc,		   char **comment){  register long wpos,pos;  long wnum;  double weight;  char featurepair[1000],junk[1000];  (*queryid)=0;  (*slackid)=0;  (*costfactor)=1;  pos=0;  (*comment)=NULL;  while(line[pos] ) {      /* cut off comments */    if((line[pos] == '') && (!(*comment))) {      line[pos]=0;      (*comment)=&(line[pos+1]);    }    if(line[pos] == '\n') { /* strip the CR */      line[pos]=0;    }    pos++;  }  if(!(*comment)) (*comment)=&(line[pos]);  /* printf("Comment: '%s'\n",(*comment)); */  wpos=0;  /* check, that line starts with target value or zero, but not with     feature pair */  if(sscanf(line,"%s",featurepair) == EOF) return(0);  pos=0;  while((featurepair[pos] != ':') && featurepair[pos]) pos++;  if(featurepair[pos] == ':') {	perror ("Line must start with label or 0!!!\n"); 	printf("LINE: %s\n",line);	exit (1);   }  /* read the target value */  if(sscanf(line,"%lf",label) == EOF) return(0);  pos=0;  while(space_or_null((int)line[pos])) pos++;  while((!space_or_null((int)line[pos])) && line[pos]) pos++;  while((pos+=read_word(line+pos,featurepair)) &&	(featurepair[0]) && 	(wpos<max_words_doc)) {    /* printf("%s\n",featurepair); */    if(sscanf(featurepair,"qid:%ld%s",&wnum,junk)==1) {      /* it is the query id */      (*queryid)=(long)wnum;    }    else if(sscanf(featurepair,"sid:%ld%s",&wnum,junk)==1) {      /* it is the slack id */      if(wnum > 0) 	(*slackid)=(long)wnum;      else {	perror ("Slack-id must be greater or equal to 1!!!\n"); 	printf("LINE: %s\n",line);	exit (1);       }    }    else if(sscanf(featurepair,"cost:%lf%s",&weight,junk)==1) {      /* it is the example-dependent cost factor */      (*costfactor)=(double)weight;    }    else if(sscanf(featurepair,"%ld:%lf%s",&wnum,&weight,junk)==2) {      /* it is a regular feature */      if(wnum<=0) { 	perror ("Feature numbers must be larger or equal to 1!!!\n"); 	printf("LINE: %s\n",line);	exit (1);       }      if((wpos>0) && ((words[wpos-1]).wnum >= wnum)) { 	perror ("Features must be in increasing order!!!\n"); 	printf("LINE: %s\n",line);	exit (1);       }      (words[wpos]).wnum=wnum;      (words[wpos]).weight=(FVAL)weight;       wpos++;    }    else {      perror ("Cannot parse feature/value pair!!!\n");       printf("'%s' in LINE: %s\n",featurepair,line);      exit (1);     }  }  (words[wpos]).wnum=0;  (*numwords)=wpos+1;  return(1);} if((wpos>0) && ((words[wpos-1]).wnum >= wnum)) { 	perror ("Features must be in increasing order!!!\n"); 	printf("LINE: %s\n",line);	exit (1); }<jupyter_code># Okay, so this is where we're falling down..
# The position is the pointer on the current line for parseing the file..
# and wpos is the word position..
# so the conditional we're failing @ is: if the word position is > 0 and the value (struct, I assume) at some 
# memory location (representing the final word)s wnum value is >= wnum, then we exit..
# now.. what's wnum..?
# I'm half tempted to just remove this conditional and recompile, lol

# it looks like wnum is the (temporarily) binary property that we're using to say that this feature is present
# within this data point, it's weird because each word has a wnum value and there appears to be some kind of
# global wnum..

# I believe.. We can just remove this 'naughtily' remove this conditional form the source code and recompile..
# because I really don't see what I'm doing wrong.
# jesus, now features have to start from 1, not zero, kill me lord
training_dict_of_tokens
# okay, so if I re-introduce the conditional into the source code for the SVM, it doesn't run
# I believe it's because the feature values (on the left hand side) are not increaseing from 1 to 14
# so... let's make some changes and see if modifying that increases the accuracy from ~61%
training_set_compart
with codecs.open("./feats.train", "r", "UTF-8") as file:
    with codecs.open("./feats.train.ordered", "w", "UTF-8") as write_file:
        for line in sorted(file.readlines(), key=lambda line: int(line.split()[0])):
            if len(line) > 1:
                write_file.write(line)
'newest' in training_dict_of_tokens.values()
training_set_compart_pre<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17693194925028835, 'fraction_numerical': 0.01245674740484429, 'mean_word_length': 3.8828828828828827, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '425010', 'license': 'MIT', 'n_tokens_mistral': 1519, 'n_tokens_neox': 1467, 'n_words': 451, 'path': 'notebooks/old/Prorotype.ipynb', 'repo_name': 'jonathondilworth/ttds_cw_two_code', 'size': '4335'}