<commit_before>

#include <stdio.h>
#include <stdlib.h>
#include <memory.h>
#include <string.h>

#include <assert.h>
#define null 0
#define MAX_SYM 1024

/* otherwise we need more than 31 bits for a triple. 

Note:
   1290^3 < 2^31 < 1291^3 
   
Some of these sets may be spurious, so maybe we could go
upto 1293 ???; it might be possible to use the top bit; 
it would be real work to move to a long.
maybe three levels of switches ... (never turn S,P,O into
single value)
Not clear what real max is. 3 or 4 MB?
*/
#define MAX_SET 5000
#define MAX_TRIPLES 5000
#define O(x)  ((x)&1023)
#define P(x)  (((x)>>10)&1023)
#define S(x)  (((x)>>20)&1023)
#define SPO(s,p,o) (((s)<<20)|((p)<<10)|(o))
#define MAX_RESULTS 500000

/* Special subcategory support:
    orphan
   Used to detected orphaned rdf:'List' s and
   orphaned owl:'OntologyProperty' s.
*/
int orphan = -1;
int notype = -1;
int cyclic = -1;
#define SPECIALSYM(n) ((n)<3)
int rdffirst;
int rdfrest;
int rdftype;
int orphanSet = -1;
int notypeSet = -1;
int orphanNotypeSet = -1;
int rdfList;
int owlOntologyProperty;

int notType[5];
int rdfsSubClassOf;
int owldisjointWith;
int owlequivalentClass;

#define DIM(array) (sizeof(array)/sizeof(array[0]))
struct { char * str; int * sym; } specials[] = {
  { "orphan", &orphan },
  { "notype", &notype },
  { "cyclic", &cyclic },<commit_msg>Latest changes - the precompute is large > 500000 records. Updated MAX_RESULTS constant to be more realistic.<commit_after>

#include <stdio.h>
#include <stdlib.h>
#include <memory.h>
#include <string.h>

#include <assert.h>
#define null 0
#define MAX_SYM 1024

/* otherwise we need more than 31 bits for a triple. 

Note:
   1290^3 < 2^31 < 1291^3 
   
Some of these sets may be spurious, so maybe we could go
upto 1293 ???; it might be possible to use the top bit; 
it would be real work to move to a long.
maybe three levels of switches ... (never turn S,P,O into
single value)
Not clear what real max is. 3 or 4 MB?
*/
#define MAX_SET 5000
#define MAX_TRIPLES 5000
#define O(x)  ((x)&1023)
#define P(x)  (((x)>>10)&1023)
#define S(x)  (((x)>>20)&1023)
#define SPO(s,p,o) (((s)<<20)|((p)<<10)|(o))
#define MAX_RESULTS 2000000

/* Special subcategory support:
    orphan
   Used to detected orphaned rdf:'List' s and
   orphaned owl:'OntologyProperty' s.
*/
int orphan = -1;
int notype = -1;
int cyclic = -1;
#define SPECIALSYM(n) ((n)<3)
int rdffirst;
int rdfrest;
int rdftype;
int orphanSet = -1;
int notypeSet = -1;
int orphanNotypeSet = -1;
int rdfList;
int owlOntologyProperty;

int notType[5];
int rdfsSubClassOf;
int owldisjointWith;
int owlequivalentClass;

#define DIM(array) (sizeof(array)/sizeof(array[0]))
struct { char * str; int * sym; } specials[] = {
  { "orphan", &orphan },
  { "notype", &notype },
  { "cyclic", &cyclic },
{'fraction_non_alphanumeric': 0.14628899247041952, 'fraction_numerical': 0.05198996055934026, 'mean_word_length': 4.205223880597015, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3378116', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1052, 'n_words': 362}