<commit_before>/*
 *  The OpenDiamond Platform for Interactive Search
 *  Version 2
 *
 *  Copyright (c) 2002-2005 Intel Corporation
 *  All rights reserved.
 *
 *  This software is distributed under the terms of the Eclipse Public
 *  License, Version 1.0 which can be found in the file named LICENSE.
 *  ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS SOFTWARE CONSTITUTES
 *  RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT
 */

#ifndef _FILTER_PRIV_H_
#define _FILTER_PRIV_H_

#include <stdint.h>
#include <limits.h>

#include "rtimer.h"
#include "rgraph.h"
#include "lib_filter.h"         /* for filter_proto */
#include "queue.h"
#include "diamond_consts.h"
#include "rcomb.h"


/*
 * This the the header file used to keep track of the 
 * filter state for each of the filters in the current
 * application.
 */

typedef enum filter_output_type {
    FO_UNMODIFIED = 0,          /* default */
    FO_NEW,
    FO_CLONE,
    FO_COPY_ATTR
} filter_output_type_t;


/*
 * name is set to a strdup'd string when reading filter info.
 * resolve_filter_deps will change pointer to filter*. not needed elsewhere? 
 */
// struct filter_info;
typedef struct filter_dep_t
{
	char           *name;
	// struct filter_info *filter;
	// TAILQ_ENTRY(filter_dep_t) link;
}
filter_dep_t;


/*
 * These are some definitions related to the filter id
 * data type.
 */
#define FILTER_ID_MASK  0x000000FF
#define FILTER_ID(x)    ((filter_id_t)(((uint32_t)(x))&FILTER_ID_MASK))

typedef uint8_t filter_id_t;
#define INVALID_FILTER_ID   0xFF


typedef struct filter_info {
	/*
	 * general filter info (not device-specific)
	 */
	char            fi_name[MAX_FILTER_NAME];
	char            fi_eval_name[MAX_FILTER_FUNC_NAME];
	char            fi_init_name[MAX_FILTER_FUNC_NAME];
	char            fi_fini_name[MAX_FILTER_FUNC_NAME];
	filter_init_proto fi_init_fp;
	filter_eval_proto fi_eval_fp;
	filter_fini_proto fi_fini_fp;
	int             fi_threshold;
	int             fi_merit;
	int             fi_numargs;
	int             fi_maxargs;
	char          **fi_arglist;
	filter_id_t     fi_filterid;    /* id of this filter */
	void           *fi_filt_arg;    /* associated argument data */

	int             fi_blob_len;    /* associated blob len */
	void           *fi_blob_data;   /* associated blob of data */
	sig_val_t	fi_blob_sig;	/* checksum of the blob data */

	/*
	 * dependency info 
	 */
	int             fi_color;   /* used by dfs */
	int             fi_depcount;
	filter_dep_t    fi_deps[MAX_NUM_DEPS];
	node_t         *fi_gnode;

	/*
	 * bypass information, the fi_bpcnt keeps track of where we are in the
	 * loop.  We logically process fi_bprun/fi_bpmax of these objects. 
	 */

	unsigned int             fi_bpthresh;
	int             fi_firstgroup;

	/*
	 * input characteristics 
	 */
	int             fi_blocksize_in;

	/*
	 * output characteristics 
	 */
	int             fi_blocksize_out;
	filter_output_type_t fi_type_out;

	/*
	 * statistics. these should be local to each device.
	 */
	int             fi_called;  /* # of times called */
	int             fi_bypassed;/* # of times we would have run */
	int             fi_drop;    /* # times below threshold */
	int             fi_pass;    /* # times above threshold */
	int             fi_error;   /* # errors running filter */
	rtime_t         fi_time_ns; /* total time used */
	int64_t         fi_added_bytes; /* XXX debug */
	/* JIAYING */
	char	    	lib_name[PATH_MAX];
	sig_val_t	fi_sig;
	void	*   	cache_table;
	int	    	fi_cache_drop; /* # of objs dropped through cache lookup */
	int		fi_cache_pass; /* # of objs skipped by using cache */
	int		fi_compute;    /* # of objs computed */
} filter_info_t;


typedef struct filter_prob {
	LIST_ENTRY(filter_prob) prob_link;
	int             num_pass;   /* # of times this combination passed */
	int             num_exec;   /* # of times this combination was run */
	int             num_prev;   /* # of previus filters run */
	filter_id_t     cur_fid;    /* current filter id */
	filter_id_t     prev_id[0]; /* list of previous IDs */
} filter_prob_t;


#define FILTER_PROB_SIZE(x)     \
        (sizeof(filter_prob_t)+((x)* sizeof(filter_id_t)))

/*
 * must be power of 2 
 */
#define PROB_HASH_BUCKETS   64
<commit_msg>added counts for hit classification<commit_after>/*
 *  The OpenDiamond Platform for Interactive Search
 *  Version 2
 *
 *  Copyright (c) 2002-2005 Intel Corporation
 *  All rights reserved.
 *
 *  This software is distributed under the terms of the Eclipse Public
 *  License, Version 1.0 which can be found in the file named LICENSE.
 *  ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS SOFTWARE CONSTITUTES
 *  RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT
 */

#ifndef _FILTER_PRIV_H_
#define _FILTER_PRIV_H_

#include <stdint.h>
#include <limits.h>

#include "rtimer.h"
#include "rgraph.h"
#include "lib_filter.h"         /* for filter_proto */
#include "queue.h"
#include "diamond_consts.h"
#include "rcomb.h"


/*
 * This the the header file used to keep track of the 
 * filter state for each of the filters in the current
 * application.
 */

typedef enum filter_output_type {
    FO_UNMODIFIED = 0,          /* default */
    FO_NEW,
    FO_CLONE,
    FO_COPY_ATTR
} filter_output_type_t;


/*
 * name is set to a strdup'd string when reading filter info.
 * resolve_filter_deps will change pointer to filter*. not needed elsewhere? 
 */
// struct filter_info;
typedef struct filter_dep_t
{
	char           *name;
	// struct filter_info *filter;
	// TAILQ_ENTRY(filter_dep_t) link;
}
filter_dep_t;


/*
 * These are some definitions related to the filter id
 * data type.
 */
#define FILTER_ID_MASK  0x000000FF
#define FILTER_ID(x)    ((filter_id_t)(((uint32_t)(x))&FILTER_ID_MASK))

typedef uint8_t filter_id_t;
#define INVALID_FILTER_ID   0xFF


typedef struct filter_info {
	/*
	 * general filter info (not device-specific)
	 */
	char            fi_name[MAX_FILTER_NAME];
	char            fi_eval_name[MAX_FILTER_FUNC_NAME];
	char            fi_init_name[MAX_FILTER_FUNC_NAME];
	char            fi_fini_name[MAX_FILTER_FUNC_NAME];
	filter_init_proto fi_init_fp;
	filter_eval_proto fi_eval_fp;
	filter_fini_proto fi_fini_fp;
	int             fi_threshold;
	int             fi_merit;
	int             fi_numargs;
	int             fi_maxargs;
	char          **fi_arglist;
	filter_id_t     fi_filterid;    /* id of this filter */
	void           *fi_filt_arg;    /* associated argument data */

	int             fi_blob_len;    /* associated blob len */
	void           *fi_blob_data;   /* associated blob of data */
	sig_val_t	fi_blob_sig;	/* checksum of the blob data */

	/*
	 * dependency info 
	 */
	int             fi_color;   /* used by dfs */
	int             fi_depcount;
	filter_dep_t    fi_deps[MAX_NUM_DEPS];
	node_t         *fi_gnode;

	/*
	 * bypass information, the fi_bpcnt keeps track of where we are in the
	 * loop.  We logically process fi_bprun/fi_bpmax of these objects. 
	 */

	unsigned int             fi_bpthresh;
	int             fi_firstgroup;

	/*
	 * input characteristics 
	 */
	int             fi_blocksize_in;

	/*
	 * output characteristics 
	 */
	int             fi_blocksize_out;
	filter_output_type_t fi_type_out;

	/*
	 * statistics. these should be local to each device.
	 */
	int             fi_called;  /* # of times called */
	int             fi_bypassed;/* # of times we would have run */
	int             fi_drop;    /* # times below threshold */
	int             fi_pass;    /* # times above threshold */
	int             fi_error;   /* # errors running filter */
	rtime_t         fi_time_ns; /* total time used */
	int64_t         fi_added_bytes; /* XXX debug */
	/* JIAYING */
	char	    	lib_name[PATH_MAX];
	sig_val_t	fi_sig;
	void	*   	cache_table;
	int	    	fi_cache_drop; /* # of objs dropped through cache lookup */
	int		fi_cache_pass; /* # of objs skipped by using cache */
	int		fi_compute;    /* # of objs evaluated */
	int		fi_hits_inter_session;	/* hits computed before this session */
	int		fi_hits_inter_query;	/* hits computed in session, before query */
	int		fi_hits_intra_query;	/* hits computed this query */
} filter_info_t;


typedef struct filter_prob {
	LIST_ENTRY(filter_prob) prob_link;
	int             num_pass;   /* # of times this combination passed */
	int             num_exec;   /* # of times this combination was run */
	int             num_prev;   /* # of previus filters run */
	filter_id_t     cur_fid;    /* current filter id */
	filter_id_t     prev_id[0]; /* list of previous IDs */
} filter_prob_t;


#define FILTER_PROB_SIZE(x)     \
        (sizeof(filter_prob_t)+((x)* sizeof(filter_id_t)))

/*
 * must be power of 2 
 */
#define PROB_HASH_BUCKETS   64

{'fraction_non_alphanumeric': 0.08748845798707294, 'fraction_numerical': 0.006694367497691598, 'mean_word_length': 2.5137875101378753, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3934851', 'n_tokens_mistral': 3019, 'n_tokens_neox': 2862, 'n_words': 819}