<commit_before>package roart.search;

import roart.model.IndexFiles;
import roart.model.HibernateUtil;
import roart.queue.IndexQueueElement;
import roart.queue.Queues;
import roart.lang.LanguageDetect;
import roart.model.ResultItem;

import roart.dao.SearchDao;

import java.io.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.Map;
import java.util.HashMap;
import java.util.List;
import java.util.HashSet;
 
import org.apache.tika.metadata.Metadata;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class Search {
    private static Log log = LogFactory.getLog("Search");

    //public static int indexme(String type, String md5, InputStream inputStream) {
    public static void indexme() {
    	IndexQueueElement el = Queues.indexQueue.poll();
    	if (el == null) {
    		log.error("empty queue");
    	    return;
    	}
    	// vulnerable spot
    	Queues.incIndexs();
    	long now = System.currentTimeMillis();
    	
    	String type = el.type;
     	String md5 = el.md5;
    	InputStream inputStream = el.inputStream;
    	IndexFiles dbindex = el.index;
    	String dbfilename = el.dbfilename;
	Metadata metadata = el.metadata;
	String lang = el.lang;
	String content = el.content;
	String classification = el.index.getClassification();
    	List<ResultItem> retlist = el.retlist;

    int retsize = 0;

    retsize = SearchDao.indexme(type, md5, inputStream, dbfilename, metadata.toString(), lang, content, classification, retlist);

    if (retsize < 0) {
	dbindex.setNoindexreason("Exception"); // later, propagate the exception
    } else {

    log.info("size2 " + retsize);
	el.size = retsize;
	dbindex.setIndexed(Boolean.TRUE);
	dbindex.setTimestamp("" + System.currentTimeMillis());
	dbindex.setConvertsw(el.convertsw);
	//dbindex.save();
	long time = System.currentTimeMillis() - now;
	dbindex.setTimeindex(time);
	log.info("timerStop filename " + time);

	String myclassify = roart.util.Prop.getProp().getProperty("myclassify");
	boolean doclassify = myclassify != null && myclassify.length() > 0;

	ResultItem ri = new ResultItem();
	ri.add(md5);
	ri.add(dbfilename);
	ri.add(lang);
	if (doclassify) {
	    ri.add(el.index.getClassification());
	}
	ri.add(el.index.getTimestamp());
	ri.add(el.index.getConvertsw());
	ri.add(el.index.getConverttime("%.2f"));
	ri.add(el.index.getTimeindex("%.2f"));
	if (doclassify) {
	    ri.add(el.index.getTimeclass("%.2f"));
	}
	retlist.add(ri);
	retlist.add(new ResultItem("Indexed " + dbfilename + " " + md5 + " " + retsize + " " + el.convertsw + " " + time));
    try {
		inputStream.close();
	} catch (IOException e) {
		// TODO Auto-generated catch block
		log.error("Exception", e);
	}
    }
    Queues.decIndexs();
    
	}

    public static void indexme(String type) {
	SearchDao.indexme(type);
    }

    public static ResultItem[] searchme(String type, String str) {
		ResultItem[] strarr = new ResultItem[0];
		strarr = SearchDao.searchme(type, str);
    return strarr;
}

    public static ResultItem[] searchme2(String str, String searchtype) {
	String type = "all";
	int stype = new Integer(searchtype).intValue();
		ResultItem[] strarr = new ResultItem[0];
		
		strarr = SearchDao.searchme2(str, searchtype);
    return strarr;
}

    // not yet usable, lacking termvector
    public static ResultItem[] searchsimilar(String md5i) {
	String type = "all";
		ResultItem[] strarr = new ResultItem[0];
    return strarr;
}

    // not yet usable, lacking termvector
    public static void docsLike(int id, int max) throws IOException {
    }

    public static void deleteme(String str) {
    }

    // outdated, did run once, had a bug which made duplicates
    public static List<String> removeDuplicate() throws Exception {
	return null;
    }//End of removeDuplicate method

    // outdated, used once, when bug added filename instead of md5
    public static List<String> cleanup2() throws Exception {
	return null;
    }//End of removeDuplicate method

}
<commit_msg>tiny cleanup<commit_after>package roart.search;

import roart.model.IndexFiles;
import roart.model.HibernateUtil;
import roart.queue.IndexQueueElement;
import roart.queue.Queues;
import roart.lang.LanguageDetect;
import roart.model.ResultItem;

import roart.dao.SearchDao;

import java.io.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.Map;
import java.util.HashMap;
import java.util.List;
import java.util.HashSet;
 
import org.apache.tika.metadata.Metadata;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class Search {
    private static Log log = LogFactory.getLog("Search");

    //public static int indexme(String type, String md5, InputStream inputStream) {
    public static void indexme() {
    	IndexQueueElement el = Queues.indexQueue.poll();
    	if (el == null) {
    		log.error("empty queue");
    	    return;
    	}
    	// vulnerable spot
    	Queues.incIndexs();
    	long now = System.currentTimeMillis();
    	
    	String type = el.type;
     	String md5 = el.md5;
    	InputStream inputStream = el.inputStream;
    	IndexFiles dbindex = el.index;
    	String dbfilename = el.dbfilename;
	Metadata metadata = el.metadata;
	String lang = el.lang;
	String content = el.content;
	String classification = el.index.getClassification();
    	List<ResultItem> retlist = el.retlist;

    int retsize = 0;

    retsize = SearchDao.indexme(type, md5, inputStream, dbfilename, metadata.toString(), lang, content, classification, retlist);

    if (retsize < 0) {
	dbindex.setNoindexreason("Exception"); // later, propagate the exception
    } else {

    log.info("size2 " + retsize);
	el.size = retsize;
	dbindex.setIndexed(Boolean.TRUE);
	dbindex.setTimestamp("" + System.currentTimeMillis());
	dbindex.setConvertsw(el.convertsw);
	//dbindex.save();
	long time = System.currentTimeMillis() - now;
	dbindex.setTimeindex(time);
	log.info("timerStop filename " + time);

	String myclassify = roart.util.Prop.getProp().getProperty("myclassify");
	boolean doclassify = myclassify != null && myclassify.length() > 0;

	ResultItem ri = new ResultItem();
	ri.add(md5);
	ri.add(dbfilename);
	ri.add(lang);
	if (doclassify) {
	    ri.add(el.index.getClassification());
	}
	ri.add(el.index.getTimestamp());
	ri.add(el.index.getConvertsw());
	ri.add(el.index.getConverttime("%.2f"));
	ri.add(el.index.getTimeindex("%.2f"));
	if (doclassify) {
	    ri.add(el.index.getTimeclass("%.2f"));
	}
	retlist.add(ri);
    try {
		inputStream.close();
	} catch (IOException e) {
		// TODO Auto-generated catch block
		log.error("Exception", e);
	}
    }
    Queues.decIndexs();
    
	}

    public static void indexme(String type) {
	SearchDao.indexme(type);
    }

    public static ResultItem[] searchme(String type, String str) {
		ResultItem[] strarr = new ResultItem[0];
		strarr = SearchDao.searchme(type, str);
    return strarr;
}

    public static ResultItem[] searchme2(String str, String searchtype) {
	String type = "all";
	int stype = new Integer(searchtype).intValue();
		ResultItem[] strarr = new ResultItem[0];
		
		strarr = SearchDao.searchme2(str, searchtype);
    return strarr;
}

    // not yet usable, lacking termvector
    public static ResultItem[] searchsimilar(String md5i) {
	String type = "all";
		ResultItem[] strarr = new ResultItem[0];
    return strarr;
}

    // not yet usable, lacking termvector
    public static void docsLike(int id, int max) throws IOException {
    }

    public static void deleteme(String str) {
    }

    // outdated, did run once, had a bug which made duplicates
    public static List<String> removeDuplicate() throws Exception {
	return null;
    }//End of removeDuplicate method

    // outdated, used once, when bug added filename instead of md5
    public static List<String> cleanup2() throws Exception {
	return null;
    }//End of removeDuplicate method

}

{'fraction_non_alphanumeric': 0.12639311043566362, 'fraction_numerical': 0.005192502532928065, 'mean_word_length': 3.72874251497006, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3938690', 'n_tokens_mistral': 2641, 'n_tokens_neox': 2571, 'n_words': 622}