<commit_before>package roart.content;

import java.io.File;
import java.util.Date;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import roart.dir.Traverse;
import roart.model.IndexFiles;
import roart.model.ResultItem;
import roart.queue.Queues;
import roart.queue.TikaQueueElement;
import roart.util.Constants;
import roart.util.ExecCommand;
import roart.filesystem.FileSystemDao;

import org.apache.tika.metadata.Metadata;

public class OtherHandler {
	
	private static Logger log = LoggerFactory.getLogger(OtherHandler.class);

    public static volatile int timeout = 3600;
	
    public static void doOther()  {
    	TikaQueueElement el = Queues.otherQueue.poll();
    	if (el == null) {
    		log.error("empty queue");
    	    return;
    	}
    	// vulnerable spot
    	Queues.incOthers();
    	long now = System.currentTimeMillis();
    	
    	String dbfilename = el.dbfilename;
    	String filename = el.filename;
    	String md5 = el.md5;
    	IndexFiles index = el.index;
    	List<ResultItem> retlist = el.retlist;
    	List<ResultItem> retlistnot = el.retlistnot;
	Metadata metadata = el.metadata;

	if (filename.startsWith(FileSystemDao.FILE)) {
	    filename = filename.substring(5);
	}

	String output = null;
	boolean retry = false;
	String lowercase = filename.toLowerCase();
	File temp = null;
	String tmp = null;
    try {
	temp = File.createTempFile("other", ".txt");
    tmp = temp.getAbsolutePath();
    } catch (Exception e) {
     log.error(Constants.EXCEPTION, e);
    }
    String mimetype = el.mimetype;
	// epub 2nd try
    // ebook-convert tries all, no fully good mime software
    // (lowercase.endsWith(".mobi") || lowercase.endsWith(".pdb") || lowercase.endsWith(".epub") || lowercase.endsWith(".lit") || lowercase.endsWith(".djvu") || lowercase.endsWith(".djv") || lowercase.endsWith(".dj") || lowercase.endsWith(".chm") || lowercase.endsWith(".docx"))
    if (true || mimetype.equals("application/x-mobipocket-ebook") || mimetype.equals("application/x-aportisdoc") || mimetype.equals("application/epub+zip") || mimetype.equals("image/vnd.djvu") || mimetype.equals("application/vnd.ms-htmlhelp") || mimetype.equals("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) {
	    File file = new File(filename);
	    String dirname = file.getParent();
	    File dir = new File(dirname);
	    boolean w = dir.canWrite();
	    if (!w) {
		dir.setWritable(true);<commit_msg>return if exception, can not continue<commit_after>package roart.content;

import java.io.File;
import java.util.Date;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import roart.dir.Traverse;
import roart.model.IndexFiles;
import roart.model.ResultItem;
import roart.queue.Queues;
import roart.queue.TikaQueueElement;
import roart.util.Constants;
import roart.util.ExecCommand;
import roart.filesystem.FileSystemDao;

import org.apache.tika.metadata.Metadata;

public class OtherHandler {
	
	private static Logger log = LoggerFactory.getLogger(OtherHandler.class);

    public static volatile int timeout = 3600;
	
    public static void doOther()  {
    	TikaQueueElement el = Queues.otherQueue.poll();
    	if (el == null) {
    		log.error("empty queue");
    	    return;
    	}
    	// vulnerable spot
    	Queues.incOthers();
    	long now = System.currentTimeMillis();
    	
    	String dbfilename = el.dbfilename;
    	String filename = el.filename;
    	String md5 = el.md5;
    	IndexFiles index = el.index;
    	List<ResultItem> retlist = el.retlist;
    	List<ResultItem> retlistnot = el.retlistnot;
	Metadata metadata = el.metadata;

	if (filename.startsWith(FileSystemDao.FILE)) {
	    filename = filename.substring(5);
	}

	String output = null;
	boolean retry = false;
	String lowercase = filename.toLowerCase();
	File temp = null;
	String tmp = null;
    try {
	temp = File.createTempFile("other", ".txt");
    tmp = temp.getAbsolutePath();
    } catch (Exception e) {
     log.error(Constants.EXCEPTION, e);
     Queues.decOthers();
     return;
    }
    String mimetype = el.mimetype;
	// epub 2nd try
    // ebook-convert tries all, no fully good mime software
    // (lowercase.endsWith(".mobi") || lowercase.endsWith(".pdb") || lowercase.endsWith(".epub") || lowercase.endsWith(".lit") || lowercase.endsWith(".djvu") || lowercase.endsWith(".djv") || lowercase.endsWith(".dj") || lowercase.endsWith(".chm") || lowercase.endsWith(".docx"))
    if (true || mimetype.equals("application/x-mobipocket-ebook") || mimetype.equals("application/x-aportisdoc") || mimetype.equals("application/epub+zip") || mimetype.equals("image/vnd.djvu") || mimetype.equals("application/vnd.ms-htmlhelp") || mimetype.equals("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) {
	    File file = new File(filename);
	    String dirname = file.getParent();
	    File dir = new File(dirname);
	    boolean w = dir.canWrite();
	    if (!w) {
		dir.setWritable(true);
{'fraction_non_alphanumeric': 0.132815259625574, 'fraction_numerical': 0.0035323207347227126, 'mean_word_length': 4.106402164111812, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2131547', 'n_tokens_mistral': 1842, 'n_tokens_neox': 1780, 'n_words': 362}