<commit_before>/*
 * $Log: CleanupOldFilesPipe.java,v $
 * Revision 1.3  2007-07-10 07:29:33  europe\L190409
 * improved logging
 *
 * Revision 1.2  2007/01/02 09:56:59  <NAME> <<EMAIL>>
 * updated javadoc
 *
 * Revision 1.1  2006/08/24 07:10:36  <NAME> <<EMAIL>>
 * moved CleanupOldFilesPipe to pipes-package
 *
 * Revision 1.5  2006/05/19 09:28:36  <NAME> <<EMAIL>>
 * Restore java files from batch package after unwanted deletion.
 *
 * Revision 1.3  2005/12/20 09:57:20  <NAME> <<EMAIL>>
 * updated javadoc
 *
 * Revision 1.2  2005/11/08 09:31:08  <NAME> <<EMAIL>>
 * Bug concerning filenames resolved
 *
 * Revision 1.1  2005/11/01 08:54:00  <NAME> <<EMAIL>>
 * Initial version
 *
 */
package nl.nn.adapterframework.pipes;

import java.io.File;
import java.io.FileFilter;
import java.util.ArrayList;
import java.util.Iterator;

import nl.nn.adapterframework.configuration.ConfigurationException;
import nl.nn.adapterframework.core.PipeLineSession;
import nl.nn.adapterframework.core.PipeRunException;
import nl.nn.adapterframework.core.PipeRunResult;
import nl.nn.adapterframework.util.FileUtils;

import org.apache.commons.lang.StringUtils;

/**
 * Pipe for deleting files.
 *
 * <p><b>Configuration:</b>
 * <table border="1">
 * <tr><th>attributes</th><th>description</th><th>default</th></tr>
 * <tr><td>classname</td><td>nl.nn.adapterframework.batch.CleanupOldFilesPipe</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setName(String) name}</td><td>name of the Pipe</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setMaxThreads(int) maxThreads}</td><td>maximum number of threads that may call {@link #doPipe(Object, nl.nn.adapterframework.core.PipeLineSession)} simultaneously</td><td>0 (unlimited)</td></tr>
 * <tr><td>{@link #setFilePattern(String) filePattern}</td><td>files that match this pattern will be deleted. Parameters of the pipe are applied to this pattern</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setSubdirectories(boolean) subdirectories}</td><td>when <code>true</code>, files  in subdirectories will be deleted, too</td><td>false</td></tr>
 * <tr><td>{@link #setLastModifiedDelta(long) lastModifiedDelta}</td><td>time in milliseconds that must have passed at least before a file will be deleted</td><td>&nbsp;</td></tr>
 * </table>
 * </p>
 * 
 * @author: <NAME>
 * @since:  4.2
 */
public class CleanupOldFilesPipe extends FixedForwardPipe {
	public static final String version = "$RCSfile: CleanupOldFilesPipe.java,v $  $Revision: 1.3 $ $Date: 2007-07-10 07:29:33 $";
	
	private String filePattern;
	private boolean subdirectories;
	private long lastModifiedDelta;

	private _FileFilter fileFilter = new _FileFilter();
	private _DirFilter dirFilter = new _DirFilter();
		
	public CleanupOldFilesPipe() {
	}
	
	public void configure() throws ConfigurationException {
		super.configure();
		
		if (StringUtils.isEmpty(getFilePattern())) {
			throw new ConfigurationException("Property [filePattern] is not set");
		}
	}
	
	/** 
	 * @see nl.nn.adapterframework.core.IPipe#doPipe(Object, PipeLineSession)
	 */
	public PipeRunResult doPipe(Object input, PipeLineSession session) throws PipeRunException {
		try {
			File in = (input == null) ? null : new File(input.toString());
			String filename = FileUtils.getFilename(null, session, in, getFilePattern());
			ArrayList delFiles = getFilesForDeletion(filename);
			if (delFiles != null && delFiles.size() > 0) {
				for (Iterator fileIt = delFiles.iterator(); fileIt.hasNext();) {
					File file = (File)fileIt.next();
					String curfilename=file.getName();
					file.delete();
					log.info(getLogPrefix(session)+"deleted file ["+curfilename+"]");
				}
			} else {
				log.info(getLogPrefix(session)+"no files match pattern ["+filename+"]");
			}
			return new PipeRunResult(getForward(), input);
		}
		catch(Exception e) {
			throw new PipeRunException(this, "Error while deleting file(s)", e); 
		}
	}

	private ArrayList getFilesForDeletion(String filename) {
		File file = new File(filename);
		if (file.exists()) {<commit_msg>fix switching between by input and by pattern<commit_after>/*
 * $Log: CleanupOldFilesPipe.java,v $
 * Revision 1.4  2007-07-10 15:17:01  europe\L190409
 * fix switching between by input and by pattern
 *
 * Revision 1.3  2007/07/10 07:29:33  <NAME> <<EMAIL>>
 * improved logging
 *
 * Revision 1.2  2007/01/02 09:56:59  <NAME> <<EMAIL>>
 * updated javadoc
 *
 * Revision 1.1  2006/08/24 07:10:36  <NAME> <<EMAIL>>
 * moved CleanupOldFilesPipe to pipes-package
 *
 * Revision 1.5  2006/05/19 09:28:36  <NAME> <<EMAIL>>
 * Restore java files from batch package after unwanted deletion.
 *
 * Revision 1.3  2005/12/20 09:57:20  <NAME> <<EMAIL>>
 * updated javadoc
 *
 * Revision 1.2  2005/11/08 09:31:08  <NAME> <<EMAIL>>
 * Bug concerning filenames resolved
 *
 * Revision 1.1  2005/11/01 08:54:00  <NAME> <<EMAIL>>
 * Initial version
 *
 */
package nl.nn.adapterframework.pipes;

import java.io.File;
import java.io.FileFilter;
import java.util.ArrayList;
import java.util.Iterator;

import nl.nn.adapterframework.configuration.ConfigurationException;
import nl.nn.adapterframework.core.PipeLineSession;
import nl.nn.adapterframework.core.PipeRunException;
import nl.nn.adapterframework.core.PipeRunResult;
import nl.nn.adapterframework.parameters.ParameterList;
import nl.nn.adapterframework.util.FileUtils;

import org.apache.commons.lang.StringUtils;

/**
 * Pipe for deleting files.
 *
 * <p><b>Configuration:</b>
 * <table border="1">
 * <tr><th>attributes</th><th>description</th><th>default</th></tr>
 * <tr><td>classname</td><td>nl.nn.adapterframework.batch.CleanupOldFilesPipe</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setName(String) name}</td><td>name of the Pipe</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setMaxThreads(int) maxThreads}</td><td>maximum number of threads that may call {@link #doPipe(Object, nl.nn.adapterframework.core.PipeLineSession)} simultaneously</td><td>0 (unlimited)</td></tr>
 * <tr><td>{@link #setFilePattern(String) filePattern}</td><td>files that match this pattern will be deleted. Parameters of the pipe are applied to this pattern. If this attribute is not set, the input of the pipe is interpreted as the file to be removed</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setSubdirectories(boolean) subdirectories}</td><td>when <code>true</code>, files  in subdirectories will be deleted, too</td><td>false</td></tr>
 * <tr><td>{@link #setLastModifiedDelta(long) lastModifiedDelta}</td><td>time in milliseconds that must have passed at least before a file will be deleted</td><td>0</td></tr>
 * </table>
 * </p>
 * 
 * @author: <NAME>
 * @since:  4.2
 */
public class CleanupOldFilesPipe extends FixedForwardPipe {
	public static final String version = "$RCSfile: CleanupOldFilesPipe.java,v $  $Revision: 1.4 $ $Date: 2007-07-10 15:17:01 $";
	
	private String filePattern;
	private boolean subdirectories=false;
	private long lastModifiedDelta=0;

	private _FileFilter fileFilter = new _FileFilter();
	private _DirFilter dirFilter = new _DirFilter();
		
		
	public PipeRunResult doPipe(Object input, PipeLineSession session) throws PipeRunException {
		try {
			String filename;
			if (StringUtils.isNotEmpty(getFilePattern())) {
				filename = FileUtils.getFilename(getParameterList(), session, "", getFilePattern());
			} else {
				if (StringUtils.isEmpty((String)input)) {
					throw new PipeRunException(this, "input empty, but should contain filename to delete");
				} else {
					File in = new File(input.toString());
					filename = in.getName();
				}
			}
			
			ArrayList delFiles = getFilesForDeletion(filename);
			if (delFiles != null && delFiles.size() > 0) {
				for (Iterator fileIt = delFiles.iterator(); fileIt.hasNext();) {
					File file = (File)fileIt.next();
					String curfilename=file.getName();
					file.delete();
					log.info(getLogPrefix(session)+"deleted file ["+file.getAbsolutePath()+"]");
				}
			} else {
				log.info(getLogPrefix(session)+"no files match pattern ["+filename+"]");
			}
			return new PipeRunResult(getForward(), input);
		}
		catch(Exception e) {
			throw new PipeRunException(this, "Error while deleting file(s)", e); 
		}
	}

	private ArrayList getFilesForDeletion(String filename) {
		File file = new File(filename);
		if (file.exists()) {
{'fraction_non_alphanumeric': 0.15188644910833435, 'fraction_numerical': 0.035909256338711636, 'mean_word_length': 4.7051903114186855, 'pattern_counts': {'":': 0, '<': 172, '<?xml version=': 0, '>': 174, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3834311', 'n_tokens_mistral': 3044, 'n_tokens_neox': 2703, 'n_words': 644}