<commit_before>/*
   Copyright 2013 Nationale-Nederlanden

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package nl.nn.adapterframework.pipes;

import java.io.File;

import nl.nn.adapterframework.configuration.ConfigurationException;
import nl.nn.adapterframework.core.IPipeLineSession;
import nl.nn.adapterframework.core.PipeRunException;
import nl.nn.adapterframework.core.PipeRunResult;
import nl.nn.adapterframework.util.FileUtils;
import nl.nn.adapterframework.util.WildCardFilter;

import org.apache.commons.lang.StringUtils;

/**
 * Pipe for moving files to another directory.
 *
 * <p><b>Configuration:</b>
 * <table border="1">
 * <tr><th>attributes</th><th>description</th><th>default</th></tr>
 * <tr><td>classname</td><td>nl.nn.ibis4fundation.FtpSender</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setName(String) name}</td><td>name of the sender</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setDirectory(String) directory}</td><td>base directory where files are moved from</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setFilename(String) filename}</td><td>name of the file to move (if not specified, the input for this pipe is assumed to be the name of the file</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setWildcard(String) wildcard}</td><td>filter of files to replace</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setWildcardSessionKey(String) wildcardSessionKey}</td><td>session key that contains the name of the filter to use (only used if wildcard is not set)</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setMove2dir(String) move2dir}</td><td>destination directory</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setMove2file(String) move2file}</td><td>name of the destination file (if not specified, the name of the file to move is taken)</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setMove2fileSessionKey(String) move2fileSessionKey}</td><td>session key that contains the name of the file to use (only used if move2file is not set)</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setNumberOfBackups(int) numberOfBackups}</td><td>number of copies held of a file with the same name. Backup files have a dot and a number suffixed to their name. If set to 0, no backups will be kept.</td><td>5</td></tr>
 * <tr><td>{@link #setOverwrite(boolean) overwrite}</td><td>when set <code>true</code>, the destination file will be deleted if it already exists</td><td>false</td></tr>
 * <tr><td>{@link #setNumberOfAttempts(int) numberOfAttempts}</td><td>maximum number of attempts before throwing an exception</td><td>10</td></tr>
 * <tr><td>{@link #setWaitBeforeRetry(long) waitBeforeRetry}</td><td>time between attempts</td><td>1000 [ms]</td></tr>
 * <tr><td>{@link #setAppend(boolean) append}</td><td> when set <code>true</code> and the destination file already exists, the content of the file to move is written to the end of the destination file. This implies <code>overwrite=false</code></td><td>false</td></tr>
 * <tr><td>{@link #setDeleteEmptyDirectory(boolean) deleteEmptyDirectory}</td><td>when set to <code>true</code>, the directory from which a file is moved is deleted when it contains no other files</td><td>false</td></tr>
 * <tr><td>{@link #setCreateDirectory(boolean) createDirectory}</td><td>when set to <code>true</code>, the directory to move to is created if it does not exist</td><td>false</td></tr>
 * <tr><td>{@link #setPrefix(String) prefix}</td><td>string which is inserted at the start of the destination file</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setSuffix(String) suffix}</td><td>string which is inserted at the end of the destination file (and replaces the extension if present)</td><td>&nbsp;</td></tr>
 * </table>
 * </p>
 * 
 * @author  <NAME>
 * @author  <NAME> (***<EMAIL>)
 * @author  <NAME>
 */
public class MoveFilePipe extends FixedForwardPipe {

	private String directory;
	private String filename;
	private String wildcard;
	private String wildcardSessionKey;
	private String move2dir;
	private String move2file;
	protected String move2fileSessionKey;
	private int numberOfAttempts = 10;
	private long waitBeforeRetry = 1000;
	private int numberOfBackups = 5;
	private boolean overwrite = false;
	private boolean append = false;
	private boolean deleteEmptyDirectory = false;
	private boolean createDirectory = false;
	private String prefix;
	private String suffix;
	
	public void configure() throws ConfigurationException {
		super.configure();
		if (isAppend()) {
			setOverwrite(false);
		}
		if (StringUtils.isEmpty(getMove2dir())) {
			throw new ConfigurationException("Property [move2dir] is not set");
		}
	}
	
	/** 
* @see nl.nn.adapterframework.core.IPipe#doPipe(Object, IPipeLineSession)
	 */
	public PipeRunResult doPipe(Object input, IPipeLineSession session) throws PipeRunException {
		File srcFile=null;
		File dstFile=null;

		if (StringUtils.isEmpty(getWildcard()) && StringUtils.isEmpty(getWildcardSessionKey())) {
			if (StringUtils.isEmpty(getDirectory())) {
				if (StringUtils.isEmpty(getFilename())) {
					srcFile = new File(input.toString());
				} else {
					srcFile = new File(getFilename());
				}
			} else {
				if (StringUtils.isEmpty(getFilename())) {
					srcFile = new File(getDirectory(), input.toString());
				} else {
					srcFile = new File(getDirectory(), getFilename());
				}
			}
			if (StringUtils.isEmpty(getMove2file())) {
				if (StringUtils.isEmpty(getMove2fileSessionKey())) {
					dstFile = new File(getMove2dir(), retrieveDestinationChild(srcFile.getName()));
				} else {
					dstFile = new File(getMove2dir(), retrieveDestinationChild((String)session.get(getMove2fileSessionKey())));
				}
			} else {
				dstFile = new File(getMove2dir(), retrieveDestinationChild(getMove2file()));
			}
			moveFile(session, srcFile, dstFile);
		} else {
			if (StringUtils.isEmpty(getDirectory())) {
				if (StringUtils.isEmpty(getFilename())) {
					srcFile = new File(input.toString());
				} else {
					srcFile = new File(getFilename());
				}
			} else {
				srcFile = new File(getDirectory());
			}
			String wc;
			if (StringUtils.isEmpty(getWildcardSessionKey())) {
				wc = getWildcard();
			} else {
				wc = (String)session.get(getWildcardSessionKey());
			}
			WildCardFilter filter = new WildCardFilter(wc);
			File[] srcFiles = srcFile.listFiles(filter);
			int count = (srcFiles == null ? 0 : srcFiles.length);
			if (count==0) {
				log.info(getLogPrefix(session) + "no files with wildcard [" + wc + "] found in directory [" + srcFile.getAbsolutePath() +"]");
			}
			for (int i = 0; i < count; i++) {
				dstFile = new File(getMove2dir(), retrieveDestinationChild(srcFiles[i].getName()));
				moveFile(session, srcFiles[i], dstFile);
			}
		}

		/* if parent source directory is empty, delete the directory */
		if (isDeleteEmptyDirectory()) {
			File srcDirectory;
			if (srcFile.isDirectory()) {
				srcDirectory = srcFile.getAbsoluteFile();
			} else {
				srcDirectory = srcFile.getParentFile();
			}
			if (srcDirectory.exists() && srcDirectory.list().length==0) {
				boolean success = srcDirectory.delete();
				if (!success){
				   log.warn( getLogPrefix(session) + "could not delete directory [" + srcDirectory.getAbsolutePath() +"]");
				} 
				else {
				   log.info(getLogPrefix(session) + "deleted directory [" + srcDirectory.getAbsolutePath() +"]");
				} 
			} else {
				   log.info(getLogPrefix(session) + "directory [" + srcDirectory.getAbsolutePath() +"] doesn't exist or is not empty");
			}
		}
		<commit_msg>fixed bug for deleteEmptyDirectory attribute when using wildcard<commit_after>/*
   Copyright 2013 Nationale-Nederlanden

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package nl.nn.adapterframework.pipes;

import java.io.File;

import nl.nn.adapterframework.configuration.ConfigurationException;
import nl.nn.adapterframework.core.IPipeLineSession;
import nl.nn.adapterframework.core.PipeRunException;
import nl.nn.adapterframework.core.PipeRunResult;
import nl.nn.adapterframework.util.FileUtils;
import nl.nn.adapterframework.util.WildCardFilter;

import org.apache.commons.lang.StringUtils;

/**
 * Pipe for moving files to another directory.
 *
 * <p><b>Configuration:</b>
 * <table border="1">
 * <tr><th>attributes</th><th>description</th><th>default</th></tr>
 * <tr><td>classname</td><td>nl.nn.ibis4fundation.FtpSender</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setName(String) name}</td><td>name of the sender</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setDirectory(String) directory}</td><td>base directory where files are moved from</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setFilename(String) filename}</td><td>name of the file to move (if not specified, the input for this pipe is assumed to be the name of the file</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setWildcard(String) wildcard}</td><td>filter of files to replace</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setWildcardSessionKey(String) wildcardSessionKey}</td><td>session key that contains the name of the filter to use (only used if wildcard is not set)</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setMove2dir(String) move2dir}</td><td>destination directory</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setMove2file(String) move2file}</td><td>name of the destination file (if not specified, the name of the file to move is taken)</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setMove2fileSessionKey(String) move2fileSessionKey}</td><td>session key that contains the name of the file to use (only used if move2file is not set)</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setNumberOfBackups(int) numberOfBackups}</td><td>number of copies held of a file with the same name. Backup files have a dot and a number suffixed to their name. If set to 0, no backups will be kept.</td><td>5</td></tr>
 * <tr><td>{@link #setOverwrite(boolean) overwrite}</td><td>when set <code>true</code>, the destination file will be deleted if it already exists</td><td>false</td></tr>
 * <tr><td>{@link #setNumberOfAttempts(int) numberOfAttempts}</td><td>maximum number of attempts before throwing an exception</td><td>10</td></tr>
 * <tr><td>{@link #setWaitBeforeRetry(long) waitBeforeRetry}</td><td>time between attempts</td><td>1000 [ms]</td></tr>
 * <tr><td>{@link #setAppend(boolean) append}</td><td> when set <code>true</code> and the destination file already exists, the content of the file to move is written to the end of the destination file. This implies <code>overwrite=false</code></td><td>false</td></tr>
 * <tr><td>{@link #setDeleteEmptyDirectory(boolean) deleteEmptyDirectory}</td><td>when set to <code>true</code>, the directory from which a file is moved is deleted when it contains no other files</td><td>false</td></tr>
 * <tr><td>{@link #setCreateDirectory(boolean) createDirectory}</td><td>when set to <code>true</code>, the directory to move to is created if it does not exist</td><td>false</td></tr>
 * <tr><td>{@link #setPrefix(String) prefix}</td><td>string which is inserted at the start of the destination file</td><td>&nbsp;</td></tr>
 * <tr><td>{@link #setSuffix(String) suffix}</td><td>string which is inserted at the end of the destination file (and replaces the extension if present)</td><td>&nbsp;</td></tr>
 * </table>
 * </p>
 * 
 * @author  <NAME>
 * @author  <NAME> (***<EMAIL>)
 * @author  <NAME>
 */
public class MoveFilePipe extends FixedForwardPipe {

	private String directory;
	private String filename;
	private String wildcard;
	private String wildcardSessionKey;
	private String move2dir;
	private String move2file;
	protected String move2fileSessionKey;
	private int numberOfAttempts = 10;
	private long waitBeforeRetry = 1000;
	private int numberOfBackups = 5;
	private boolean overwrite = false;
	private boolean append = false;
	private boolean deleteEmptyDirectory = false;
	private boolean createDirectory = false;
	private String prefix;
	private String suffix;
	
	public void configure() throws ConfigurationException {
		super.configure();
		if (isAppend()) {
			setOverwrite(false);
		}
		if (StringUtils.isEmpty(getMove2dir())) {
			throw new ConfigurationException("Property [move2dir] is not set");
		}
	}
	
	/** 
* @see nl.nn.adapterframework.core.IPipe#doPipe(Object, IPipeLineSession)
	 */
	public PipeRunResult doPipe(Object input, IPipeLineSession session) throws PipeRunException {
		File srcFile=null;
		File dstFile=null;

		if (StringUtils.isEmpty(getWildcard()) && StringUtils.isEmpty(getWildcardSessionKey())) {
			if (StringUtils.isEmpty(getDirectory())) {
				if (StringUtils.isEmpty(getFilename())) {
					srcFile = new File(input.toString());
				} else {
					srcFile = new File(getFilename());
				}
			} else {
				if (StringUtils.isEmpty(getFilename())) {
					srcFile = new File(getDirectory(), input.toString());
				} else {
					srcFile = new File(getDirectory(), getFilename());
				}
			}
			if (StringUtils.isEmpty(getMove2file())) {
				if (StringUtils.isEmpty(getMove2fileSessionKey())) {
					dstFile = new File(getMove2dir(), retrieveDestinationChild(srcFile.getName()));
				} else {
					dstFile = new File(getMove2dir(), retrieveDestinationChild((String)session.get(getMove2fileSessionKey())));
				}
			} else {
				dstFile = new File(getMove2dir(), retrieveDestinationChild(getMove2file()));
			}
			moveFile(session, srcFile, dstFile);
		} else {
			if (StringUtils.isEmpty(getDirectory())) {
				if (StringUtils.isEmpty(getFilename())) {
					srcFile = new File(input.toString());
				} else {
					srcFile = new File(getFilename());
				}
			} else {
				srcFile = new File(getDirectory());
			}
			String wc;
			if (StringUtils.isEmpty(getWildcardSessionKey())) {
				wc = getWildcard();
			} else {
				wc = (String)session.get(getWildcardSessionKey());
			}
			WildCardFilter filter = new WildCardFilter(wc);
			File[] srcFiles = srcFile.listFiles(filter);
			int count = (srcFiles == null ? 0 : srcFiles.length);
			if (count==0) {
				log.info(getLogPrefix(session) + "no files with wildcard [" + wc + "] found in directory [" + srcFile.getAbsolutePath() +"]");
			}
			for (int i = 0; i < count; i++) {
				dstFile = new File(getMove2dir(), retrieveDestinationChild(srcFiles[i].getName()));
				moveFile(session, srcFiles[i], dstFile);
			}
		}

		/* if parent source directory is empty, delete the directory */
		if (isDeleteEmptyDirectory()) {
			File srcDirectory;
			if (StringUtils.isEmpty(getWildcard()) && StringUtils.isEmpty(getWildcardSessionKey())) {
				srcDirectory = srcFile.getParentFile();
			} else {
				srcDirectory = srcFile.getAbsoluteFile();
			}
			log.debug("srcFile ["+srcFile.getPath()+"] srcDirectory ["+srcDirectory.getPath()+"]");
			if (srcDirectory.exists()) {
				if (srcDirectory.list().length==0) {
					boolean success = srcDirectory.delete();
					if (!success){
					   log.warn( getLogPrefix(session) + "could not delete directory [" + srcDirectory.getAbsolutePath() +"]");
					} 
					else {
					   log.info(getLogPrefix(session) + "deleted directory [" + srcDirectory.getAbsolutePath() +"]");
					} 
				} else {
					   log.info(getLogPrefix(session) + "directory [" + srcDirectory.getAbsolutePath() +"] is not empty");
				}
			} else {
				   log.info(getLogPrefix(session) + "directory [" + srcDirectory.getAbsolutePath() +"] doesn't exist");
			}
		}
		
{'fraction_non_alphanumeric': 0.15028081219527248, 'fraction_numerical': 0.0060482626674072705, 'mean_word_length': 4.7440623892236795, 'pattern_counts': {'":': 0, '<': 349, '<?xml version=': 0, '>': 347, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '374639', 'n_tokens_mistral': 5368, 'n_tokens_neox': 4923, 'n_words': 1322}