<commit_before>package be.ibridge.kettle.trans.step.fileinput;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.commons.vfs.FileDepthSelector;
import org.apache.commons.vfs.FileObject;
import org.apache.commons.vfs.FileSelectInfo;

import be.ibridge.kettle.core.Const;
import be.ibridge.kettle.core.LogWriter;
import be.ibridge.kettle.core.util.StringUtil;
import be.ibridge.kettle.core.vfs.KettleVFS;

public class FileInputList
{
    private List                files              = new ArrayList();
    private List                nonExistantFiles   = new ArrayList(1);
    private List                nonAccessibleFiles = new ArrayList(1);
    
    private static final String YES                = "Y";

    public static String getRequiredFilesDescription(List nonExistantFiles)
    {
        StringBuffer buffer = new StringBuffer();
        for (Iterator iter = nonExistantFiles.iterator(); iter.hasNext();)
        {
            FileObject file = (FileObject) iter.next();
            buffer.append(file.getName().getURI());
            buffer.append(Const.CR);
        }
        return buffer.toString();
    }

    private static boolean[] includeSubdirsFalse(int iLength)
    {
        boolean[] includeSubdirs = new boolean[iLength];
        for (int i = 0; i < iLength; i++)
            includeSubdirs[i] = false;
        return includeSubdirs;
    }
    
    public static String[] createFilePathList(String[] fileName, String[] fileMask, String[] fileRequired)
    {
        boolean[] includeSubdirs = includeSubdirsFalse(fileName.length);
        return createFilePathList(fileName, fileMask, fileRequired, includeSubdirs);
    }
    
    public static String[] createFilePathList(String[] fileName, String[] fileMask, String[] fileRequired,
        boolean[] includeSubdirs)
    {
        List fileList = createFileList(fileName, fileMask, fileRequired, includeSubdirs).getFiles();
        String[] filePaths = new String[fileList.size()];
        for (int i = 0; i < filePaths.length; i++)
        {
            filePaths[i] = ((FileObject) fileList.get(i)).getName().getURI();
            // filePaths[i] = KettleVFS.getFilename((FileObject) fileList.get(i));
        }
        return filePaths;
    }

    public static FileInputList createFileList(String[] fileName, String[] fileMask, String[] fileRequired)
    {
        boolean[] includeSubdirs = includeSubdirsFalse(fileName.length);
        return createFileList(fileName, fileMask, fileRequired, includeSubdirs);
    }
    
    public static FileInputList createFileList(String[] fileName, String[] fileMask, String[] fileRequired, boolean[] includeSubdirs)
    {
        FileInputList fileInputList = new FileInputList();

        // Replace possible environment variables...
        final String realfile[] = StringUtil.environmentSubstitute(fileName);
        final String realmask[] = StringUtil.environmentSubstitute(fileMask);

        for (int i = 0; i < realfile.length; i++)
        {
            final String onefile = realfile[i];
            final String onemask = realmask[i];
            final boolean onerequired = YES.equalsIgnoreCase(fileRequired[i]);
            final boolean subdirs = includeSubdirs[i];
            
            if (onefile == null) continue;

            // 
            // If a wildcard is set we search for files
            //
            if (!Const.isEmpty(onemask))
            {
                try
                {<commit_msg>Don't consider empty directories (from grid input we get "")<commit_after>package be.ibridge.kettle.trans.step.fileinput;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.commons.vfs.FileDepthSelector;
import org.apache.commons.vfs.FileObject;
import org.apache.commons.vfs.FileSelectInfo;

import be.ibridge.kettle.core.Const;
import be.ibridge.kettle.core.LogWriter;
import be.ibridge.kettle.core.util.StringUtil;
import be.ibridge.kettle.core.vfs.KettleVFS;

public class FileInputList
{
    private List                files              = new ArrayList();
    private List                nonExistantFiles   = new ArrayList(1);
    private List                nonAccessibleFiles = new ArrayList(1);
    
    private static final String YES                = "Y";

    public static String getRequiredFilesDescription(List nonExistantFiles)
    {
        StringBuffer buffer = new StringBuffer();
        for (Iterator iter = nonExistantFiles.iterator(); iter.hasNext();)
        {
            FileObject file = (FileObject) iter.next();
            buffer.append(file.getName().getURI());
            buffer.append(Const.CR);
        }
        return buffer.toString();
    }

    private static boolean[] includeSubdirsFalse(int iLength)
    {
        boolean[] includeSubdirs = new boolean[iLength];
        for (int i = 0; i < iLength; i++)
            includeSubdirs[i] = false;
        return includeSubdirs;
    }
    
    public static String[] createFilePathList(String[] fileName, String[] fileMask, String[] fileRequired)
    {
        boolean[] includeSubdirs = includeSubdirsFalse(fileName.length);
        return createFilePathList(fileName, fileMask, fileRequired, includeSubdirs);
    }
    
    public static String[] createFilePathList(String[] fileName, String[] fileMask, String[] fileRequired,
        boolean[] includeSubdirs)
    {
        List fileList = createFileList(fileName, fileMask, fileRequired, includeSubdirs).getFiles();
        String[] filePaths = new String[fileList.size()];
        for (int i = 0; i < filePaths.length; i++)
        {
            filePaths[i] = ((FileObject) fileList.get(i)).getName().getURI();
            // filePaths[i] = KettleVFS.getFilename((FileObject) fileList.get(i));
        }
        return filePaths;
    }

    public static FileInputList createFileList(String[] fileName, String[] fileMask, String[] fileRequired)
    {
        boolean[] includeSubdirs = includeSubdirsFalse(fileName.length);
        return createFileList(fileName, fileMask, fileRequired, includeSubdirs);
    }
    
    public static FileInputList createFileList(String[] fileName, String[] fileMask, String[] fileRequired, boolean[] includeSubdirs)
    {
        FileInputList fileInputList = new FileInputList();

        // Replace possible environment variables...
        final String realfile[] = StringUtil.environmentSubstitute(fileName);
        final String realmask[] = StringUtil.environmentSubstitute(fileMask);

        for (int i = 0; i < realfile.length; i++)
        {
            final String onefile = realfile[i];
            final String onemask = realmask[i];
            final boolean onerequired = YES.equalsIgnoreCase(fileRequired[i]);
            final boolean subdirs = includeSubdirs[i];
            
            if (Const.isEmpty(onefile)) continue;

            // 
            // If a wildcard is set we search for files
            //
            if (!Const.isEmpty(onemask))
            {
                try
                {
{'fraction_non_alphanumeric': 0.1031502648452746, 'fraction_numerical': 0.0013939224979091162, 'mean_word_length': 2.6073403720462545, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5394649', 'n_tokens_mistral': 1976, 'n_tokens_neox': 1933, 'n_words': 464}