<commit_before>package util;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.SequenceInputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;
import java.util.Vector;

import org.apache.commons.codec.digest.DigestUtils;
import org.jfree.io.IOUtils;

public class FileUtil
{
	public static class CSVFile
	{
		public List<String> comments;
		public List<String[]> content;
	}

	public static CSVFile readCSV(String filename)
	{
		try
		{
			List<String[]> l = new ArrayList<String[]>();
			List<String> c = new ArrayList<String>();

			BufferedReader b = new BufferedReader(new FileReader(new File(filename)));
			String s = "";
			while ((s = b.readLine()) != null)
			{
				if (s.trim().length() == 0)
					continue;
				if (s.startsWith("#"))
					c.add(s);
				else
					l.add(ArrayUtil.cast(String.class, VectorUtil.fromCSVString(s, false).toArray()));
			}
			b.close();

			CSVFile csv = new CSVFile();
			csv.comments = c;
			csv.content = l;
			return csv;
		}
		catch (Exception e)
		{
			e.printStackTrace();
			return null;
		}
	}

	/**
	 * renameto is not reliable to windows
	 * 
	 * @param source
	 * @param dest
	 * @return
	 */
	public static boolean robustRenameTo(File source, File dest)
	{
		if (OSUtil.isWindows())
		{
			try
			{
				String line;
				String cmd = "cmd /c MOVE /Y " + source.getAbsolutePath() + " " + dest.getAbsolutePath();
				System.out.println(cmd);
				Process p = Runtime.getRuntime().exec(cmd);
				BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()));
				while ((line = input.readLine()) != null)
					System.out.println(line);
				input.close();
				BufferedReader input2 = new BufferedReader(new InputStreamReader(p.getErrorStream()));
				while ((line = input2.readLine()) != null)
					System.out.println(line);
				input2.close();
				p.waitFor();
				// System.err.println(p.exitValue());
				return p.exitValue() == 0;
			}
			catch (Exception err)
			{
				err.printStackTrace();
				return false;
			}
		}
		else
			return source.renameTo(dest);
	}

	/**
	 * replace a backslash in windows with a double-backslash
	 * 
	 * @param f
	 * @return
	 */
	public static String getAbsolutePathEscaped(File f)
	{
		if (OSUtil.isWindows())
			return f.getAbsolutePath().replaceAll("\\\\", "\\\\\\\\");
		else
			return f.getAbsolutePath();
	}

	public static boolean copy(File source, File dest)
	{
		FileInputStream from = null;
		FileOutputStream to = null;
		try
		{
			from = new FileInputStream(source);
			to = new FileOutputStream(dest);
			byte[] buffer = new byte[4096];
			int bytesRead;
			while ((bytesRead = from.read(buffer)) != -1)
				to.write(buffer, 0, bytesRead); // write
			return true;
		}
		catch (Exception e)
		{
			e.printStackTrace();
			return false;
		}
		finally
		{
			if (from != null)
				try
				{
					from.close();<commit_msg>add a concat for files<commit_after>package util;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.SequenceInputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;
import java.util.Vector;

import org.apache.commons.codec.digest.DigestUtils;
import org.jfree.io.IOUtils;

public class FileUtil
{
	public static class CSVFile
	{
		public List<String> comments;
		public List<String[]> content;
	}

	public static CSVFile readCSV(String filename)
	{
		try
		{
			List<String[]> l = new ArrayList<String[]>();
			List<String> c = new ArrayList<String>();

			BufferedReader b = new BufferedReader(new FileReader(new File(filename)));
			String s = "";
			while ((s = b.readLine()) != null)
			{
				if (s.trim().length() == 0)
					continue;
				if (s.startsWith("#"))
					c.add(s);
				else
					l.add(ArrayUtil.cast(String.class, VectorUtil.fromCSVString(s, false).toArray()));
			}
			b.close();

			CSVFile csv = new CSVFile();
			csv.comments = c;
			csv.content = l;
			return csv;
		}
		catch (Exception e)
		{
			e.printStackTrace();
			return null;
		}
	}

	/**
	 * renameto is not reliable to windows
	 * 
	 * @param source
	 * @param dest
	 * @return
	 */
	public static boolean robustRenameTo(File source, File dest)
	{
		if (OSUtil.isWindows())
		{
			try
			{
				String line;
				String cmd = "cmd /c MOVE /Y " + source.getAbsolutePath() + " " + dest.getAbsolutePath();
				System.out.println(cmd);
				Process p = Runtime.getRuntime().exec(cmd);
				BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()));
				while ((line = input.readLine()) != null)
					System.out.println(line);
				input.close();
				BufferedReader input2 = new BufferedReader(new InputStreamReader(p.getErrorStream()));
				while ((line = input2.readLine()) != null)
					System.out.println(line);
				input2.close();
				p.waitFor();
				// System.err.println(p.exitValue());
				return p.exitValue() == 0;
			}
			catch (Exception err)
			{
				err.printStackTrace();
				return false;
			}
		}
		else
			return source.renameTo(dest);
	}

	/**
	 * replace a backslash in windows with a double-backslash
	 * 
	 * @param f
	 * @return
	 */
	public static String getAbsolutePathEscaped(File f)
	{
		if (OSUtil.isWindows())
			return f.getAbsolutePath().replaceAll("\\\\", "\\\\\\\\");
		else
			return f.getAbsolutePath();
	}

	public static boolean concat(File dest, List<File> source)
	{
		return concat(dest, source, false);
	}

	public static boolean concat(File dest, List<File> source, boolean append)
	{
		if (dest.exists() && !append)
			if (!dest.delete())
				return false;
		for (File s : source)
			if (!copy(s, dest, true))
				return false;
		return true;
	}

	public static boolean copy(File source, File dest)
	{
		return copy(source, dest, false);
	}

	public static boolean copy(File source, File dest, boolean append)
	{
		FileInputStream from = null;
		FileOutputStream to = null;
		try
		{
			from = new FileInputStream(source);
			to = new FileOutputStream(dest, append);
			byte[] buffer = new byte[4096];
			int bytesRead;
			while ((bytesRead = from.read(buffer)) != -1)
				to.write(buffer, 0, bytesRead); // write
			return true;
		}
		catch (Exception e)
		{
			e.printStackTrace();
			return false;
		}
		finally
		{
			if (from != null)
				try
				{
					from.close();
{'fraction_non_alphanumeric': 0.13921826965305226, 'fraction_numerical': 0.00322061191626409, 'mean_word_length': 3.3076923076923075, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6307553', 'n_tokens_mistral': 2650, 'n_tokens_neox': 2219, 'n_words': 550}