<commit_before>/*
 *
 * Adapted from the Rosetta Code entry for Java LZW
 *
 */
package classes.demo;

import java.util.*;
import java.io.*;
 
public class Lzw {
    /** Compress a string to a list of output symbols. */
    public static List<Integer> compress(String uncompressed) {
        // Build the dictionary.
        int dictSize = 256;
        Map<String,Integer> dictionary = new HashMap<String,Integer>();
        for (int i = 0; i < 256; i++)
            dictionary.put("" + (char)i, i);
 
        String w = "";
        List<Integer> result = new ArrayList<Integer>();
        for (char c : uncompressed.toCharArray()) {
            String wc = w + c;
            if (dictionary.containsKey(wc))
                w = wc;
            else {
                result.add(dictionary.get(w));
                // Add wc to the dictionary.
                dictionary.put(wc, dictSize++);
                w = "" + c;
            }
        }
 
        // Output the code for w.
        if (!w.equals(""))
            result.add(dictionary.get(w));
        return result;
    }
 
    /** Decompress a list of output ks to a string. */
    public static String decompress(List<Integer> compressed) {
        // Build the dictionary.
        int dictSize = 256;
        Map<Integer,String> dictionary = new HashMap<Integer,String>();
        for (int i = 0; i < 256; i++)
            dictionary.put(i, "" + (char)i);
 
        String w = "" + (char)(int)compressed.remove(0);
        String result = w;
        for (int k : compressed) {
            String entry;
            if (dictionary.containsKey(k))
                entry = dictionary.get(k);
            else if (k == dictSize)
                entry = w + w.charAt(0);
            else
                throw new IllegalArgumentException("Bad compressed k: " + k);
 
            result += entry;
 
            // Add w+entry[0] to the dictionary.
            dictionary.put(dictSize++, w + entry.charAt(0));
 
            w = entry;
        }
        return result;
    }

    public static void compressFile(String infile, String outfile) 
    throws FileNotFoundException, IOException {
      // slurp into string
      String text = new Scanner(new File(infile)).useDelimiter("\\A").next();
      OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(outfile));
      for (Integer i : compress(text)) {
        out.write((char)i.intValue());
      }
      out.close();
    }

    public static void decompressFile(String infile, String outfile)
    throws FileNotFoundException, IOException {
      List<Integer> compressed = new ArrayList<Integer>();<commit_msg>Using Lzw without java.util.Scanner, because Scanner needs mmap!<commit_after>package classes.demo;

import java.util.*;
import java.io.*;
 
public class Lzw {
    /** Compress a string to a list of output symbols. */
    public static List<Integer> compress(String uncompressed) {
        // Build the dictionary.
        int dictSize = 256;
        Map<String,Integer> dictionary = new HashMap<String,Integer>();
        for (int i = 0; i < 256; i++)
            dictionary.put("" + (char)i, i);
 
        String w = "";
        List<Integer> result = new ArrayList<Integer>();
        for (char c : uncompressed.toCharArray()) {
            String wc = w + c;
            if (dictionary.containsKey(wc))
                w = wc;
            else {
                result.add(dictionary.get(w));
                // Add wc to the dictionary.
                dictionary.put(wc, dictSize++);
                w = "" + c;
            }
        }
 
        // Output the code for w.
        if (!w.equals(""))
            result.add(dictionary.get(w));
        return result;
    }
 
    /** Decompress a list of output ks to a string. */
    public static String decompress(List<Integer> compressed) {
        // Build the dictionary.
        int dictSize = 256;
        Map<Integer,String> dictionary = new HashMap<Integer,String>();
        for (int i = 0; i < 256; i++)
            dictionary.put(i, "" + (char)i);
 
        String w = "" + (char)(int)compressed.remove(0);
        String result = w;
        for (int k : compressed) {
            String entry;
            if (dictionary.containsKey(k))
                entry = dictionary.get(k);
            else if (k == dictSize)
                entry = w + w.charAt(0);
            else
                throw new IllegalArgumentException("Bad compressed k: " + k);
 
            result += entry;
 
            // Add w+entry[0] to the dictionary.
            dictionary.put(dictSize++, w + entry.charAt(0));
 
            w = entry;
        }
        return result;
    }

    static String fileToString(String filename)
    throws FileNotFoundException, IOException {
      File f = new File(filename);
      BufferedReader in = new BufferedReader(new FileReader(f));
      char[] buffer = new char[(int)f.length()];
      int i = 0;
      int c;
      while ((c = in.read()) != -1) {
        buffer[i++] = (char)c;
      }
      return new String(buffer);
    }


    public static void compressFile(String infile, String outfile) 
    throws FileNotFoundException, IOException {
      // slurp into string
      String text = fileToString(infile);
      OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(outfile));
      for (Integer i : compress(text)) {
        out.write((char)i.intValue());
      }
      out.close();
    }

    public static void decompressFile(String infile, String outfile)
    throws FileNotFoundException, IOException {
      List<Integer> compressed = new ArrayList<Integer>();
{'fraction_non_alphanumeric': 0.11963681680612427, 'fraction_numerical': 0.006765177140822503, 'mean_word_length': 1.8958762886597937, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2058317', 'n_tokens_mistral': 1610, 'n_tokens_neox': 1502, 'n_words': 452}