<commit_before>package org.vitrivr.cineast.core.db;

import org.vitrivr.adam.grpc.AdamGrpc.InsertMessage.TupleInsertMessage;
import org.vitrivr.cineast.core.config.Config;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;


public class ProtobufFileWriter extends ProtobufTupleGenerator {

	private static File baseFolder = new File(Config.getExtractorConfig().getOutputLocation(), "proto");
		
	private FileOutputStream out;
	
	public ProtobufFileWriter(){
		super("id", "feature");
	}
	
	public ProtobufFileWriter(String...names){
		super(names);
	}
	
	@Override
	public boolean open(String name) {
		baseFolder.mkdirs();
		try {
			this.out = new FileOutputStream(new File(baseFolder, name + ".bin"));
			return true;
		} catch (FileNotFoundException e) {
			return false;
		}
	}

	@Override
	public boolean close() {
		if(out == null){
			return true;
		}
		
		try {
			out.flush();
			out.close();
			out = null;
			return true;
		} catch (IOException e) {
			return false;
		}
	}

	@Override
	public boolean idExists(String id) {
		return false;
	}

	@Override
	public boolean persist(PersistentTuple<TupleInsertMessage> tuple) {
		try {
			tuple.getPersistentRepresentation().writeDelimitedTo(out);
			out.flush();
			return true;
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return false;
		}
	}

	@Override
	public boolean persist(List<PersistentTuple> persistentTuples) {
		throw new UnsupportedOperationException();
	}


	public static void setFolder(File outputFolder) {
		if(outputFolder == null){
			throw new NullPointerException("outputfolder cannot be null");
		}
		baseFolder = outputFolder;
		baseFolder.mkdirs();
	}

	@Override
	public boolean exists(String key, String value) {
		return false;
	}

  @Override
  protected void finalize() throws Throwable {
    close();
    super.finalize();
  }

	

}<commit_msg>added missing batch write method to ProtobufFileWriter<commit_after>package org.vitrivr.cineast.core.db;

import org.vitrivr.adam.grpc.AdamGrpc.InsertMessage.TupleInsertMessage;
import org.vitrivr.cineast.core.config.Config;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;


public class ProtobufFileWriter extends ProtobufTupleGenerator {

	private static File baseFolder = new File(Config.getExtractorConfig().getOutputLocation(), "proto");
		
	private FileOutputStream out;
	
	public ProtobufFileWriter(){
		super("id", "feature");
	}
	
	public ProtobufFileWriter(String...names){
		super(names);
	}
	
	@Override
	public boolean open(String name) {
		baseFolder.mkdirs();
		try {
			this.out = new FileOutputStream(new File(baseFolder, name + ".bin"));
			return true;
		} catch (FileNotFoundException e) {
			return false;
		}
	}

	@Override
	public boolean close() {
		if(out == null){
			return true;
		}
		
		try {
			out.flush();
			out.close();
			out = null;
			return true;
		} catch (IOException e) {
			return false;
		}
	}

	@Override
	public boolean idExists(String id) {
		return false;
	}

	@Override
	public boolean persist(PersistentTuple<TupleInsertMessage> tuple) {
		try {
			tuple.getPersistentRepresentation().writeDelimitedTo(out);
			out.flush();
			return true;
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return false;
		}
	}

	@SuppressWarnings({ "rawtypes", "unchecked" })
  @Override
	public boolean persist(List<PersistentTuple> persistentTuples) {
		boolean success = true;
	  for(PersistentTuple<TupleInsertMessage> t : persistentTuples){
		  success &= persist(t);
		}
	  return success;
	}


	public static void setFolder(File outputFolder) {
		if(outputFolder == null){
			throw new NullPointerException("outputfolder cannot be null");
		}
		baseFolder = outputFolder;
		baseFolder.mkdirs();
	}

	@Override
	public boolean exists(String key, String value) {
		return false;
	}

  @Override
  protected void finalize() throws Throwable {
    close();
    super.finalize();
  }

	

}
{'fraction_non_alphanumeric': 0.11258758154143512, 'fraction_numerical': 0.0, 'mean_word_length': 4.073529411764706, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4820091', 'n_tokens_mistral': 1457, 'n_tokens_neox': 1267, 'n_words': 310}