<commit_before>package com.efurture.file;

import com.efurture.file.compress.GZip;
import com.efurture.file.io.BlockFileInputStream;
import com.efurture.file.io.BlockOutputStream;
import com.efurture.file.meta.Meta;
import com.efurture.file.meta.MetaOutputStream;
import com.efurture.file.meta.MetaUtils;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Map;

/**
 * 򵥸ЧСļ洢ݿ,Դ洢Сļ, ļ1G
 * Сļϲһļ, Appendģʽ, д벻Ӱ, ܺá
 * Created by (jianbai.gbj) on 2017/6/23.
 */
public class FileStore {

    /**
     * С
     * */
    public  static final  int BLOCK_BUFFER_SIZE = 1024*1024;

    /**
     * ļд
     * */
    private BlockOutputStream outputStream;

    /**
     * ļԭϢд
     * */
    private MetaOutputStream metaOutputStream;

    /**
     * 洢ļ
     * */
    private String dbFile;

    /**
     * ļϢ,ļλϢ
     * */
    private Map<String, Meta> indexMeta;





    /**
     * һļĴ洢
     * */
    public FileStore(String file) throws IOException{
        String metaFile = file + Meta.FILE_SUFFIX;
        indexMeta = MetaUtils.readMeta(metaFile);
        metaOutputStream = new MetaOutputStream(metaFile);
        outputStream = new BlockOutputStream(file, indexMeta);
        dbFile = file;
    }


    /**
     * 洢ļݿ
     * */
    public void put(String fileName, byte[] bts) throws IOException {
        boolean zip = false;
        if(fileName.endsWith(".dat")
                || fileName.endsWith(".txt")
                || fileName.endsWith(".json")
                || fileName.endsWith(".html")
                || fileName.endsWith(".htm")){
            zip = true;
        }
        put(fileName, bts, zip);
    }

    /**
     * @param  fileName 
     * @param  bts  ֽڸ
     * 洢ļݿ, Ƿѹ
     * */
    public void put(String fileName, byte[] bts, boolean zip) throws IOException {
        synchronized (this) {
            indexMeta.remove(fileName);
            if(zip){
                bts =  GZip.compress(bts, 0, bts.length);
            }
            outputStream.write(fileName, bts, 0, bts.length);
            outputStream.flush();
            if(zip){
                Meta meta = indexMeta.get(fileName);
                meta.version = Meta.GZIP_VERSION;
            }
            metaOutputStream.writeMeta(indexMeta.get(fileName));
            metaOutputStream.flush();
        }
    }

    /**
     * 洢ַ,Ĭѹ洢
     * */
    public void put(String fileName, String content) throws IOException {
         put(fileName, content.getBytes(), true);
    }

    public void put(String fileName, String content, boolean zip) throws IOException {
        put(fileName, content.getBytes(), zip);
    }

    /**
     * ȡļ,ļ,򷵻ؿ, ļ,ļ
     * */
    public byte[] get(String fileName) throws IOException {
        Meta meta = indexMeta.get(fileName);
        if(meta == null){
            return  null;
        }
        BlockFileInputStream inputStream = new BlockFileInputStream(dbFile, meta.blocks);
        ByteArrayOutputStream data = new ByteArrayOutputStream(1024*4);
        byte[] buffer = new byte[1024*4];
        int read = 0;
        while ((read = inputStream.read(buffer, 0)) > 0){
            data.write(buffer, 0, read);
        }<commit_msg>增加bufferSize大小<commit_after>package com.efurture.file;

import com.efurture.file.compress.GZip;
import com.efurture.file.io.BlockFileInputStream;
import com.efurture.file.io.BlockOutputStream;
import com.efurture.file.meta.Meta;
import com.efurture.file.meta.MetaOutputStream;
import com.efurture.file.meta.MetaUtils;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Map;

/**
 * 򵥸ЧСļ洢ݿ,Դ洢Сļ, ļ1G
 * Сļϲһļ, Appendģʽ, д벻Ӱ, ܺá
 * Created by (jianbai.gbj) on 2017/6/23.
 */
public class FileStore {

    /**
     * С
     * */
    public  static final  int BLOCK_BUFFER_SIZE = 1024*1024;

    /**
     * ļд
     * */
    private BlockOutputStream outputStream;

    /**
     * ļԭϢд
     * */
    private MetaOutputStream metaOutputStream;

    /**
     * 洢ļ
     * */
    private String dbFile;

    /**
     * ļϢ,ļλϢ
     * */
    private Map<String, Meta> indexMeta;





    /**
     * һļĴ洢
     * */
    public FileStore(String file) throws IOException{
        String metaFile = file + Meta.FILE_SUFFIX;
        indexMeta = MetaUtils.readMeta(metaFile);
        metaOutputStream = new MetaOutputStream(metaFile);
        outputStream = new BlockOutputStream(file, indexMeta);
        dbFile = file;
    }


    /**
     * 洢ļݿ
     * */
    public void put(String fileName, byte[] bts) throws IOException {
        boolean zip = false;
        if(fileName.endsWith(".dat")
                || fileName.endsWith(".txt")
                || fileName.endsWith(".json")
                || fileName.endsWith(".html")
                || fileName.endsWith(".htm")){
            zip = true;
        }
        put(fileName, bts, zip);
    }

    /**
     * @param  fileName 
     * @param  bts  ֽڸ
     * 洢ļݿ, Ƿѹ
     * */
    public void put(String fileName, byte[] bts, boolean zip) throws IOException {
        synchronized (this) {
            indexMeta.remove(fileName);
            if(zip){
                bts =  GZip.compress(bts, 0, bts.length);
            }
            outputStream.write(fileName, bts, 0, bts.length);
            outputStream.flush();
            if(zip){
                Meta meta = indexMeta.get(fileName);
                meta.version = Meta.GZIP_VERSION;
            }
            metaOutputStream.writeMeta(indexMeta.get(fileName));
            metaOutputStream.flush();
        }
    }

    /**
     * 洢ַ,Ĭѹ洢
     * */
    public void put(String fileName, String content) throws IOException {
         put(fileName, content.getBytes(), true);
    }

    public void put(String fileName, String content, boolean zip) throws IOException {
        put(fileName, content.getBytes(), zip);
    }

    /**
     * ȡļ,ļ,򷵻ؿ, ļ,ļ
     * */
    public byte[] get(String fileName) throws IOException {
        Meta meta = indexMeta.get(fileName);
        if(meta == null){
            return  null;
        }
        BlockFileInputStream inputStream = new BlockFileInputStream(dbFile, meta.blocks);
        ByteArrayOutputStream data = new ByteArrayOutputStream(1024*4);
        byte[] buffer = new byte[1024*8];
        int read = 0;
        while ((read = inputStream.read(buffer, 0)) > 0){
            data.write(buffer, 0, read);
        }
{'fraction_non_alphanumeric': 0.12188626037913207, 'fraction_numerical': 0.010026633244555851, 'mean_word_length': 2.0184397163120567, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4551732', 'n_tokens_mistral': 2118, 'n_tokens_neox': 2077, 'n_words': 406}