<commit_before>package com.flipkart.foxtrot.core.datastore.impl.hbase;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.flipkart.foxtrot.common.Document;
import com.flipkart.foxtrot.core.datastore.DataStore;
import com.flipkart.foxtrot.core.datastore.DataStoreException;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;

import java.util.ArrayList;
import java.util.List;
import java.util.Vector;

/**
 * User: <NAME> (<EMAIL>)
 * Date: 13/03/14
 * Time: 7:54 PM
 */
public class HbaseDataStore implements DataStore {
    private static final byte[] COLUMN_FAMILY = Bytes.toBytes("d");
    private static final byte[] DATA_FIELD_NAME = Bytes.toBytes("data");
    private static final byte[] TIMESTAMP_FIELD_NAME = Bytes.toBytes("timestamp");

    private HbaseTableConnection tableWrapper;
    private ObjectMapper mapper;

    public HbaseDataStore(HbaseTableConnection tableWrapper, ObjectMapper mapper) {
        this.tableWrapper = tableWrapper;
        this.mapper = mapper;
    }

    @Override
    public void save(final String table, Document document) throws DataStoreException {
        try {
            tableWrapper.getTable().put(getPutForDocument(table, document));
        } catch (Throwable t) {
            throw new DataStoreException(DataStoreException.ErrorCode.STORE_SINGLE_SAVE,
                                                        "Saving document error: " + t.getMessage(), t);
        }
    }

    @Override
    public void save(final String table, List<Document> documents) throws DataStoreException {
        try {
            List<Put> puts = new Vector<Put>();
            for(Document document : documents) {
                puts.add(getPutForDocument(table, document));
            }
            tableWrapper.getTable().put(puts);
        } catch (Throwable t) {
            throw new DataStoreException(DataStoreException.ErrorCode.STORE_MULTI_SAVE,
                    "Saving document error: " + t.getMessage(), t);
        }
    }

    @Override
    public Document get(final String table, String id) throws DataStoreException {
        try {
            Get get = new Get(Bytes.toBytes(id + ":" + table))
                    .addColumn(COLUMN_FAMILY, DATA_FIELD_NAME)
                    .setMaxVersions(1);
            Result getResult = tableWrapper.getTable().get(get);
            if(!getResult.isEmpty()) {
                byte[] data = getResult.getValue(COLUMN_FAMILY, DATA_FIELD_NAME);
                byte[] timestamp = getResult.getValue(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME);
                long time = (null != timestamp) ? Bytes.toLong(timestamp) : System.currentTimeMillis();
                if(null != data) {
                    return new Document(id, time, mapper.readTree(data));
                }
            }
        } catch (Throwable t) {
            throw new DataStoreException(DataStoreException.ErrorCode.STORE_SINGLE_GET,
                                t.getMessage(), t);
        }
        throw new DataStoreException(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_ID,
                            String.format("No data found for ID: %s", id));
    }

    @Override
    public List<Document> get(final String table, List<String> ids) throws DataStoreException {
        try {
            List<Get> gets = new ArrayList<Get>(ids.size());
            for(String id: ids) {
                Get get = new Get(Bytes.toBytes(id + ":" + table))
                    .addColumn(COLUMN_FAMILY, DATA_FIELD_NAME)
                    .setMaxVersions(1);
                gets.add(get);
            }
            Result[] getResults = tableWrapper.getTable().get(gets);
            List<Document> results = new ArrayList<Document>(ids.size());
            for(int index = 0; index < getResults.length; index++) {
                boolean found = false;
                Result getResult = getResults[index];
                if(!getResult.isEmpty()) {
                    byte[] data = getResult.getValue(COLUMN_FAMILY, DATA_FIELD_NAME);
                    if(null != data) {
                        byte[] timestamp = getResult.getValue(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME);
                        long time = (null != timestamp) ? Bytes.toLong(timestamp) : System.currentTimeMillis();
                        results.add(new Document(Bytes.toString(getResult.getRow()).split(":")[0],
                                                    time, mapper.readTree(data)));
                        found = true;
                    }
                }
                if(!found)
                {
                    throw new DataStoreException(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_IDS,
                            String.format("No data found for ID: %s", ids.get(index)));
                }
            }
            if(results.isEmpty()) {
                throw new DataStoreException(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_IDS,
                        String.format("No data found for ID: %s", mapper.writeValueAsString(ids)));
            }<commit_msg>Added timestamp to list of retrieved fields<commit_after>package com.flipkart.foxtrot.core.datastore.impl.hbase;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.flipkart.foxtrot.common.Document;
import com.flipkart.foxtrot.core.datastore.DataStore;
import com.flipkart.foxtrot.core.datastore.DataStoreException;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;

import java.util.ArrayList;
import java.util.List;
import java.util.Vector;

/**
 * User: <NAME> (<EMAIL>)
 * Date: 13/03/14
 * Time: 7:54 PM
 */
public class HbaseDataStore implements DataStore {
    private static final byte[] COLUMN_FAMILY = Bytes.toBytes("d");
    private static final byte[] DATA_FIELD_NAME = Bytes.toBytes("data");
    private static final byte[] TIMESTAMP_FIELD_NAME = Bytes.toBytes("timestamp");

    private HbaseTableConnection tableWrapper;
    private ObjectMapper mapper;

    public HbaseDataStore(HbaseTableConnection tableWrapper, ObjectMapper mapper) {
        this.tableWrapper = tableWrapper;
        this.mapper = mapper;
    }

    @Override
    public void save(final String table, Document document) throws DataStoreException {
        try {
            tableWrapper.getTable().put(getPutForDocument(table, document));
        } catch (Throwable t) {
            throw new DataStoreException(DataStoreException.ErrorCode.STORE_SINGLE_SAVE,
                                                        "Saving document error: " + t.getMessage(), t);
        }
    }

    @Override
    public void save(final String table, List<Document> documents) throws DataStoreException {
        try {
            List<Put> puts = new Vector<Put>();
            for(Document document : documents) {
                puts.add(getPutForDocument(table, document));
            }
            tableWrapper.getTable().put(puts);
        } catch (Throwable t) {
            throw new DataStoreException(DataStoreException.ErrorCode.STORE_MULTI_SAVE,
                    "Saving document error: " + t.getMessage(), t);
        }
    }

    @Override
    public Document get(final String table, String id) throws DataStoreException {
        try {
            Get get = new Get(Bytes.toBytes(id + ":" + table))
                    .addColumn(COLUMN_FAMILY, DATA_FIELD_NAME)
                    .addColumn(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME)
                    .setMaxVersions(1);
            Result getResult = tableWrapper.getTable().get(get);
            if(!getResult.isEmpty()) {
                byte[] data = getResult.getValue(COLUMN_FAMILY, DATA_FIELD_NAME);
                byte[] timestamp = getResult.getValue(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME);
                System.out.println("TIMESTAMP: " + (null == timestamp));
                long time = (null != timestamp) ? Bytes.toLong(timestamp) : System.currentTimeMillis();
                if(null != data) {
                    return new Document(id, time, mapper.readTree(data));
                }
            }
        } catch (Throwable t) {
            throw new DataStoreException(DataStoreException.ErrorCode.STORE_SINGLE_GET,
                                t.getMessage(), t);
        }
        throw new DataStoreException(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_ID,
                            String.format("No data found for ID: %s", id));
    }

    @Override
    public List<Document> get(final String table, List<String> ids) throws DataStoreException {
        try {
            List<Get> gets = new ArrayList<Get>(ids.size());
            for(String id: ids) {
                Get get = new Get(Bytes.toBytes(id + ":" + table))
                    .addColumn(COLUMN_FAMILY, DATA_FIELD_NAME)
                    .addColumn(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME)
                    .setMaxVersions(1);
                gets.add(get);
            }
            Result[] getResults = tableWrapper.getTable().get(gets);
            List<Document> results = new ArrayList<Document>(ids.size());
            for(int index = 0; index < getResults.length; index++) {
                boolean found = false;
                Result getResult = getResults[index];
                if(!getResult.isEmpty()) {
                    byte[] data = getResult.getValue(COLUMN_FAMILY, DATA_FIELD_NAME);
                    if(null != data) {
                        byte[] timestamp = getResult.getValue(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME);
                        long time = (null != timestamp) ? Bytes.toLong(timestamp) : System.currentTimeMillis();
                        results.add(new Document(Bytes.toString(getResult.getRow()).split(":")[0],
                                                    time, mapper.readTree(data)));
                        found = true;
                    }
                }
                if(!found)
                {
                    throw new DataStoreException(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_IDS,
                            String.format("No data found for ID: %s", ids.get(index)));
                }
            }
            if(results.isEmpty()) {
                throw new DataStoreException(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_IDS,
                        String.format("No data found for ID: %s", mapper.writeValueAsString(ids)));
            }
{'fraction_non_alphanumeric': 0.10302573286831936, 'fraction_numerical': 0.0024507493637477614, 'mean_word_length': 2.062049062049062, 'pattern_counts': {'":': 6, '<': 27, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3025822', 'n_tokens_mistral': 2922, 'n_tokens_neox': 2834, 'n_words': 571}