<commit_before>package edu.berkeley.cs.amplab.carat.android.storage;

import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;

import org.apache.thrift.meta_data.FieldMetaData;

import edu.berkeley.cs.amplab.carat.thrift.BatteryDetails;
import edu.berkeley.cs.amplab.carat.thrift.CpuStatus;
import edu.berkeley.cs.amplab.carat.thrift.Feature;
import edu.berkeley.cs.amplab.carat.thrift.NetworkDetails;
import edu.berkeley.cs.amplab.carat.thrift.ProcessInfo;
import edu.berkeley.cs.amplab.carat.thrift.Sample;
import edu.berkeley.cs.amplab.carat.thrift.Sample._Fields;

/**
 * 
 * This class reads HashMaps from the database and outputs a Sample with fields
 * filled in.
 * 
 * TODO: FIXME: This class needs to be modified when the protocol is changed. Otherwise new fields will not be included in Samples.
 * 
 * @author <NAME>
 * 
 */
public class SampleReader {

    /**
     * Library class, instantiation prohibited.
     */
    private SampleReader() {
    }
    
    /**
     * For simplicity, this method relies on the fact that
     * only the piList of Sample has any List type elements.
     * This will fail to record those from substructs (NetworkDetails, BatteryDetails, CpuStatus),
     * and will need to be changed if those are added.
     * 
     * Does not record CallInfo, CellInfo, or CallMonth types.
     */
    public static final HashMap<String, String> writeSample(Sample s) {
        HashMap<String, String> m = new HashMap<String, String>();
        for (_Fields sf: Sample.metaDataMap.keySet()){
            FieldMetaData md = Sample.metaDataMap.get(sf);
            switch (md.valueMetaData.type) {
            case org.apache.thrift.protocol.TType.STRING:
            case org.apache.thrift.protocol.TType.I32:
            case org.apache.thrift.protocol.TType.DOUBLE:
                m.put(sf.getFieldName(), s.getFieldValue(sf).toString());
                break;
            case org.apache.thrift.protocol.TType.STRUCT:
                if (md.fieldName.equals(Sample._Fields.NETWORK_DETAILS.getFieldName()) && s.networkDetails != null) {
                    int len = NetworkDetails._Fields.values().length;
                    StringBuilder b = new StringBuilder();
                    for (int i = 1; i <= len; i++){
                        b.append(""+s.networkDetails.getFieldValue(NetworkDetails._Fields.findByThriftId(i)));
                        if ( i < len)
                            b.append("\n");
                    }
                    m.put(sf.getFieldName(), b.toString());
                } else if (md.fieldName.equals(Sample._Fields.BATTERY_DETAILS.getFieldName()) && s.batteryDetails != null) {
                    int len = BatteryDetails._Fields.values().length;
                    StringBuilder b = new StringBuilder();
                    for (int i = 1; i <= len; i++){
                        b.append(""+s.batteryDetails.getFieldValue(BatteryDetails._Fields.findByThriftId(i)));
                        if ( i < len)
                            b.append("\n");
                    }
                    m.put(sf.getFieldName(), b.toString());
                } else if (md.fieldName.equals(Sample._Fields.CPU_STATUS.getFieldName()) && s.cpuStatus != null) {
                    int len = CpuStatus._Fields.values().length;
                    StringBuilder b = new StringBuilder();
                    for (int i = 1; i <= len; i++){
                        b.append(""+s.cpuStatus.getFieldValue(CpuStatus._Fields.findByThriftId(i)));
                        if ( i < len)
                            b.append("\n");
                    }
                    m.put(sf.getFieldName(), b.toString());
                }/*
                  * else if (md.fieldName.equals("CallInfo")){ }
                  */
                break;
            case org.apache.thrift.protocol.TType.LIST:
                if (md.fieldName.equals(Sample._Fields.EXTRA.getFieldName()) && s.extra != null) {
                    StringBuilder b = new StringBuilder();
                    for (Feature f : s.extra) {
                        b.append(f.key + ";" + f.value + "\n");
                    }
                    b.deleteCharAt(b.lastIndexOf("\n"));
                    m.put(sf.getFieldName(), b.toString());
                } else if (md.fieldName.equals(Sample._Fields.LOCATION_PROVIDERS.getFieldName()) && s.locationProviders != null) {
                    StringBuilder b = new StringBuilder();
                    for (String lp : s.locationProviders)
                        b.append(lp + "\n");
                    b.deleteCharAt(b.lastIndexOf("\n"));
                    m.put(sf.getFieldName(), b.toString());
                } else if (md.fieldName.equals(Sample._Fields.PI_LIST.getFieldName()) && s.piList != null) {
                    StringBuilder b = new StringBuilder();
                    for (ProcessInfo pi : s.piList) {
                        int len = ProcessInfo._Fields.values().length;
                        for (int i = 1; i <= len; i++) {
                            ProcessInfo._Fields pif = ProcessInfo._Fields
                                    .findByThriftId(i);
                            FieldMetaData pmd = ProcessInfo.metaDataMap
                                    .get(pif);
                            if (pmd.valueMetaData.type == org.apache.thrift.protocol.TType.LIST) {
                                if (pi.appSignatures != null) {
                                    for (int j = 0; j < pi.appSignatures.size(); j++) {
                                        String sig = pi.appSignatures.get(j);
                                        b.append(sig);
                                        if (j + 1 < len)
                                            b.append("#");
                                    }
                                }
                            } else {
                                b.append("" + pi.getFieldValue(pif));
                            }
                            if (i < len)
                                b.append(";");
                        }
                        b.append("\n");
                    }
                    b.deleteCharAt(b.lastIndexOf("\n"));
                    m.put(sf.getFieldName(), b.toString());
                }<commit_msg>Fix reading empty lists.<commit_after>package edu.berkeley.cs.amplab.carat.android.storage;

import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;

import org.apache.thrift.meta_data.FieldMetaData;

import edu.berkeley.cs.amplab.carat.thrift.BatteryDetails;
import edu.berkeley.cs.amplab.carat.thrift.CpuStatus;
import edu.berkeley.cs.amplab.carat.thrift.Feature;
import edu.berkeley.cs.amplab.carat.thrift.NetworkDetails;
import edu.berkeley.cs.amplab.carat.thrift.ProcessInfo;
import edu.berkeley.cs.amplab.carat.thrift.Sample;
import edu.berkeley.cs.amplab.carat.thrift.Sample._Fields;

/**
 * 
 * This class reads HashMaps from the database and outputs a Sample with fields
 * filled in.
 * 
 * TODO: FIXME: This class needs to be modified when the protocol is changed. Otherwise new fields will not be included in Samples.
 * 
 * @author <NAME>
 * 
 */
public class SampleReader {

    /**
     * Library class, instantiation prohibited.
     */
    private SampleReader() {
    }
    
    /**
     * For simplicity, this method relies on the fact that
     * only the piList of Sample has any List type elements.
     * This will fail to record those from substructs (NetworkDetails, BatteryDetails, CpuStatus),
     * and will need to be changed if those are added.
     * 
     * Does not record CallInfo, CellInfo, or CallMonth types.
     */
    public static final HashMap<String, String> writeSample(Sample s) {
        HashMap<String, String> m = new HashMap<String, String>();
        for (_Fields sf: Sample.metaDataMap.keySet()){
            FieldMetaData md = Sample.metaDataMap.get(sf);
            switch (md.valueMetaData.type) {
            case org.apache.thrift.protocol.TType.STRING:
            case org.apache.thrift.protocol.TType.I32:
            case org.apache.thrift.protocol.TType.DOUBLE:
                m.put(sf.getFieldName(), s.getFieldValue(sf).toString());
                break;
            case org.apache.thrift.protocol.TType.STRUCT:
                if (md.fieldName.equals(Sample._Fields.NETWORK_DETAILS.getFieldName()) && s.networkDetails != null) {
                    int len = NetworkDetails._Fields.values().length;
                    StringBuilder b = new StringBuilder();
                    for (int i = 1; i <= len; i++){
                        b.append(""+s.networkDetails.getFieldValue(NetworkDetails._Fields.findByThriftId(i)));
                        if ( i < len)
                            b.append("\n");
                    }
                    m.put(sf.getFieldName(), b.toString());
                } else if (md.fieldName.equals(Sample._Fields.BATTERY_DETAILS.getFieldName()) && s.batteryDetails != null) {
                    int len = BatteryDetails._Fields.values().length;
                    StringBuilder b = new StringBuilder();
                    for (int i = 1; i <= len; i++){
                        b.append(""+s.batteryDetails.getFieldValue(BatteryDetails._Fields.findByThriftId(i)));
                        if ( i < len)
                            b.append("\n");
                    }
                    m.put(sf.getFieldName(), b.toString());
                } else if (md.fieldName.equals(Sample._Fields.CPU_STATUS.getFieldName()) && s.cpuStatus != null) {
                    int len = CpuStatus._Fields.values().length;
                    StringBuilder b = new StringBuilder();
                    for (int i = 1; i <= len; i++){
                        b.append(""+s.cpuStatus.getFieldValue(CpuStatus._Fields.findByThriftId(i)));
                        if ( i < len)
                            b.append("\n");
                    }
                    m.put(sf.getFieldName(), b.toString());
                }/*
                  * else if (md.fieldName.equals("CallInfo")){ }
                  */
                break;
            case org.apache.thrift.protocol.TType.LIST:
                if (md.fieldName.equals(Sample._Fields.EXTRA.getFieldName()) && s.extra != null) {
                    StringBuilder b = new StringBuilder();
                    for (Feature f : s.extra) {
                        b.append(f.key + ";" + f.value + "\n");
                    }
                    if (b.length() > 1)
                    	b.deleteCharAt(b.lastIndexOf("\n"));
                    m.put(sf.getFieldName(), b.toString());
                } else if (md.fieldName.equals(Sample._Fields.LOCATION_PROVIDERS.getFieldName()) && s.locationProviders != null) {
                    StringBuilder b = new StringBuilder();
                    for (String lp : s.locationProviders)
                        b.append(lp + "\n");
                    if (b.length() > 1)
                    	b.deleteCharAt(b.lastIndexOf("\n"));
                    m.put(sf.getFieldName(), b.toString());
                } else if (md.fieldName.equals(Sample._Fields.PI_LIST.getFieldName()) && s.piList != null) {
                    StringBuilder b = new StringBuilder();
                    for (ProcessInfo pi : s.piList) {
                        int len = ProcessInfo._Fields.values().length;
                        for (int i = 1; i <= len; i++) {
                            ProcessInfo._Fields pif = ProcessInfo._Fields
                                    .findByThriftId(i);
                            FieldMetaData pmd = ProcessInfo.metaDataMap
                                    .get(pif);
                            if (pmd.valueMetaData.type == org.apache.thrift.protocol.TType.LIST) {
                                if (pi.appSignatures != null) {
                                    for (int j = 0; j < pi.appSignatures.size(); j++) {
                                        String sig = pi.appSignatures.get(j);
                                        b.append(sig);
                                        if (j + 1 < len)
                                            b.append("#");
                                    }
                                }
                            } else {
                                b.append("" + pi.getFieldValue(pif));
                            }
                            if (i < len)
                                b.append(";");
                        }
                        b.append("\n");
                    }
                    if (b.length() > 1)
                    	b.deleteCharAt(b.lastIndexOf("\n"));
                    m.put(sf.getFieldName(), b.toString());
                }
{'fraction_non_alphanumeric': 0.11684144623571764, 'fraction_numerical': 0.0014869306620754422, 'mean_word_length': 1.3766040543053748, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1893629', 'n_tokens_mistral': 3414, 'n_tokens_neox': 3286, 'n_words': 684}