<commit_before>package org.openmhealth.shim.fitbit;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.jayway.jsonpath.JsonPath;
import net.minidev.json.JSONObject;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpRequestBase;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.openmhealth.schema.pojos.*;
import org.openmhealth.schema.pojos.build.*;
import org.openmhealth.schema.pojos.generic.DurationUnitValue;
import org.openmhealth.schema.pojos.generic.LengthUnitValue;
import org.openmhealth.schema.pojos.generic.MassUnitValue;
import org.openmhealth.shim.*;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;

import java.io.IOException;
import java.io.StringWriter;
import java.math.BigDecimal;
import java.net.URL;
import java.util.*;

public class FitbitShim extends OAuth1ShimBase {
    public static final String SHIM_KEY = "fitbit";

    private static final String DATA_URL = "https://api.fitbit.com";

    private static final String REQUEST_TOKEN_URL = "https://api.fitbit.com/oauth/request_token";

    private static final String AUTHORIZE_URL = "https://www.fitbit.com/oauth/authenticate";

    private static final String TOKEN_URL = "https://api.fitbit.com/oauth/access_token";

    //public static final String FITBIT_CLIENT_ID = "7da3c2e5e74d4492ab6bb3286fc32c6b";

    //public static final String FITBIT_CLIENT_SECRET = "455a383f80de45d6a4f9b09e841da1f4";

    public static final String FITBIT_CLIENT_ID = "7c6dd32fc6784dcca9ebfe54733b03dc";

    public static final String FITBIT_CLIENT_SECRET = "75f350a491c74796b5e79819483def47";

    public FitbitShim(AuthorizationRequestParametersRepo authorizationRequestParametersRepo,
                      ShimServerConfig shimServerConfig) {
        super(authorizationRequestParametersRepo, shimServerConfig);
    }

    @Override
    public List<String> getScopes() {
        return null; //noop!
    }

    @Override
    public String getShimKey() {
        return SHIM_KEY;
    }

    @Override
    public String getClientSecret() {
        return FITBIT_CLIENT_SECRET;
    }

    @Override
    public String getClientId() {
        return FITBIT_CLIENT_ID;
    }

    @Override
    public String getBaseRequestTokenUrl() {
        return REQUEST_TOKEN_URL;
    }

    @Override
    public String getBaseAuthorizeUrl() {
        return AUTHORIZE_URL;
    }

    @Override
    public String getBaseTokenUrl() {
        return TOKEN_URL;
    }

    protected HttpMethod getRequestTokenMethod() {
        return HttpMethod.POST;
    }

    protected HttpMethod getAccessTokenMethod() {
        return HttpMethod.POST;
    }

    private static DateTimeFormatter formatterMins =
        DateTimeFormat.forPattern("yyyy-MM-dd HH:mm");

    private static final DateTimeFormatter dayFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");

    @Override
    public ShimDataType[] getShimDataTypes() {
        return FitbitDataType.values();
    }

    public enum FitbitDataType implements ShimDataType {

        WEIGHT(
            "body/log/weight",
            new JsonDeserializer<ShimDataResponse>() {
                @Override
                public ShimDataResponse deserialize(JsonParser jsonParser,
                                                    DeserializationContext ctxt)
                    throws IOException, JsonProcessingException {
                    JsonNode responseNode = jsonParser.getCodec().readTree(jsonParser);
                    String rawJson = responseNode.toString();

                    List<BodyWeight> bodyWeights = new ArrayList<>();
                    JsonPath bodyWeightsPath = JsonPath.compile("$result.content.weight[*]");

                    List<Object> fbWeights = JsonPath.read(rawJson, bodyWeightsPath.getPath());
                    if (CollectionUtils.isEmpty(fbWeights)) {
                        return ShimDataResponse.result(null);
                    }
                    ObjectMapper mapper = new ObjectMapper();
                    for (Object fva : fbWeights) {
                        JsonNode fbWeight = mapper.readTree(((JSONObject) fva).toJSONString());

                        String dateStr = fbWeight.get("date").asText();
                        dateStr += fbWeight.get("time") != null ? "T" + fbWeight.get("time").asText() : "";

                        DateTime dateTimeWhen = new DateTime(dateStr);
                        BodyWeight bodyWeight = new BodyWeightBuilder()
                            .setWeight(
                                fbWeight.get("weight").asText(),
                                MassUnitValue.MassUnit.kg.toString())
                            .setTimeTaken(dateTimeWhen).build();

                        bodyWeights.add(bodyWeight);
                    }
                    Map<String, Object> results = new HashMap<>();
                    results.put(BodyWeight.SCHEMA_BODY_WEIGHT, bodyWeights);
                    return ShimDataResponse.result(results);
                }
            }
        ),

        HEART(
            "heart",
            new JsonDeserializer<ShimDataResponse>() {
                @Override
                public ShimDataResponse deserialize(JsonParser jsonParser,
                                                    DeserializationContext ctxt)
                    throws IOException {
                    JsonNode responseNode = jsonParser.getCodec().readTree(jsonParser);
                    String rawJson = responseNode.toString();

                    List<HeartRate> heartRates = new ArrayList<>();
                    JsonPath heartPath = JsonPath.compile("$.result.content.heart[*]");

                    String dateString = JsonPath.read(rawJson, "$.result.date").toString();

                    List<Object> fbHearts = JsonPath.read(rawJson, heartPath.getPath());
                    if (CollectionUtils.isEmpty(fbHearts)) {
                        return ShimDataResponse.result(null);
                    }

                    ObjectMapper mapper = new ObjectMapper();
                    for (Object fva : fbHearts) {
                        JsonNode fbHeart = mapper.readTree(((JSONObject) fva).toJSONString());

                        String heartDate = dateString;
                        if (fbHeart.get("time") != null) {
                            heartDate += "T" + fbHeart.get("time").asText();
                        }

                        heartRates.add(new HeartRateBuilder()
                            .setRate(fbHeart.get("heartRate").asText())
                            .setTimeTaken(new DateTime(heartDate)).build());

                    }
                    Map<String, Object> results = new HashMap<>();
                    results.put(HeartRate.SCHEMA_HEART_RATE, heartRates);
                    return ShimDataResponse.result(results);
                }
            }
        ),

        BLOOD_PRESSURE(
            "bp",
            new JsonDeserializer<ShimDataResponse>() {
                @Override
                public ShimDataResponse deserialize(JsonParser jsonParser,
                                                    DeserializationContext ctxt)
                    throws IOException, JsonProcessingException {
                    JsonNode responseNode = jsonParser.getCodec().readTree(jsonParser);
                    String rawJson = responseNode.toString();

                    List<BloodPressure> bloodPressures = new ArrayList<>();
                    JsonPath bpPath = JsonPath.compile("$.result.content.bp[*]");

                    String dateString = JsonPath.read(rawJson, "$.result.date").toString();

                    List<Object> fbBloodPressures = JsonPath.read(rawJson, bpPath.getPath());
                    if (CollectionUtils.isEmpty(fbBloodPressures)) {
                        return ShimDataResponse.result(null);
                    }

                    ObjectMapper mapper = new ObjectMapper();
                    for (Object fva : fbBloodPressures) {
                        JsonNode fbBp = mapper.readTree(((JSONObject) fva).toJSONString());

                        String bpDate = dateString;
                        if (fbBp.get("time") != null) {
                            bpDate += "T" + fbBp.get("time").asText();
                        }

                        bloodPressures.add(new BloodPressureBuilder()
                            .setTimeTaken(new DateTime(bpDate))
                            .setValues(
                                new BigDecimal(fbBp.get("systolic").asText()),
                                new BigDecimal(fbBp.get("diastolic").asText())
                            ).build());
                    }
                    Map<String, Object> results = new HashMap<>();
                    results.put(BloodPressure.SCHEMA_BLOOD_PRESSURE, bloodPressures);
                    return ShimDataResponse.result(results);
                }
            }
        ),

        BLOOD_GLUCOSE(
            "glucose",
            new JsonDeserializer<ShimDataResponse>() {
                @Override
                public ShimDataResponse deserialize(JsonParser jsonParser,
                                                    DeserializationContext ctxt)
                    throws IOException, JsonProcessingException {
                    JsonNode responseNode = jsonParser.getCodec().readTree(jsonParser);
                    String rawJson = responseNode.toString();

                    List<BloodGlucose> bloodGlucoses = new ArrayList<>();
                    JsonPath bpPath = JsonPath.compile("$.result.content.glucose[*]");

                    String dateString = JsonPath.read(rawJson, "$.result.date").toString();

                    List<Object> fbBloodPressures = JsonPath.read(rawJson, bpPath.getPath());
                    if (CollectionUtils.isEmpty(fbBloodPressures)) {
                        return ShimDataResponse.result(null);
                    }

                    ObjectMapper mapper = new ObjectMapper();
                    for (Object fva : fbBloodPressures) {
                        JsonNode fbBp = mapper.readTree(((JSONObject) fva).toJSONString());

                        String bpDate = dateString;
                        if (fbBp.get("time") != null) {
                            bpDate += "T" + fbBp.get("time").asText();
                        }

                        bloodGlucoses.add(new BloodGlucoseBuilder()
                            .setTimeTaken(new DateTime(bpDate))
                            .setValue(fbBp.get("glucose").asText()).build());
                    }
                    Map<String, Object> results = new HashMap<>();
                    results.put(BloodGlucose.SCHEMA_BLOOD_GLUCOSE, bloodGlucoses);
                    return ShimDataResponse.result(results);
                }
            }
        ),

        STEPS("activities/steps", new JsonDeserializer<ShimDataResponse>() {
            @Override
            public ShimDataResponse deserialize(JsonParser jsonParser,
                                                DeserializationContext deserializationContext)
                throws IOException {
                return ShimDataResponse.empty();
            }
        }),

        ACTIVITY(
            "activities",
            new JsonDeserializer<ShimDataResponse>() {
                @Override
                public ShimDataResponse deserialize(JsonParser jsonParser,
                                                    DeserializationContext deserializationContext)
                    throws IOException {
                    JsonNode responseNode = jsonParser.getCodec().readTree(jsonParser);
                    String rawJson = responseNode.toString();

                    List<Activity> activities = new ArrayList<>();
                    List<NumberOfSteps> numberOfStepsList = new ArrayList<>();

                    JsonPath activityPath = JsonPath.compile("$.result.content.activities[*]");

                    final List<Object> fitbitActivities = JsonPath.read(rawJson, activityPath.getPath());
                    if (CollectionUtils.isEmpty(fitbitActivities)) {
                        return ShimDataResponse.result(null);
                    }
                    ObjectMapper mapper = new ObjectMapper();
                    for (Object fva : fitbitActivities) {
                        final JsonNode fitbitActivity = mapper.readTree(((JSONObject) fva).toJSONString());

                        String dateString = fitbitActivity.get("startDate").asText()
                            + (fitbitActivity.get("startTime") != null ?
                            " " + fitbitActivity.get("startTime").asText() : "");

                        DateTime startTime = formatterMins.parseDateTime(dateString);

                        Activity activity = new ActivityBuilder()
                            .setActivityName(fitbitActivity.get("activityParentName").asText())
                            .setDistance(fitbitActivity.get("distance").asText(),
                                LengthUnitValue.LengthUnit.m.toString())
                            .setDuration(fitbitActivity.get("duration").asText(),
                                DurationUnitValue.DurationUnit.ms.toString())
                            .setStartTime(startTime).build();

                        NumberOfSteps numberOfSteps = new NumberOfStepsBuilder()
                            .setSteps(fitbitActivity.get("steps").asInt()).build();
                        numberOfSteps.setEffectiveTimeFrame(activity.getEffectiveTimeFrame());

                        activities.add(activity);
                        numberOfStepsList.add(numberOfSteps);
                    }

                    Map<String, Object> results = new HashMap<>();
                    results.put(Activity.SCHEMA_ACTIVITY, activities);
                    results.put(NumberOfSteps.SCHEMA_NUMBER_OF_STEPS, numberOfStepsList);
                    return ShimDataResponse.result(results);
                }
            }
        );

        private String endPoint;

        private JsonDeserializer<ShimDataResponse> normalizer;

        FitbitDataType(String endPoint, JsonDeserializer<ShimDataResponse> normalizer) {
            this.endPoint = endPoint;
            this.normalizer = normalizer;
        }

        @Override
        public JsonDeserializer<ShimDataResponse> getNormalizer() {
            return normalizer;
        }

        public String getEndPoint() {
            return endPoint;
        }
    }<commit_msg>More updates to fitbit shim with deserializer.<commit_after>package org.openmhealth.shim.fitbit;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.jayway.jsonpath.JsonPath;
import net.minidev.json.JSONArray;
import net.minidev.json.JSONObject;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpRequestBase;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.openmhealth.schema.pojos.*;
import org.openmhealth.schema.pojos.build.*;
import org.openmhealth.schema.pojos.generic.DurationUnitValue;
import org.openmhealth.schema.pojos.generic.LengthUnitValue;
import org.openmhealth.schema.pojos.generic.MassUnitValue;
import org.openmhealth.shim.*;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;

import java.io.IOException;
import java.io.StringWriter;
import java.math.BigDecimal;
import java.net.URL;
import java.util.*;

public class FitbitShim extends OAuth1ShimBase {
    public static final String SHIM_KEY = "fitbit";

    private static final String DATA_URL = "https://api.fitbit.com";

    private static final String REQUEST_TOKEN_URL = "https://api.fitbit.com/oauth/request_token";

    private static final String AUTHORIZE_URL = "https://www.fitbit.com/oauth/authenticate";

    private static final String TOKEN_URL = "https://api.fitbit.com/oauth/access_token";

    //public static final String FITBIT_CLIENT_ID = "7da3c2e5e74d4492ab6bb3286fc32c6b";

    //public static final String FITBIT_CLIENT_SECRET = "455a383f80de45d6a4f9b09e841da1f4";

    public static final String FITBIT_CLIENT_ID = "7c6dd32fc6784dcca9ebfe54733b03dc";

    public static final String FITBIT_CLIENT_SECRET = "75f350a491c74796b5e79819483def47";

    public FitbitShim(AuthorizationRequestParametersRepo authorizationRequestParametersRepo,
                      ShimServerConfig shimServerConfig) {
        super(authorizationRequestParametersRepo, shimServerConfig);
    }

    @Override
    public List<String> getScopes() {
        return null; //noop!
    }

    @Override
    public String getShimKey() {
        return SHIM_KEY;
    }

    @Override
    public String getClientSecret() {
        return FITBIT_CLIENT_SECRET;
    }

    @Override
    public String getClientId() {
        return FITBIT_CLIENT_ID;
    }

    @Override
    public String getBaseRequestTokenUrl() {
        return REQUEST_TOKEN_URL;
    }

    @Override
    public String getBaseAuthorizeUrl() {
        return AUTHORIZE_URL;
    }

    @Override
    public String getBaseTokenUrl() {
        return TOKEN_URL;
    }

    protected HttpMethod getRequestTokenMethod() {
        return HttpMethod.POST;
    }

    protected HttpMethod getAccessTokenMethod() {
        return HttpMethod.POST;
    }

    private static DateTimeFormatter formatterMins =
        DateTimeFormat.forPattern("yyyy-MM-dd HH:mm");

    private static final DateTimeFormatter dayFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");

    @Override
    public ShimDataType[] getShimDataTypes() {
        return FitbitDataType.values();
    }

    public enum FitbitDataType implements ShimDataType {

        WEIGHT(
            "body/log/weight",
            new JsonDeserializer<ShimDataResponse>() {
                @Override
                public ShimDataResponse deserialize(JsonParser jsonParser,
                                                    DeserializationContext ctxt)
                    throws IOException, JsonProcessingException {
                    JsonNode responseNode = jsonParser.getCodec().readTree(jsonParser);
                    String rawJson = responseNode.toString();

                    List<BodyWeight> bodyWeights = new ArrayList<>();
                    JsonPath bodyWeightsPath = JsonPath.compile("$result.content.weight[*]");

                    List<Object> fbWeights = JsonPath.read(rawJson, bodyWeightsPath.getPath());
                    if (CollectionUtils.isEmpty(fbWeights)) {
                        return ShimDataResponse.result(null);
                    }
                    ObjectMapper mapper = new ObjectMapper();
                    for (Object fva : fbWeights) {
                        JsonNode fbWeight = mapper.readTree(((JSONObject) fva).toJSONString());

                        String dateStr = fbWeight.get("date").asText();
                        dateStr += fbWeight.get("time") != null ? "T" + fbWeight.get("time").asText() : "";

                        DateTime dateTimeWhen = new DateTime(dateStr);
                        BodyWeight bodyWeight = new BodyWeightBuilder()
                            .setWeight(
                                fbWeight.get("weight").asText(),
                                MassUnitValue.MassUnit.kg.toString())
                            .setTimeTaken(dateTimeWhen).build();

                        bodyWeights.add(bodyWeight);
                    }
                    Map<String, Object> results = new HashMap<>();
                    results.put(BodyWeight.SCHEMA_BODY_WEIGHT, bodyWeights);
                    return ShimDataResponse.result(results);
                }
            }
        ),

        HEART(
            "heart",
            new JsonDeserializer<ShimDataResponse>() {
                @Override
                public ShimDataResponse deserialize(JsonParser jsonParser,
                                                    DeserializationContext ctxt)
                    throws IOException {
                    JsonNode responseNode = jsonParser.getCodec().readTree(jsonParser);
                    String rawJson = responseNode.toString();

                    List<HeartRate> heartRates = new ArrayList<>();
                    JsonPath heartPath = JsonPath.compile("$.result.content.heart[*]");

                    String dateString = JsonPath.read(rawJson, "$.result.date").toString();

                    List<Object> fbHearts = JsonPath.read(rawJson, heartPath.getPath());
                    if (CollectionUtils.isEmpty(fbHearts)) {
                        return ShimDataResponse.result(null);
                    }

                    ObjectMapper mapper = new ObjectMapper();
                    for (Object fva : fbHearts) {
                        JsonNode fbHeart = mapper.readTree(((JSONObject) fva).toJSONString());

                        String heartDate = dateString;
                        if (fbHeart.get("time") != null) {
                            heartDate += "T" + fbHeart.get("time").asText();
                        }

                        heartRates.add(new HeartRateBuilder()
                            .setRate(fbHeart.get("heartRate").asText())
                            .setTimeTaken(new DateTime(heartDate)).build());

                    }
                    Map<String, Object> results = new HashMap<>();
                    results.put(HeartRate.SCHEMA_HEART_RATE, heartRates);
                    return ShimDataResponse.result(results);
                }
            }
        ),

        BLOOD_PRESSURE(
            "bp",
            new JsonDeserializer<ShimDataResponse>() {
                @Override
                public ShimDataResponse deserialize(JsonParser jsonParser,
                                                    DeserializationContext ctxt)
                    throws IOException, JsonProcessingException {
                    JsonNode responseNode = jsonParser.getCodec().readTree(jsonParser);
                    String rawJson = responseNode.toString();

                    List<BloodPressure> bloodPressures = new ArrayList<>();
                    JsonPath bpPath = JsonPath.compile("$.result.content.bp[*]");

                    String dateString = JsonPath.read(rawJson, "$.result.date").toString();

                    List<Object> fbBloodPressures = JsonPath.read(rawJson, bpPath.getPath());
                    if (CollectionUtils.isEmpty(fbBloodPressures)) {
                        return ShimDataResponse.result(null);
                    }

                    ObjectMapper mapper = new ObjectMapper();
                    for (Object fva : fbBloodPressures) {
                        JsonNode fbBp = mapper.readTree(((JSONObject) fva).toJSONString());

                        String bpDate = dateString;
                        if (fbBp.get("time") != null) {
                            bpDate += "T" + fbBp.get("time").asText();
                        }

                        bloodPressures.add(new BloodPressureBuilder()
                            .setTimeTaken(new DateTime(bpDate))
                            .setValues(
                                new BigDecimal(fbBp.get("systolic").asText()),
                                new BigDecimal(fbBp.get("diastolic").asText())
                            ).build());
                    }
                    Map<String, Object> results = new HashMap<>();
                    results.put(BloodPressure.SCHEMA_BLOOD_PRESSURE, bloodPressures);
                    return ShimDataResponse.result(results);
                }
            }
        ),

        BLOOD_GLUCOSE(
            "glucose",
            new JsonDeserializer<ShimDataResponse>() {
                @Override
                public ShimDataResponse deserialize(JsonParser jsonParser,
                                                    DeserializationContext ctxt)
                    throws IOException, JsonProcessingException {
                    JsonNode responseNode = jsonParser.getCodec().readTree(jsonParser);
                    String rawJson = responseNode.toString();

                    List<BloodGlucose> bloodGlucoses = new ArrayList<>();
                    JsonPath bpPath = JsonPath.compile("$.result.content.glucose[*]");

                    String dateString = JsonPath.read(rawJson, "$.result.date").toString();

                    List<Object> fbBloodPressures = JsonPath.read(rawJson, bpPath.getPath());
                    if (CollectionUtils.isEmpty(fbBloodPressures)) {
                        return ShimDataResponse.result(null);
                    }

                    ObjectMapper mapper = new ObjectMapper();
                    for (Object fva : fbBloodPressures) {
                        JsonNode fbBp = mapper.readTree(((JSONObject) fva).toJSONString());

                        String bpDate = dateString;
                        if (fbBp.get("time") != null) {
                            bpDate += "T" + fbBp.get("time").asText();
                        }

                        bloodGlucoses.add(new BloodGlucoseBuilder()
                            .setTimeTaken(new DateTime(bpDate))
                            .setValue(fbBp.get("glucose").asText()).build());
                    }
                    Map<String, Object> results = new HashMap<>();
                    results.put(BloodGlucose.SCHEMA_BLOOD_GLUCOSE, bloodGlucoses);
                    return ShimDataResponse.result(results);
                }
            }
        ),

        STEPS("activities/steps", new JsonDeserializer<ShimDataResponse>() {
            @Override
            public ShimDataResponse deserialize(JsonParser jsonParser,
                                                DeserializationContext deserializationContext)
                throws IOException {

                JsonNode responseNode = jsonParser.getCodec().readTree(jsonParser);
                String rawJson = responseNode.toString();

                List<NumberOfSteps> steps = new ArrayList<>();
                JsonPath stepsPath = JsonPath.compile("$.[*].result.content[*]");

                Object oneMinStepEntries = JsonPath.read(rawJson, stepsPath.getPath());

                if (oneMinStepEntries == null) {
                    return ShimDataResponse.empty();
                }

                DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
                ObjectMapper mapper = new ObjectMapper();

                /**
                 * Determine if many items were returned or just one
                 * and cast appropriately.
                 */
                ArrayNode nodes;
                String jsonString;
                if (oneMinStepEntries instanceof JSONArray) {
                    jsonString = ((JSONArray) oneMinStepEntries).toJSONString();
                } else {
                    jsonString = "[" +
                        ((JSONObject) oneMinStepEntries).toJSONString() + "]";
                }
                nodes = (ArrayNode) mapper.readTree(jsonString);

                for (Object node1 : nodes) {
                    JsonNode fbStepNode = (JsonNode) node1;

                    String dateString =
                        (fbStepNode.get("activities-steps")).get(0).get("dateTime").asText();

                    ArrayNode dataset = (ArrayNode)
                        fbStepNode.get("activities-steps-intraday").get("dataset");

                    for (JsonNode stepMinute : dataset) {
                        if (stepMinute.get("value").asInt() > 0) {
                            steps.add(new NumberOfStepsBuilder()
                                .setStartTime(formatter.parseDateTime(
                                    dateString + " " + stepMinute.get("time").asText()))
                                .setDuration("1", DurationUnitValue.DurationUnit.min.toString())
                                .setSteps(stepMinute.get("value").asInt())
                                .build());
                        }
                    }
                }
                Map<String, Object> results = new HashMap<>();
                results.put(NumberOfSteps.SCHEMA_NUMBER_OF_STEPS, steps);
                return ShimDataResponse.result(results);
            }
        }),

        ACTIVITY(
            "activities",
            new JsonDeserializer<ShimDataResponse>() {
                @Override
                public ShimDataResponse deserialize(JsonParser jsonParser,
                                                    DeserializationContext deserializationContext)
                    throws IOException {
                    JsonNode responseNode = jsonParser.getCodec().readTree(jsonParser);
                    String rawJson = responseNode.toString();

                    List<Activity> activities = new ArrayList<>();

                    JsonPath activityPath = JsonPath.compile("$.result.content.activities[*]");

                    final List<Object> fitbitActivities = JsonPath.read(rawJson, activityPath.getPath());
                    if (CollectionUtils.isEmpty(fitbitActivities)) {
                        return ShimDataResponse.result(null);
                    }
                    ObjectMapper mapper = new ObjectMapper();
                    for (Object fva : fitbitActivities) {
                        final JsonNode fitbitActivity = mapper.readTree(((JSONObject) fva).toJSONString());

                        String dateString = fitbitActivity.get("startDate").asText()
                            + (fitbitActivity.get("startTime") != null ?
                            " " + fitbitActivity.get("startTime").asText() : "");

                        DateTime startTime = formatterMins.parseDateTime(dateString);

                        Activity activity = new ActivityBuilder()
                            .setActivityName(fitbitActivity.get("activityParentName").asText())
                            .setDistance(fitbitActivity.get("distance").asText(),
                                LengthUnitValue.LengthUnit.m.toString())
                            .setDuration(fitbitActivity.get("duration").asText(),
                                DurationUnitValue.DurationUnit.ms.toString())
                            .setStartTime(startTime).build();

                        activities.add(activity);
                    }

                    Map<String, Object> results = new HashMap<>();
                    results.put(Activity.SCHEMA_ACTIVITY, activities);
                    return ShimDataResponse.result(results);
                }
            }
        );

        private String endPoint;

        private JsonDeserializer<ShimDataResponse> normalizer;

        FitbitDataType(String endPoint, JsonDeserializer<ShimDataResponse> normalizer) {
            this.endPoint = endPoint;
            this.normalizer = normalizer;
        }

        @Override
        public JsonDeserializer<ShimDataResponse> getNormalizer() {
            return normalizer;
        }

        public String getEndPoint() {
            return endPoint;
        }
    }
{'fraction_non_alphanumeric': 0.09171222932690823, 'fraction_numerical': 0.005035060080931641, 'mean_word_length': 1.6976085326222814, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 80, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 15}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5214478', 'n_tokens_mistral': 8293, 'n_tokens_neox': 7557, 'n_words': 1371}