<jupyter_start><jupyter_text>Validate evaluation of phones This notebook retrieves the evaluation results for a particular experiment and validates them  Experiment parameters If you only want to run experiments, you only need to edit these variables. The notebook will retrieve the appropriate spec, use it to find the evaluation periods from the database and validate the evaluation. You probably want to publish this notebook along with your results so that others can examine it as well.<jupyter_code>DATASTORE_URL = "http://cardshark.cs.berkeley.edu"
AUTHOR_EMAIL = "<EMAIL>" # e.g. <EMAIL>
CURR_SPEC_ID = "many_unimodal_trips_sb" # e.g. sfba_calibration_only_1
MAX_DURATION_VARIATION = 5 * 60 # seconds<jupyter_output><empty_output><jupyter_text>Setup some basic imports<jupyter_code>import logging
logging.getLogger().setLevel(logging.DEBUG)
import copy
import matplotlib.pyplot as plt
import arrow
import pandas as pd
%matplotlib notebook
import requests
import folium
import folium.features as fof
import folium.plugins as fpl
import folium.utilities as ful
import branca.element as bre<jupyter_output><empty_output><jupyter_text>Define some simple utility functions<jupyter_code>def get_row_count(n_maps, cols):
    rows = int(n_maps / cols)
    if (n_maps % cols != 0):
        rows = rows + 1
    return rows<jupyter_output><empty_output><jupyter_text>Setup the ability to make calls to the server<jupyter_code>def retrieve_data_from_server(user_label, key_list, start_ts, end_ts):
    post_msg = {
        "user": user_label,
        "key_list": key_list,
        "start_time": start_ts,
        "end_time": end_ts
    }
    # print("About to retrieve messages using %s" % post_msg)
    response = requests.post(DATASTORE_URL+"/datastreams/find_entries/timestamp", json=post_msg)
    # print("response = %s" % response)
    response.raise_for_status()
    ret_list = response.json()["phone_data"]
    # print("Found %d entries" % len(ret_list))
    return ret_list

def retrieve_all_data_from_server(user_label, key_list):
    return retrieve_data_from_server(user_label, key_list, 0, arrow.get().timestamp)<jupyter_output><empty_output><jupyter_text>Find the current spec<jupyter_code>all_spec_entry_list = retrieve_all_data_from_server(AUTHOR_EMAIL, ["config/evaluation_spec"])
curr_spec_entry = None
for s in all_spec_entry_list:
    if s["data"]["label"]["id"] == CURR_SPEC_ID:
        curr_spec_entry = s
curr_spec_wrapper = curr_spec_entry["data"]
curr_spec = curr_spec_wrapper["label"]
curr_spec["name"]<jupyter_output><empty_output><jupyter_text>Find all evaluation transitions within the start and end times of this spec<jupyter_code>eval_start_ts = curr_spec_wrapper["start_ts"]
eval_end_ts = curr_spec_wrapper["end_ts"] + 7 * 24 * 60 * 60 # 7 days
eval_tz = curr_spec["region"]["timezone"]
print("Evaluation ran from %s -> %s" % (arrow.get(eval_start_ts).to(eval_tz), arrow.get(eval_end_ts).to(eval_tz)))
phone_labels = curr_spec["phones"]<jupyter_output><empty_output><jupyter_text>Data model here is:```eval_transitions    - android        - ucb.sdb.android.1            - list of evaluation transitions        - ....    - ios```<jupyter_code>eval_transitions = copy.deepcopy(phone_labels)
for phoneOS, phone_map in eval_transitions.items():
    print("Reading data for %s phones" % phoneOS)
    for phone_label in phone_map:
        print("Loading transitions for phone %s" % phone_label)
        curr_phone_transitions = retrieve_data_from_server(phone_label, ["manual/evaluation_transition"], eval_start_ts, eval_end_ts)
        curr_phone_role = phone_map[phone_label]
        phone_map[phone_label] = {"role": curr_phone_role}
        phone_map[phone_label]["transitions"] = curr_phone_transitions<jupyter_output><empty_output><jupyter_text>Find evaluation transitions, validate and map them to ranges From here onwards, we will add the results of manipulation to each phone entry - e.g.```eval_transitions    - android        - ucb.sdb.android.1            - transitions (all transition entries, added in previous step)            - evaluation_transitions (evaluation transitions, will be added in this step)        - ....    - ios```<jupyter_code># # imported to emeval/input/phone_view.py
for phoneOS, phone_map in eval_transitions.items():
    print("Processing data for %s phones" % phoneOS)
    for phone_label in phone_map:
        print("Processing transitions for phone %s" % phone_label)
        curr_phone_transitions = [t["data"] for t in phone_map[phone_label]["transitions"]]
        # print(curr_phone_transitions)
        curr_evaluation_transitions = [t for t in curr_phone_transitions if (t["transition"] in ["START_EVALUATION_PERIOD", "STOP_EVALUATION_PERIOD", 2, 3])]
        print("Filtered %d total -> %d evaluation transitions " % (len(curr_phone_transitions), len(curr_evaluation_transitions)))
        phone_map[phone_label]["evaluation_transitions"] = curr_evaluation_transitions
ios_1_transitions = eval_transitions["android"]["ucb-sdb-android-2"]["evaluation_transitions"]
print("\n".join([str((t["transition"], t["trip_id"], t["ts"], arrow.get(t["ts"]).to(eval_tz))) for t in ios_1_transitions]))
ios_1_transitions = eval_transitions["ios"]["ucb-sdb-ios-3"]["evaluation_transitions"]
print("\n".join([str((t["transition"], t["trip_id"], t["ts"], arrow.get(t["ts"]).to(eval_tz))) for t in ios_1_transitions]))
# We expect that transitions occur in pairs
def transitions_to_ranges(transition_list, start_tt, end_tt, start_ti, end_ti):
    start_transitions = transition_list[::2]
    end_transitions = transition_list[1::2]
    if len(transition_list) % 2 == 0:
        print("All evaluation is complete, nothing to change")
    else:
        print("Ongoing evaluation, adding fake end transition")
        last_start_transition = phone_map[phone_label]["transitions"][-1]
        fake_end_transition = copy.copy(last_start_transition)
        fake_end_transition["data"]["transition"] = end_tt
        curr_ts = arrow.get().timestamp
        fake_end_transition["data"]["ts"] = curr_ts
        if "fmt_time" in last_start_transition["data"]:
            fake_end_transition["data"]["fmt_time"] = arrow.get(curr_ts).to(eval_tz)
        fake_end_transition["metadata"]["write_ts"] = curr_ts
        if "write_fmt_time" in last_start_transition["metadata"]:
            fake_end_transition["metadata"]["write_fmt_time"] = arrow.get(curr_ts).to(eval_tz)
        fake_end_transition["metadata"]["platform"] = "fake"
        if "local_dt" in fake_end_transition["data"]:
            del fake_end_transition["data"]["local_dt"]

    range_list = []
    for (s, e) in zip(start_transitions, end_transitions):
        # print("------------------------------------- \n %s -> \n %s" % (s, e))
        assert s["transition"] == start_tt or s["transition"] == start_ti, "Start transition has %s transition" % s["transition"]
        assert e["transition"] == end_tt or s["transition"] == end_ti, "Stop transition has %s transition" % s["transition"]
        assert s["trip_id"] == e["trip_id"], "trip_id mismatch! %s != %s" % (s["trip_id"], e["trip_id"])
        assert e["ts"] > s["ts"], "end %s is before start %s" % (arrow.get(e["ts"]), arrow.get(s["ts"]))
        for f in ["spec_id", "device_manufacturer", "device_model", "device_version"]:
            assert s[f] == e[f], "Field %s mismatch! %s != %s" % (f, s[f], e[f])
        curr_range = {"trip_id": s["trip_id"], "start_ts": s["ts"], "end_ts": e["ts"], "duration": (e["ts"] - s["ts"])}
        range_list.append(curr_range)
        
    return range_list
transitions_to_ranges(eval_transitions["ios"]["ucb-sdb-ios-1"]["evaluation_transitions"], "START_EVALUATION_PERIOD", "STOP_EVALUATION_PERIOD", 2, 3)
for phoneOS, phone_map in eval_transitions.items():
    print("Processing data for %s phones" % phoneOS)
    for phone_label in phone_map:
        curr_evaluation_ranges = transitions_to_ranges(phone_map[phone_label]["evaluation_transitions"], "START_EVALUATION_PERIOD", "STOP_EVALUATION_PERIOD", 2, 3)
        print("Found %d ranges for phone %s" % (len(curr_evaluation_ranges), phone_label))
        phone_map[phone_label]["evaluation_ranges"] = curr_evaluation_ranges<jupyter_output><empty_output><jupyter_text>Validate the ranges for individual phonesThis involves two main checks:- that we have at least one evaluation range for each test in the spec. Note that we do not currently enforce that we have exactly one evaluation range for each test, on the theory that more evaluation is always good. But I am open to argument about this- that the settings in the evaluation range are consistent with the spec<jupyter_code>expected_config_map = {
    "fixed:ACCURACY_CONTROL": {
        "is_duty_cycling": False,
        "accuracy": ["PRIORITY_HIGH_ACCURACY","kCLLocationAccuracyBest"],
        "filter": 1,
    },
    "fixed:POWER_CONTROL": {
        "is_duty_cycling": False,
        "accuracy": ["PRIORITY_NO_POWER","kCLLocationAccuracyThreeKilometers"],
        "filter": 1200,
    }
}
for ct in curr_spec["sensing_settings"]:
    for s in ct["sensing_configs"]:
        expected_config_map["%s:%s" % (ct["name"], s["id"])] = s["sensing_config"]
# print(expected_config_map)
# Current accuracy constants
# Since we can't read these from the phone, we hardcoded them from the documentation
# If there are validation failures, these need to be updated
# In the future, we could upload the options from the phone (maybe the accuracy control)
# but that seems like overkill here

accuracy_options = {
    "android": {
        "PRIORITY_HIGH_ACCURACY": 100,
        "PRIORITY_BALANCED_POWER_ACCURACY": 102,
        "PRIORITY_LOW_POWER": 104,
        "PRIORITY_NO_POWER": 105
    },
    "ios": {
        "kCLLocationAccuracyBestForNavigation": -2,
        "kCLLocationAccuracyBest": -1,
        "kCLLocationAccuracyNearestTenMeters": 10,
        "kCLLocationAccuracyHundredMeters": 100,
        "kCLLocationAccuracyKilometer": 1000,
        "kCLLocationAccuracyThreeKilometers": 3000,
    }
}
opt_array_idx = lambda phoneOS: 0 if phoneOS == "android" else 1

def validate_filter(phoneOS, config_during_test, expected_config):
    # filter checking is a bit tricky because the expected value has two possible values and the real config has two possible values
    expected_filter = expected_config["filter"]
    if type(expected_filter) == int:
        ev = expected_filter
    else:
        assert type(expected_filter) == list, "platform specific filters should be specified in array, not %s" % expected_filter
        ev = expected_filter[opt_array_idx(phoneOS)]
        
    if phoneOS == "android":
        cvf = "filter_time"
    elif phoneOS == "ios":
        cvf = "filter_distance"
        
    assert config_during_test[cvf] == ev, "Field filter mismatch! %s != %s" % (config_during_test, expected_config)
    
def validate_accuracy(phoneOS, config_during_test, expected_config):
    # expected config accuracy is an array of strings ["PRIORITY_BALANCED_POWER_ACCURACY", "kCLLocationAccuracyNearestTenMeters"]
    # so we find the string at the correct index and then map it to the value from the options
    ev = accuracy_options[phoneOS][expected_config["accuracy"][opt_array_idx(phoneOS)]]
    assert config_during_test["accuracy"] == ev, "Field accuracy mismatch! %s != %s" % (config_during_test[accuracy], ev)

for phoneOS, phone_map in eval_transitions.items():
    # print("Processing data for %s phones, next level keys = %s" % (phoneOS, phone_map.keys()))
    for phone_label in phone_map:
        curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
        unique_test_ids = set(filter(lambda id: id != "fixed", [r["trip_id"].split(":")[0] for r in curr_evaluation_ranges]))
        # This is a tricky check since, unlike in the calibration case, we will have different ids for the different evaluation
        # ranges. For now, let us just assert that the evaluation range is valid
        # print("Unique test ids = %s" % unique_test_ids)
        spec_test_ids = set([ct["name"] for ct in curr_spec["sensing_settings"]])
        # print(spec_test_ids)
        # <= represents subset for set objects
        assert unique_test_ids <= spec_test_ids, "Invalid evaluation test while comparing %s, %s" % (unique_test_ids, spec_test_ids)
        for r in curr_evaluation_ranges:
            config_during_test_entries = retrieve_data_from_server(phone_label, ["config/sensor_config"], r["start_ts"], r["end_ts"])
            print("%s -> %s" % (r["trip_id"], [c["data"]["accuracy"] for c in config_during_test_entries]))
            # assert len(config_during_test_entries) == 1, "Out of band configuration? Found %d config changes" % len(config_during_test_entries)
            config_during_test = config_during_test_entries[0]["data"]
            expected_config = expected_config_map[r["trip_id"]]
            # print(config_during_test.keys(), expected_config.keys())
            validate_filter(phoneOS, config_during_test, expected_config)
            validate_accuracy(phoneOS, config_during_test, expected_config)
            for f in expected_config:
                if f != "accuracy" and f != "filter":
                    assert config_during_test[f] == expected_config[f], "Field %s mismatch! %s != %s" % (f, config_during_test[f], expected_config[f])<jupyter_output><empty_output><jupyter_text>Link evaluation ranges to each otherSince the evaluation ranges have separate IDs, it is hard to link them the way we link the calibration ranges.So let's add a common field (`eval_common_trip_id`) to each of the matching ranges for easy checking<jupyter_code>def link_common_eval_ranges():
    for phoneOS, phone_map in eval_transitions.items(): # android, ios
        print("Processing data for %s phones" % phoneOS)
        all_eval_ranges = [m["evaluation_ranges"] for m in phone_map.values()]
        # all the lengths are equal - i.e. the set of lengths has one entr
        assert len(set([len(a) for a in all_eval_ranges])) == 1
        for ctuple in zip(*all_eval_ranges):
            eval_cols = ctuple[1:-1]
            # print([ct["trip_id"] for ct in ctuple], [ct["trip_id"] for ct in eval_cols])
            get_common_name = lambda r: r["trip_id"].split(":")[0]
            get_separate_role = lambda r: r["trip_id"].split(":")[1]
            common_names = set([get_common_name(r) for r in eval_cols])
            separate_roles = [get_separate_role(r) for r in eval_cols]
            # print(separate_roles)
            assert len(common_names) == 1
            common_name = list(common_names)[0]
            # print(common_name)
            for r in ctuple:
                r["eval_common_trip_id"] = common_name
            ctuple[0]["eval_role"] = "accuracy_control"
            for r, sr in zip(eval_cols, separate_roles):
                r["eval_role"] = sr
            ctuple[-1]["eval_role"] = "power_control"
link_common_eval_ranges()
eval_transitions["android"]["ucb-sdb-android-1"]["evaluation_ranges"][0]<jupyter_output><empty_output><jupyter_text>Read and validate tripsfor each evaluation range, we have multiple possible trips<jupyter_code>phone_labels["android"]["ucb-sdb-android-1"]
accuracy_control_maps = {}
power_control_maps = {}
eval_phone_maps = {}

for phoneOS, phone_map in eval_transitions.items():
    print("Processing data for %s phones" % phoneOS)
    eval_phone_maps[phoneOS] = {}
    for phone_label in phone_map:
        curr_role = phone_map[phone_label]["role"]
        if curr_role == "accuracy_control":
            accuracy_control_maps[phoneOS] = phone_map[phone_label]
        elif curr_role == "power_control":
            power_control_maps[phoneOS] = phone_map[phone_label]
        else:
            assert curr_role.startswith("evaluation")
            eval_phone_maps[phoneOS][phone_label] = phone_map[phone_label]

print(len(accuracy_control_maps), len(power_control_maps), len(eval_phone_maps))
print(accuracy_control_maps.keys(), power_control_maps.keys(), eval_phone_maps.keys())
print(eval_phone_maps["android"].keys(), eval_phone_maps["ios"].keys())
# Fill in the control trip ranges
# Note that for the control phones, these are not split by evaluation range since there are no evaluation ranges for the controls
for phoneOS, phone_map in accuracy_control_maps.items():
    curr_control_transitions = [t["data"] for t in phone_map["transitions"]] # from control phone
    curr_evaluation_ranges = phone_map["evaluation_ranges"] # from this phone
    trip_type_check = lambda t: t["transition"] in ["START_EVALUATION_TRIP", "STOP_EVALUATION_TRIP", 4, 5]
    trip_time_check = lambda t, r: t["ts"] >= r["start_ts"] and t["ts"] <= r["end_ts"]
    for i, r in enumerate(curr_evaluation_ranges):
        # We have to get the evaluation details from one of the evaluation phones
        curr_eval_trips_transitions = [t for t in curr_control_transitions if trip_type_check(t) and trip_time_check(t, r)]
        # print("\n".join([str((t["transition"], t["trip_id"], t["ts"], arrow.get(t["ts"]).to(eval_tz))) for t in curr_eval_trips_transitions]))
        # print(len(curr_eval_trips_transitions))
        curr_eval_trips_ranges = transitions_to_ranges(curr_eval_trips_transitions, "START_EVALUATION_TRIP", "STOP_EVALUATION_TRIP", 4, 5)
        print("%s: Found %s trips for evaluation %s" % (phoneOS, len(curr_eval_trips_ranges), r["trip_id"]))
        # print(curr_eval_trips_ranges)
        print("\n".join([str((tr["trip_id"], tr["duration"], arrow.get(tr["start_ts"]).to(eval_tz), arrow.get(tr["end_ts"]).to(eval_tz))) for tr in curr_eval_trips_ranges]))
        r["evaluation_trip_ranges"] = curr_eval_trips_ranges<jupyter_output><empty_output><jupyter_text>Copy the ranges to the power control<jupyter_code>for phoneOS in accuracy_control_maps.keys():
    matching_accuracy_control_map = accuracy_control_maps[phoneOS]
    matching_power_control_map = power_control_maps[phoneOS]
    print(matching_power_control_map.keys())
    curr_power_evaluation_ranges = matching_power_control_map["evaluation_ranges"]
    curr_accuracy_evaluation_ranges = matching_accuracy_control_map["evaluation_ranges"]
    assert len(curr_power_evaluation_ranges) == len(curr_accuracy_evaluation_ranges)
    for i, (rp, ra) in enumerate(zip(curr_power_evaluation_ranges, curr_accuracy_evaluation_ranges)):
        accuracy_eval_trips_ranges = ra["evaluation_trip_ranges"] # from this phone
        print("%s: Copying %s accuracy trips to power, before = %s" % (phoneOS, len(accuracy_eval_trips_ranges),
            len(rp["evaluation_trip_ranges"]) if "evaluation_trip_ranges" in rp else 0))
        rp["evaluation_trip_ranges"] = copy.deepcopy(accuracy_eval_trips_ranges)
        # print(curr_eval_trips_ranges)
        print("\n".join([str((tr["trip_id"], tr["duration"], arrow.get(tr["start_ts"]).to(eval_tz), arrow.get(tr["end_ts"]).to(eval_tz))) for tr in curr_eval_trips_ranges]))<jupyter_output><empty_output><jupyter_text>Copy the ranges to the evaluation mapsSimilar to the power control, but there are potentially `n` of them<jupyter_code># phone_map has keys like 
for phoneOS in accuracy_control_maps.keys():
    matching_accuracy_control_map = accuracy_control_maps[phoneOS]
    matching_eval_phone_maps = eval_phone_maps[phoneOS]
    print(matching_eval_phone_maps.keys())
    for phone_label, phone_map in matching_eval_phone_maps.items():
        curr_eval_evaluation_ranges = phone_map["evaluation_ranges"]
        curr_accuracy_evaluation_ranges = matching_accuracy_control_map["evaluation_ranges"]
        assert len(curr_eval_evaluation_ranges) == len(curr_accuracy_evaluation_ranges)
        for i, (re, ra) in enumerate(zip(curr_eval_evaluation_ranges, curr_accuracy_evaluation_ranges)):
            accuracy_eval_trips_ranges = ra["evaluation_trip_ranges"] # from this phone
            print("%s: Copying %s accuracy trips to %s, before = %s" % (phoneOS, phone_label, len(accuracy_eval_trips_ranges),
                len(re["evaluation_trip_ranges"]) if "evaluation_trip_ranges" in re else 0))
            re["evaluation_trip_ranges"] = copy.deepcopy(accuracy_eval_trips_ranges)
            # print(curr_eval_trips_ranges)
            print("\n".join([str((tr["trip_id"], tr["duration"], arrow.get(tr["start_ts"]).to(eval_tz), arrow.get(tr["end_ts"]).to(eval_tz))) for tr in curr_eval_trips_ranges]))<jupyter_output><empty_output><jupyter_text>Validate ranges across phonesThis effectively has one test right now - is the duration of the tests across phones consistent?TODO: We should add a reasonable fuzz factor based on real evaluation.We are going to create a pandas dataframe with the following structure```                    android_ android_ android_ .......```Then, we can transpose it to get```                       ....android_android_...```then, we can get a series of durations for each `trip_id` as a series and compare it<jupyter_code>duration_map = {}
for phoneOS, phone_map in eval_transitions.items():
    print("Processing data for %s phones" % phoneOS)
    for phone_label in phone_map:
        curr_phone_duration_map = {}
        curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
        for r in curr_evaluation_ranges:
            duration_map[phone_label] = r["duration"]
#        duration_map[phoneOS+"_"+phone_label] = curr_phone_duration_map
print(duration_map)
        
duration_series = pd.Series(duration_map)
duration_series<jupyter_output><empty_output><jupyter_text>Since these are not statistical samples, the regular standard deviation/variation don't have much meaning. The variation is really caused by human control of the evaluation start/stop and the durations should be within a few minutes of each other. The expected variation defined in `MAX_DURATION_VARIATION`<jupyter_code>duration_variation = duration_series - duration_series.median()
print("duration_variation = %s" % (duration_variation.tolist()))
assert duration_variation.abs().max() < MAX_DURATION_VARIATION,\
    "INVALID: for %s, duration_variation.abs().max() %d > threshold %d" % (col, duration_variation.abs().max(), MAX_DURATION_VARIATION)
duration_map = {}
for phoneOS, phone_map in eval_phone_maps.items():
    print("Processing data for %s phones" % phoneOS)
    for phone_label in phone_map:
        curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
        for r in curr_evaluation_ranges:
            curr_phone_duration_map = {}
            for et in r["evaluation_trip_ranges"]:
                curr_phone_duration_map[et["trip_id"]] = et["duration"]
            duration_map[phoneOS+"_"+r["trip_id"]] = curr_phone_duration_map
        
duration_df = pd.DataFrame(duration_map).transpose()
duration_df<jupyter_output><empty_output><jupyter_text>Now, we can evaluate the actual values  Battery drain over time (overall) The data is in the format```android:    - :        -           - dataframe with index = ts, columns = other fields    - :        -           - dataframe with index = ts, columns = other fields...ios:    - :        -           - dataframe with index = ts, columns = other fields    - :        -           - dataframe with index = ts, columns = other fields```<jupyter_code>for phoneOS, phone_map in eval_transitions.items():
    print("Processing data for %s phones" % phoneOS)
    for phone_label in phone_map:
        curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
        for r in curr_evaluation_ranges:
            battery_entries = retrieve_data_from_server(phone_label, ["background/battery"], r["start_ts"], r["end_ts"])
            # ios entries before running the pipeline are marked with battery_level_ratio, which is a float from 0 ->1
            # convert it to % to be consistent with android and easier to understand
            if phoneOS == "ios":
                for e in battery_entries:
                    if "battery_level_pct" not in e["data"]:
                        e["data"]["battery_level_pct"] = e["data"]["battery_level_ratio"] * 100
                        del e["data"]["battery_level_ratio"]
            battery_df = pd.DataFrame([e["data"] for e in battery_entries])
            battery_df["hr"] = (battery_df.ts-r["start_ts"])/3600.0
            r["battery_df"] = battery_df
# test_battery_df = eval_transitions["android"]["ucb-sdb-android-3"]["evaluation_ranges"][0]["battery_df"]; test_battery_df
# test_battery_df = eval_transitions["android"]["ucb-sdb-android-4"]["evaluation_ranges"][0]["battery_df"]; test_battery_df
def plot_evaluation_curves(ax, phone_map):
    for phone_label in phone_map:
        curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
        for r in curr_evaluation_ranges:
            battery_df = r["battery_df"]
            ret_axes = battery_df.plot(x="hr", y="battery_level_pct", ax=ax, label=phone_label+"_"+r["trip_id"], ylim=(0,100), sharey=True)
(ifig, [android_ax, ios_ax]) = plt.subplots(ncols=1, nrows=2, figsize=(25,25))

plot_evaluation_curves(ios_ax, eval_transitions["ios"])
ios_ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
plot_evaluation_curves(android_ax, eval_transitions["android"])
android_ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
def plot_separate_evaluation_curves(fig, nRows, phone_map):
    for i, phone_label in enumerate(phone_map.keys()):
        ax = fig.add_subplot(nRows, 2, i+1, title=phone_label)
        curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
        for r in curr_evaluation_ranges:
            battery_df = r["battery_df"]
            # print(battery_df.battery_level_pct.tolist())
            battery_df.plot(x="hr", y="battery_level_pct", ax=ax, label=r["trip_id"], ylim=(0,100), sharey=True)
# (ifig, ax) = plt.subplots(figsize=(16,16))
# nRows = get_row_count(len(eval_transitions["ios"].keys()), 2)
# print(nRows)
# plot_separate_evaluation_curves(ifig, nRows, eval_transitions["android"])
# (ifig, ax) = plt.subplots(nrows=0, ncols=0, figsize=(16,16))
# nRows = get_row_count(len(eval_transitions["android"].keys()), 2)
# plot_separate_evaluation_curves(ifig, nRows, eval_transitions["ios"])<jupyter_output><empty_output><jupyter_text>Location points over time (overall) Now, we download the location points and check to see that the density is largely consistent<jupyter_code>for phoneOS, phone_map in eval_transitions.items():
    print("Processing data for %s phones" % phoneOS)
    for phone_label in phone_map:
        curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
        for r in curr_evaluation_ranges:
            all_done = False
            location_entries = []
            curr_start_ts = r["start_ts"]
            prev_retrieved_count = 0

            while not all_done:
                print("About to retrieve data for %s from %s -> %s" % (phone_label, curr_start_ts, r["end_ts"]))
                curr_location_entries = retrieve_data_from_server(phone_label, ["background/location"], curr_start_ts, r["end_ts"])
                print("Retrieved %d entries with timestamps %s..." % (len(curr_location_entries), [cle["data"]["ts"] for cle in curr_location_entries[0:10]]))
                if len(curr_location_entries) == 0 or len(curr_location_entries) == 1 or len(curr_location_entries) == prev_retrieved_count:
                    all_done = True
                else:
                    location_entries.extend(curr_location_entries)
                    curr_start_ts = curr_location_entries[-1]["data"]["ts"]
                    prev_retrieved_count = len(curr_location_entries)
            location_df = pd.DataFrame([e["data"] for e in location_entries])
            location_df["hr"] = (location_df.ts-r["start_ts"])/3600.0
            r["location_df"] = location_df
count_map = {}
for phoneOS, phone_map in eval_transitions.items():
    print("Processing data for %s phones" % phoneOS)
    for phone_label in phone_map:
        curr_phone_count_map = {}
        curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
        for r in curr_evaluation_ranges:
            curr_phone_count_map[r["trip_id"]] = len(r["location_df"])
        count_map[phoneOS+"_"+phone_label] = curr_phone_count_map
        
count_df = pd.DataFrame(count_map).transpose()
count_df            
count_df.plot(kind="bar", figsize=(16,6))
def get_location_density_df(phone_map):
    density_map = {}
    for phone_label in phone_map:
        curr_phone_density_map = {}
        curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
        for r in curr_evaluation_ranges:
            density_map[phone_label+"_"+r["trip_id"]] = r["location_df"].hr
        
    density_df = pd.DataFrame(density_map)
    return density_df
android_density_df = get_location_density_df(eval_transitions["android"])
nRows = get_row_count(len(android_density_df), 2)
print(nRows)
android_density_df.plot(kind='density', subplots=False, layout=(nRows, 2), figsize=(10,10), sharex=True, sharey=True)
def plot_separate_density_curves(fig, nRows, phone_map):
    for i, phone_label in enumerate(phone_map.keys()):
        ax = fig.add_subplot(nRows, 2, i+1, title=phone_label)
        curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
        for r in curr_evaluation_ranges:
            location_df = r["location_df"]
            # print(battery_df.battery_level_pct.tolist())
            location_df.hr.plot(kind='density', ax=ax, label=r["trip_id"], sharex=True, sharey=True)
            ax.legend()
# (ifig, ax) = plt.subplots(nrows=0, ncols=0, figsize=(16,16))
# nRows = get_row_count(len(eval_transitions["android"].keys()), 2)
# print(nRows)
# plot_separate_density_curves(ifig, nRows, eval_transitions["android"])
ios_density_df = get_location_density_df(eval_transitions["ios"])
nRows = get_row_count(len(eval_transitions["ios"].keys()), 2)
print(nRows)
ios_density_df.plot(kind='density', subplots=False, layout=(nRows, 2), figsize=(10,10), sharex=True, sharey=True)
# (ifig, ax) = plt.subplots(nrows=0, ncols=0, figsize=(16,16))
# nRows = get_row_count(len(eval_transitions["ios"].keys()), 2)
# print(nRows)
# plot_separate_density_curves(ifig, nRows, eval_transitions["ios"])
# test_loc_df = eval_transitions["ios"]["ucb-sdb-ios-1"]["evaluation_ranges"][0]["location_df"]; test_loc_df.head()
# test_loc_df = eval_transitions["ios"]["ucb-sdb-ios-2"]["evaluation_ranges"][0]["location_df"]; test_loc_df.head()
def plot_density_vs_power_curves(fig, nRows, phone_map, sel_trip_id):
    for i, phone_label in enumerate(phone_map.keys()):
        ax = fig.add_subplot(nRows, 2, i+1)
        curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
        for r in curr_evaluation_ranges:
            if r["trip_id"] == sel_trip_id:
                battery_df = r["battery_df"]
                location_df = r["location_df"]
                battery_df.plot(x="hr", y="battery_level_pct", ax=ax, label=phone_label, sharex=True, sharey=True)
                location_df.hr.plot(ax=ax, kind="density", secondary_y=True)
# fig = plt.figure(figsize=(16, 16))
# plot_density_vs_power_curves(fig, nRows, eval_transitions["android"], CURR_TRIP_ID)
# fig = plt.figure(figsize=(16, 16))
# plot_density_vs_power_curves(fig, nRows, eval_transitions["ios"], CURR_TRIP_ID)<jupyter_output><empty_output><jupyter_text>Location points over space (overall)<jupyter_code>def get_map_list_for_trip():
    map_list = []
    for phoneOS, phone_map in eval_transitions.items():
        print("Processing data for %s phones" % phoneOS)
        for phone_label in phone_map:
            curr_evaluation_ranges = phone_map[phone_label]["evaluation_ranges"]
            for r in curr_evaluation_ranges:
                curr_map = folium.Map()
                location_df = r["location_df"]
                latlng_route_coords = list(zip(location_df.latitude, location_df.longitude))
                # print(latlng_route_coords[0:10])
                pl = folium.PolyLine(latlng_route_coords,
                    popup="%s: %s" % (phoneOS, phone_label))
                pl.add_to(curr_map)
                curr_bounds = pl.get_bounds()
                print(curr_bounds)
                top_lat = curr_bounds[0][0]
                mid_lng = (curr_bounds[0][1] + curr_bounds[1][1])/2
                print("midpoint = %s, %s, plotting at %s, %s" % (top_lat,mid_lng, top_lat, mid_lng))
                folium.map.Marker(
                    [top_lat, mid_lng],
                    icon=fof.DivIcon(
                        icon_size=(200,36),
                        html='<div style="font-size: 12pt; color: green;">%s %s</div>' % (phone_label, r["trip_id"]))
                ).add_to(curr_map)
                curr_map.fit_bounds(pl.get_bounds())
                map_list.append(curr_map)
    return map_list
ha_map_list = get_map_list_for_trip()
rows = get_row_count(len(ha_map_list), 2)
evaluation_maps = bre.Figure(ratio="{}%".format((rows/4) * 100))
for i, curr_map in enumerate(ha_map_list):
    evaluation_maps.add_subplot(rows, 2, i+1).add_child(curr_map)
evaluation_maps<jupyter_output><empty_output><jupyter_text>Fill in trip-specific information and reorder maps to make it easier to compare tests<jupyter_code>def fill_trip_specific_battery_and_locations():
    for phoneOS, phone_map in eval_transitions.items(): # android, ios
        for phone_label in phone_map:
            print("Filling label %s for OS %s" % (phone_label, phoneOS))
            for r in phone_map[phone_label]["evaluation_ranges"]:
                # print(r["battery_df"].head())
                for tr in r["evaluation_trip_ranges"]:
                    query = "ts > %s & ts <= %s" % (tr["start_ts"], tr["end_ts"])
                    # print("%s %s %s" % (phone_label, tr["trip_id"], query))
                    # print(r["battery_df"].query(query).head())
                    tr["battery_df"] = r["battery_df"].query(query)
                    # print(80 * '~')
                    # print(tr["battery_df"])
                    tr["location_df"] = r["location_df"].query(query)
                    # print(80 * "-")
fill_trip_specific_battery_and_locations()
# eval_transitions["android"]["ucb-sdb-android-1"]["evaluation_ranges"][0]["battery_df"].query("ts > 1560978591 & ts <= 1560980647").head()
# eval_transitions["android"]["ucb-sdb-android-1"]["evaluation_ranges"][0]["evaluation_trip_ranges"]
def find_comparison_regimes():
    comparison_map = {}
    for phoneOS, phone_map in eval_transitions.items(): # android, ios
        print("Processing data for %s phones" % phoneOS)
        comparison_map[phoneOS] = {}
        all_eval_ranges = [m["evaluation_ranges"] for m in phone_map.values()]
        # all the lengths are equal - i.e. the set of lengths has one entr
        assert len(set([len(a) for a in all_eval_ranges])) == 1
        for ctuple in zip(*all_eval_ranges):
            common_names = set([r["eval_common_trip_id"] for r in ctuple])
            assert len(common_names) == 1
            common_name = list(common_names)[0]
            # print(common_name)
            comparison_map[phoneOS][common_name] = {}
            
            separate_roles = [r["eval_role"] for r in ctuple]
            print(separate_roles)

            eval_trips_for_range = [r["evaluation_trip_ranges"] for r in ctuple]
            # print([len(et) for et in eval_trips_for_range])
            assert len(set([len(et) for et in eval_trips_for_range])) == 1
            for ctriptuple in zip(*(eval_trips_for_range)):
                # print([ctt["trip_id"] for ctt in ctriptuple])
                common_trip_ids = set([ctt["trip_id"] for ctt in ctriptuple])
                assert(len(common_trip_ids)) == 1
                common_trip_id = list(common_trip_ids)[0]
                print(common_trip_id)
                comparison_map[phoneOS][common_name][common_trip_id] = {}
                for cr, ctt in zip(separate_roles, ctriptuple):
                    comparison_map[phoneOS][common_name][common_trip_id][cr] = ctt
    return comparison_map
comparison_map = find_comparison_regimes()
print(comparison_map["android"].keys())
print(comparison_map["android"]['HAHFDC v/s HAMFDC'].keys())
print(comparison_map["android"]['HAHFDC v/s HAMFDC']['short_walk_suburb'].keys())
print(comparison_map["android"]['HAHFDC v/s HAMFDC']['short_walk_suburb']['HAHFDC'].keys())<jupyter_output><empty_output><jupyter_text>Battery drain over time (per-trip)<jupyter_code>def plot_per_trip_evaluation_curves(fig, nRows, phone_map):
    i = 0
    for curr_eval, curr_eval_trip_map in phone_map.items():
        for curr_eval_trip_id, eval_trip_compare_map in curr_eval_trip_map.items():
            ax = fig.add_subplot(nRows, 2, i+1, title=curr_eval_trip_id)
            i = i+1
            for compare_id, compare_tr in eval_trip_compare_map.items():
                battery_df = compare_tr["battery_df"]
                if len(battery_df) > 0:
                    battery_df.plot(x="hr", y="battery_level_pct", ax=ax, label=compare_id, ylim=(0,100), sharey=True)
                else:
                    print("no battery data found for %s %s, skipping" % (curr_eval, curr_eval_trip_id))
(ifig, ax) = plt.subplots(figsize=(16,16),nrows=0,ncols=0)
exp_plot_counts = [len(curr_eval_trip_map.values()) for curr_eval, curr_eval_trip_map in comparison_map["android"].items()]
print(exp_plot_counts)
nRows = get_row_count(sum(exp_plot_counts), 2)
print(nRows)
plot_per_trip_evaluation_curves(ifig, nRows, comparison_map["ios"])
(ifig, ax) = plt.subplots(figsize=(16,16),nrows=0,ncols=0)
exp_plot_counts = [len(curr_eval_trip_map.values()) for curr_eval, curr_eval_trip_map in comparison_map["ios"].items()]
print(exp_plot_counts)
nRows = get_row_count(sum(exp_plot_counts), 2)
print(nRows)
plot_per_trip_evaluation_curves(ifig, nRows, comparison_map["android"])<jupyter_output><empty_output><jupyter_text>Location points over space (per-trip)<jupyter_code>def get_per_trip_map_list():
    map_list = []
    color_list = ['blue', 'red', 'purple', 'orange']
    for phoneOS, phone_map in comparison_map.items():
        print("Processing data for %s phones" % phoneOS)
        for curr_eval, curr_eval_trip_map in phone_map.items():
            for curr_eval_trip_id, eval_trip_compare_map in curr_eval_trip_map.items():
                curr_map = folium.Map()
                all_points = []
                for i, (compare_id, compare_tr) in enumerate(eval_trip_compare_map.items()):
                    if i == len(eval_trip_compare_map) - 1:
                        print("Skipping the last item (power_control)")
                        continue
                    location_df = compare_tr["location_df"]
                    latlng_route_coords = list(zip(location_df.latitude, location_df.longitude))
                    all_points.extend(latlng_route_coords)
                    # print(latlng_route_coords[0:10])
                    if len(latlng_route_coords) > 0:
                        print("Processing %s, %s, %s, found %d locations, adding to map" %
                          (curr_eval, curr_eval_trip_id, compare_id, len(latlng_route_coords)))
                        pl = folium.PolyLine(latlng_route_coords,
                            popup="%s" % (compare_id), color=color_list[i])
                        pl.add_to(curr_map)
                    else:
                        print("Processing %s, %s, %s, found %d locations, skipping" %
                          (curr_eval, curr_eval_trip_id, compare_id, len(latlng_route_coords)))
                curr_bounds = ful.get_bounds(all_points)
                print(curr_bounds)
                top_lat = curr_bounds[0][0]
                mid_lng = (curr_bounds[0][1] + curr_bounds[1][1])/2
                print("for trip %s with %d points, midpoint = %s, %s, plotting at %s, %s" %
                      (curr_eval_trip_id, len(all_points), top_lat,mid_lng, top_lat, mid_lng))
                folium.map.Marker(
                    [top_lat, mid_lng],
                    icon=fof.DivIcon(
                        icon_size=(200,36),
                        html='<div style="font-size: 12pt; color: green;">%s: %s</div>' % (phoneOS, curr_eval_trip_id))
                ).add_to(curr_map)
                curr_map.fit_bounds(pl.get_bounds())
                map_list.append(curr_map)
    return map_list
ha_map_list = get_per_trip_map_list()
rows = get_row_count(len(ha_map_list), 2)
print(rows)
evaluation_maps = bre.Figure(ratio="{}%".format((rows/4) * 100))
for i, curr_map in enumerate(ha_map_list):
    # print("Adding map %s at %d" % (curr_map, i))
    evaluation_maps.add_subplot(rows, 2, i+1).add_child(curr_map)
evaluation_maps<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12009249237429893, 'fraction_numerical': 0.007453507822493358, 'mean_word_length': 3.817276928546036, 'pattern_counts': {'":': 38, '<': 97, '<?xml version=': 0, '>': 104, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '183004', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 13481, 'n_tokens_neox': 13166, 'n_words': 2771, 'path': 'evaluation_templates/Validate_evaluation.ipynb', 'repo_name': 'maufadel/mobilitynet-analysis-scripts', 'size': '40690'}