<commit_before>"""
Resource for Reports. 

The Reports are specified collections of data that can easily be visualised
in a html or pdf report. 

This large file includes the following reports:

NCD Report
CD Report
Public Health Profile
NCD Profile
CD Profile
Refugee Profile
Refugee CD Report
Refugee Detailed Report
"""

from flask_restful import Resource
from sqlalchemy import or_, func, desc, Integer
from datetime import datetime, timedelta
from dateutil import parser
from sqlalchemy.sql import text
import uuid
from gettext import gettext
import logging
from meerkat_api.util import get_children, is_child, fix_dates
from meerkat_api import db, app
from meerkat_abacus.model import Data, Locations, Alerts, AggregationVariables
from meerkat_api.resources.completeness import Completeness
from meerkat_api.resources.variables import Variables
from meerkat_api.resources.epi_week import EpiWeek, epi_week_start
from meerkat_api.resources.locations import TotClinics
from meerkat_api.resources import alerts
from meerkat_api.resources.explore import QueryVariable, query_ids
from meerkat_abacus.util import get_locations, all_location_data
from meerkat_abacus import model
from meerkat_api.authentication import require_api_key

def get_disease_types(category, start_date, end_date, location, conn):
    """ 
    Get and return top five disease of a type

    Args: 
       category: the category to get the top five disease from
       start_date: the start date for the aggregation
       end_date: the end_date for the aggregation
       location: the location to incldue
       conn: db.connection

    Returns: 
       top_five_disease(list): ordered list of top five disease with percentage
    
    """
    diseases = get_variables_category(category, start_date,
                                      end_date, location, conn)
    tot_n = sum(diseases.values())
    if tot_n == 0:
        tot_n = 1
    ret = []
    for disease in top(diseases):
        if diseases[disease] > 0:
            ret.append(make_dict(disease,
                                 diseases[disease],
                                 diseases[disease] / tot_n * 100))
    return ret

def make_dict(title, quantity, percent):
    """
    Small utility to create dictionary with title, quantity and percent

    Args: 
       title: The title
       quantity: quantity
       percent: percent
    
    Returns: 
       dict(dict): Dictionary
    """
    return {"title": title,
            "quantity": quantity,
            "percent": percent}

def top(values, number=5):
    """
    Return the dict containg the top number(defaults to 5) values.
    If we ask for the top 2 and the data looks like {"A": 5, "B": 5, "C": 5}
    we will sort on the keys and take A and B. 

    Args:
       values: the dict with data
       number: how many we include (default=5)
    Returns:
       top_x(dict): dict with the number highest values
    """
    return sorted(values, key=lambda k: (-values[k], k))[:number]


# A predifend query to use in get_variable_id
qu = text("SELECT sum(CAST(data.variables ->> :variables_1 AS INTEGER)) AS sum_1  FROM data WHERE (data.variables ? :variables_2) AND data.date >= :date_1 AND data.date < :date_2 AND (data.country = :country_1 OR data.region = :region_1 OR data.district = :district_1 OR data.clinic = :clinic_1)")


def get_variable_id(variable_id, start_date, end_date, location, conn):
    """
    Get the sum of variable_id between start and end date with location

    Args: 
       variable_id: the variable_id to aggregate
       start_date: the start date for the aggregation
       end_date: the end_date for the aggregation
       location: the location to incldue
       conn: db.connection

    Returns: 
       result(int): result of the aggregation
    """

    result = conn.execute(qu, variables_1=variable_id,
                          variables_2=variable_id,
                          date_1=start_date,
                          date_2=end_date,
                          country_1=location,
                          region_1=location,
                          district_1=location,
                          clinic_1=location).fetchone()[0]
    if result:
        return result
    else:
        return 0

# Common variables_instance
variables_instance = Variables()


def map_variable( variable_id, start_date, end_date, location, conn ):
    """
    Map a given variable between dates and with location

    Args: 
       variable_id: the variable to be mapped
       start_date: the start date for the aggregation
       end_date: the end_date for the aggregation
       location: the location to incldue
       conn: db.connection
       use_ids: we use ids instead of names as keys for the return dict

    Returns: 
       dict
    """

    results = db.session.query(
        func.sum( Data.variables[variable_id].astext.cast(Integer) ).label('value'),
        Data.geolocation,
        Data.clinic
    ).filter( 
        Data.variables.has_key(variable_id ),
        Data.date >= start_date, 
        Data.date < end_date,
        or_(
            loc == location for loc in ( Data.country,
                                         Data.region,
                                         Data.district,
                                         Data.clinic)  
        )
    ).group_by("clinic", "geolocation")

    locations = get_locations(db.session)
    ret = {}
    for r in results.all():
        if r[1]:
            ret[r[2]] = {"value": r[0], "geolocation": r[1].split(","),
                         "clinic": locations[r[2]].name}

    return ret




def get_variables_category(category, start_date, end_date, location, conn, use_ids=False):
    """
    Aggregate category between dates and with location

    Args: 
       category: the category to get the top five disease from
       start_date: the start date for the aggregation
       end_date: the end_date for the aggregation
       location: the location to incldue
       conn: db.connection
       use_ids: we use ids instead of names as keys for the return dict

    Returns: 
       aggregate_category(dict): dict with {variable: number, variable2: number3, ...}
    """
    variables = variables_instance.get(category)

    return_data = {}
    for variable in variables.keys():
        r = get_variable_id(variable,
                           start_date,
                           end_date,
                            location,
                            conn)
        if use_ids:
            return_data[variable] = r
        else:
            return_data[variables[variable]["name"]] = r
    return return_data


def disease_breakdown(diseases):
    """
    Calculate the age and gender breakdown of dieases with codes of the following 
    format: "Disease, Gender AgeGroup". This is to deal with the refugee data
    that is submitted in an aggregated format instead of by cases.

    Args: 
       diseases: dict with Variable Name: value
    Returns:
      diseases(dict): formatted disease dict with dieases and age and gender breakdown

    """
    ret = {"diseases": {}}
    disease_total = {}
    age_gender_total = {}
    age_total = {}
    for d in diseases:
        split = d.split(",")
        disease_name = ",".join(split[:-1])
        age_gender = split[-1]
        disease_name = disease_name.strip()
        gender, age = age_gender.strip().split(" ")
        gender = gender.lower()
        age = age.strip()
        gender = gender.strip()
        disease_total.setdefault(disease_name, 0)
        disease_total[disease_name] += diseases[d]
        age_total.setdefault(age, 0)
        age_total[age] += diseases[d]
        age_gender_total.setdefault(age, {"male": 0, "female": 0})
        age_gender_total[age][gender] += diseases[d]
        ret["diseases"].setdefault(disease_name, {})
        ret["diseases"][disease_name].setdefault(age, {})
        ret["diseases"][disease_name][age][gender] = diseases[d]
    for d in disease_total:
        ret["diseases"][d]["total"] = disease_total[d]
    ret["age_gender"] = age_gender_total
    ret["age"] = age_total
    return ret


def get_latest_category(category, clinic, start_date, end_date):
    """
    To deal with data submitted in an aggregated way. We have e.g Population data that is
    submitted as the number of males <5, males 5-14 etc. These indicators are also not cumulative, 
    we just want to find the latest record where any of the population data(in this example) has
    been submitted. Once that record is found we sort out the variable names that are of this format
    "Category, Gender AgeGroup" into a structured dict of demographics. 

    Args:get_variables_category(category, start_date, end_date, location, conn, use_ids=False):
       category: the category to get the top five disease from
       clinic: the clinic we are looking at
       start_date: the start date for the aggregation
       end_date: the end_date for the aggregation
    Returns: 
       latest_demo(dict): the demographics from the latest record

    """
    variables = variables_instance.get(category)
    keys = sorted(variables.keys())

    result = db.session.query(Data.variables).filter(
        or_(Data.variables.has_key(key) for key in keys),
        Data.clinic == clinic,
        Data.date >= start_date,
        Data.date < end_date
            ).order_by(Data.date.desc()).first()
    ret = {}
    for key in keys:
        name = variables[key]["name"]
        gender, age = name.split(",")[1].strip().split(" ")
        gender = gender.lower()
        age = age.strip()
        gender = gender.strip()
        ret.setdefault(age, {"female": 0, "male": 0})        
        if(result and key in result[0]):
            ret[age][gender] += result[0][key]

    return ret

def refugee_disease(disease_demo):
    """
    From a dict of Variable Name: value where variable name includes
    gender and age information we want to just find the total numbers
    for each diseases.

    Args: 
       disease_demo: dict with disease data that includes demographics in the variable name
    Returns: 
       diseases(dict): diseases with their aggregated value over all demographics
    """
    diseases = {}
    for disease in disease_demo.keys():
        disease_name = disease.split(",")[0].strip()
        diseases.setdefault(disease_name, 0)
        diseases[disease_name] += disease_demo[disease]

    tot_n = sum(diseases.values())
    if tot_n == 0:
        tot_n = 1
    ret = []
    for disease in top(diseases):
        if diseases[disease] > 0:
            ret.append(make_dict(disease,
                                 diseases[disease],
                                 diseases[disease] / tot_n * 100))
    return ret


class NcdReport(Resource):
    """
    Ncd Report to show data on Hypertension and Diabetes. The data includes 
    breakdowns by age and on lab data, complications and comorbidity. We create
    tables with rows for regions. 

    Args:\n
       location: Location to generate report for\n
       start_date: Start date of report\n
       end_date: End date of report\n
    Returns:\n
       report_data\n
    """
    decorators = [require_api_key]
    def get(self, location, start_date=None, end_date=None):
        start_date, end_date = fix_dates(start_date, end_date)
        end_date_limit = end_date + timedelta(days=1)
        ret = {}
        #meta data
        ret["meta"] = {"uuid": str(uuid.uuid4()),
                       "project_id": 1,
                       "generation_timestamp": datetime.now().isoformat(),
                       "schema_version": 0.1
        }
        # Dates and Location Name
        ew = EpiWeek()
        epi_week = ew.get(end_date.isoformat())["epi_week"]
        ret["data"] = {"epi_week_num": epi_week,
                       "end_date": end_date.isoformat(),
                       "project_epoch": datetime(2015, 5, 20).isoformat(),
                       "start_date": start_date.isoformat()
        }
        location_name = db.session.query(Locations.name).filter(
            Locations.id == location).first()
        if not location_name:
            return None
        ret["data"]["project_region"] = location_name.name
        tot_clinics = TotClinics()
        ret["data"]["clinic_num"] = tot_clinics.get(location)["total"]
        # Data on Hypertension and Diabebtes, there are two tables for each disease.
        # One for the age breakdown, and one for labs and complications.
        # For each table we have rows for each Region.
        # Each of these tables have a title key for the X-axis titles
        # Data is list of rows(dicts) with a title for the y-axis title and
        # a list of values that are the values for the row in the right order
        
        ret["hypertension"] = {"age": {}, "complications": {}}
        ret["diabetes"] = {"age": {}, "complications": {}}

        diabetes_id = "ncd_1"
        hypertension_id = "ncd_2"
        diseases = {"hypertension": hypertension_id,
                    "diabetes": diabetes_id}
        ids_to_include = {"hypertension": [("lab_4", "lab_3"), ("lab_5", "lab_3"), ("lab_2", "lab_1"), ("com_1", "tot"), ("smo_2", "smo_4"), ("lab_11", "lab_10")],
                          "diabetes": [("lab_4", "lab_3"), ("lab_5", "lab_3"), ("lab_7", "lab_6"), ("lab_9", "lab_8"), ("com_2", "tot"), ("smo_2", "smo_4"), ("lab_11", "lab_10")]
        }
  
        locations, ldid, regions, districts = all_location_data(db.session)
        v = Variables()
        ages = v.get("ncd_age")
        
        # Loop through diabetes and hypertension
        for disease in diseases.keys():
            # First sort out the titles
            ret[disease]["age"]["titles"] = [gettext("reg")]
            ret[disease]["age"]["data"] = []
            for age in sorted(ages.keys()):
                ret[disease]["age"]["titles"].append(ages[age]["name"])
            ret[disease]["age"]["titles"].insert(1,"Total")
            ret[disease]["complications"]["titles"] = ["reg",
                                                       "tot",
                                                       "gen_1",
                                                       "gen_2"]
  
            for i in ids_to_include[disease]:
                ret[disease]["complications"]["titles"].append(i[0])
            ret[disease]["complications"]["data"] = []


            # Loop through each region, we add [1] to include the whole country
            for i, region in enumerate( [1] + sorted(regions) ):

                d_id = diseases[disease]
                query_variable = QueryVariable()
                # get the age breakdown
                disease_age = query_variable.get(d_id, "ncd_age",
                                                 end_date=end_date_limit.isoformat(),
                                                 start_date=start_date.isoformat(),
                                                 only_loc=region,
                                                 use_ids=True)
                loc_name = locations[region].name
                if region == 1:
                    loc_name = gettext("Total")
                ret[disease]["age"]["data"].append(
                    {"title": loc_name, "values": []}
                )
                
                for age in sorted(ages.keys()):
                    ret[disease]["age"]["data"][i]["values"].append(disease_age[age]["total"])
                ret[disease]["age"]["data"][i]["values"].insert(0,sum( [a["total"] for a in disease_age.values()]))
                # Get gender breakdown
                disease_gender = query_variable.get(d_id, "gender",
                                                    end_date=end_date_limit.isoformat(),
                                                    start_date=start_date.isoformat(),
                                                    only_loc=region)

                table_two_total = sum([disease_gender[gender]["total"] for gender in disease_gender])
                ret[disease]["complications"]["data"].append(
                    {
                        "title": loc_name,
                        "values": [table_two_total]
                    })
                if table_two_total == 0:
                    table_two_total = 1
                ret[disease]["complications"]["data"][i]["values"].append([disease_gender["Male"]["total"],  disease_gender["Male"]["total"] /table_two_total * 100])
                ret[disease]["complications"]["data"][i]["values"].append([disease_gender["Female"]["total"],  disease_gender["Female"]["total"] / table_two_total * 100])


                
                # Get the lab breakdown
                for new_id in ids_to_include[disease]:
                    if new_id[0]:
                        numerator = query_ids([d_id, new_id[0]], start_date, end_date_limit, only_loc=region)
                        if new_id[1] == "tot":
                            denominator = table_two_total
                        else:
                            denominator = query_ids([d_id, new_id[1]], start_date, end_date_limit, only_loc=region)
                        if denominator == 0:
                            denominator = 1
                        ret[disease]["complications"]["data"][i]["values"].append(
                            [numerator, numerator/ denominator * 100])
                    else:
                        # We can N/A to the table if it includes data we are not collecting
                        ret[disease]["complications"]["data"][i]["values"].append("N/A")
            
        return ret


class CdReport(Resource):
    """
    Communicable Disease Report

    This report includes data on all disease that generate alerts. For each
    of these diseases we generate a timeline of the number of cases per week. 
    We split these cases into supected and confrimed cases and only show data
    for disease where there at least one case. 

    Args:\n
       location: Location to generate report for\n
       start_date: Start date of report\n
       end_date: End date of report\n
    Returns:\n
       report_data\n
    """
    decorators = [require_api_key]
    
    def get(self, location, start_date = None,end_date=None):<commit_msg>email_summary for ncd report<commit_after>"""
Resource for Reports. 

The Reports are specified collections of data that can easily be visualised
in a html or pdf report. 

This large file includes the following reports:

NCD Report
CD Report
Public Health Profile
NCD Profile
CD Profile
Refugee Profile
Refugee CD Report
Refugee Detailed Report
"""

from flask_restful import Resource
from sqlalchemy import or_, func, desc, Integer
from datetime import datetime, timedelta
from dateutil import parser
from sqlalchemy.sql import text
import uuid
from gettext import gettext
import logging
from meerkat_api.util import get_children, is_child, fix_dates
from meerkat_api import db, app
from meerkat_abacus.model import Data, Locations, Alerts, AggregationVariables
from meerkat_api.resources.completeness import Completeness
from meerkat_api.resources.variables import Variables
from meerkat_api.resources.epi_week import EpiWeek, epi_week_start
from meerkat_api.resources.locations import TotClinics
from meerkat_api.resources import alerts
from meerkat_api.resources.explore import QueryVariable, query_ids
from meerkat_abacus.util import get_locations, all_location_data
from meerkat_abacus import model
from meerkat_api.authentication import require_api_key

def get_disease_types(category, start_date, end_date, location, conn):
    """ 
    Get and return top five disease of a type

    Args: 
       category: the category to get the top five disease from
       start_date: the start date for the aggregation
       end_date: the end_date for the aggregation
       location: the location to incldue
       conn: db.connection

    Returns: 
       top_five_disease(list): ordered list of top five disease with percentage
    
    """
    diseases = get_variables_category(category, start_date,
                                      end_date, location, conn)
    tot_n = sum(diseases.values())
    if tot_n == 0:
        tot_n = 1
    ret = []
    for disease in top(diseases):
        if diseases[disease] > 0:
            ret.append(make_dict(disease,
                                 diseases[disease],
                                 diseases[disease] / tot_n * 100))
    return ret

def make_dict(title, quantity, percent):
    """
    Small utility to create dictionary with title, quantity and percent

    Args: 
       title: The title
       quantity: quantity
       percent: percent
    
    Returns: 
       dict(dict): Dictionary
    """
    return {"title": title,
            "quantity": quantity,
            "percent": percent}

def top(values, number=5):
    """
    Return the dict containg the top number(defaults to 5) values.
    If we ask for the top 2 and the data looks like {"A": 5, "B": 5, "C": 5}
    we will sort on the keys and take A and B. 

    Args:
       values: the dict with data
       number: how many we include (default=5)
    Returns:
       top_x(dict): dict with the number highest values
    """
    return sorted(values, key=lambda k: (-values[k], k))[:number]


# A predifend query to use in get_variable_id
qu = text("SELECT sum(CAST(data.variables ->> :variables_1 AS INTEGER)) AS sum_1  FROM data WHERE (data.variables ? :variables_2) AND data.date >= :date_1 AND data.date < :date_2 AND (data.country = :country_1 OR data.region = :region_1 OR data.district = :district_1 OR data.clinic = :clinic_1)")


def get_variable_id(variable_id, start_date, end_date, location, conn):
    """
    Get the sum of variable_id between start and end date with location

    Args: 
       variable_id: the variable_id to aggregate
       start_date: the start date for the aggregation
       end_date: the end_date for the aggregation
       location: the location to incldue
       conn: db.connection

    Returns: 
       result(int): result of the aggregation
    """

    result = conn.execute(qu, variables_1=variable_id,
                          variables_2=variable_id,
                          date_1=start_date,
                          date_2=end_date,
                          country_1=location,
                          region_1=location,
                          district_1=location,
                          clinic_1=location).fetchone()[0]
    if result:
        return result
    else:
        return 0

# Common variables_instance
variables_instance = Variables()


def map_variable( variable_id, start_date, end_date, location, conn ):
    """
    Map a given variable between dates and with location

    Args: 
       variable_id: the variable to be mapped
       start_date: the start date for the aggregation
       end_date: the end_date for the aggregation
       location: the location to incldue
       conn: db.connection
       use_ids: we use ids instead of names as keys for the return dict

    Returns: 
       dict
    """

    results = db.session.query(
        func.sum( Data.variables[variable_id].astext.cast(Integer) ).label('value'),
        Data.geolocation,
        Data.clinic
    ).filter( 
        Data.variables.has_key(variable_id ),
        Data.date >= start_date, 
        Data.date < end_date,
        or_(
            loc == location for loc in ( Data.country,
                                         Data.region,
                                         Data.district,
                                         Data.clinic)  
        )
    ).group_by("clinic", "geolocation")

    locations = get_locations(db.session)
    ret = {}
    for r in results.all():
        if r[1]:
            ret[r[2]] = {"value": r[0], "geolocation": r[1].split(","),
                         "clinic": locations[r[2]].name}

    return ret




def get_variables_category(category, start_date, end_date, location, conn, use_ids=False):
    """
    Aggregate category between dates and with location

    Args: 
       category: the category to get the top five disease from
       start_date: the start date for the aggregation
       end_date: the end_date for the aggregation
       location: the location to incldue
       conn: db.connection
       use_ids: we use ids instead of names as keys for the return dict

    Returns: 
       aggregate_category(dict): dict with {variable: number, variable2: number3, ...}
    """
    variables = variables_instance.get(category)

    return_data = {}
    for variable in variables.keys():
        r = get_variable_id(variable,
                           start_date,
                           end_date,
                            location,
                            conn)
        if use_ids:
            return_data[variable] = r
        else:
            return_data[variables[variable]["name"]] = r
    return return_data


def disease_breakdown(diseases):
    """
    Calculate the age and gender breakdown of dieases with codes of the following 
    format: "Disease, Gender AgeGroup". This is to deal with the refugee data
    that is submitted in an aggregated format instead of by cases.

    Args: 
       diseases: dict with Variable Name: value
    Returns:
      diseases(dict): formatted disease dict with dieases and age and gender breakdown

    """
    ret = {"diseases": {}}
    disease_total = {}
    age_gender_total = {}
    age_total = {}
    for d in diseases:
        split = d.split(",")
        disease_name = ",".join(split[:-1])
        age_gender = split[-1]
        disease_name = disease_name.strip()
        gender, age = age_gender.strip().split(" ")
        gender = gender.lower()
        age = age.strip()
        gender = gender.strip()
        disease_total.setdefault(disease_name, 0)
        disease_total[disease_name] += diseases[d]
        age_total.setdefault(age, 0)
        age_total[age] += diseases[d]
        age_gender_total.setdefault(age, {"male": 0, "female": 0})
        age_gender_total[age][gender] += diseases[d]
        ret["diseases"].setdefault(disease_name, {})
        ret["diseases"][disease_name].setdefault(age, {})
        ret["diseases"][disease_name][age][gender] = diseases[d]
    for d in disease_total:
        ret["diseases"][d]["total"] = disease_total[d]
    ret["age_gender"] = age_gender_total
    ret["age"] = age_total
    return ret


def get_latest_category(category, clinic, start_date, end_date):
    """
    To deal with data submitted in an aggregated way. We have e.g Population data that is
    submitted as the number of males <5, males 5-14 etc. These indicators are also not cumulative, 
    we just want to find the latest record where any of the population data(in this example) has
    been submitted. Once that record is found we sort out the variable names that are of this format
    "Category, Gender AgeGroup" into a structured dict of demographics. 

    Args:get_variables_category(category, start_date, end_date, location, conn, use_ids=False):
       category: the category to get the top five disease from
       clinic: the clinic we are looking at
       start_date: the start date for the aggregation
       end_date: the end_date for the aggregation
    Returns: 
       latest_demo(dict): the demographics from the latest record

    """
    variables = variables_instance.get(category)
    keys = sorted(variables.keys())

    result = db.session.query(Data.variables).filter(
        or_(Data.variables.has_key(key) for key in keys),
        Data.clinic == clinic,
        Data.date >= start_date,
        Data.date < end_date
            ).order_by(Data.date.desc()).first()
    ret = {}
    for key in keys:
        name = variables[key]["name"]
        gender, age = name.split(",")[1].strip().split(" ")
        gender = gender.lower()
        age = age.strip()
        gender = gender.strip()
        ret.setdefault(age, {"female": 0, "male": 0})        
        if(result and key in result[0]):
            ret[age][gender] += result[0][key]

    return ret

def refugee_disease(disease_demo):
    """
    From a dict of Variable Name: value where variable name includes
    gender and age information we want to just find the total numbers
    for each diseases.

    Args: 
       disease_demo: dict with disease data that includes demographics in the variable name
    Returns: 
       diseases(dict): diseases with their aggregated value over all demographics
    """
    diseases = {}
    for disease in disease_demo.keys():
        disease_name = disease.split(",")[0].strip()
        diseases.setdefault(disease_name, 0)
        diseases[disease_name] += disease_demo[disease]

    tot_n = sum(diseases.values())
    if tot_n == 0:
        tot_n = 1
    ret = []
    for disease in top(diseases):
        if diseases[disease] > 0:
            ret.append(make_dict(disease,
                                 diseases[disease],
                                 diseases[disease] / tot_n * 100))
    return ret


class NcdReport(Resource):
    """
    Ncd Report to show data on Hypertension and Diabetes. The data includes 
    breakdowns by age and on lab data, complications and comorbidity. We create
    tables with rows for regions. 

    Args:\n
       location: Location to generate report for\n
       start_date: Start date of report\n
       end_date: End date of report\n
    Returns:\n
       report_data\n
    """
    decorators = [require_api_key]
    def get(self, location, start_date=None, end_date=None):
        start_date, end_date = fix_dates(start_date, end_date)
        end_date_limit = end_date + timedelta(days=1)
        ret = {}
        #meta data
        ret["meta"] = {"uuid": str(uuid.uuid4()),
                       "project_id": 1,
                       "generation_timestamp": datetime.now().isoformat(),
                       "schema_version": 0.1
        }
        # Dates and Location Name
        ew = EpiWeek()
        epi_week = ew.get(end_date.isoformat())["epi_week"]
        ret["data"] = {"epi_week_num": epi_week,
                       "end_date": end_date.isoformat(),
                       "project_epoch": datetime(2015, 5, 20).isoformat(),
                       "start_date": start_date.isoformat()
        }
        location_name = db.session.query(Locations.name).filter(
            Locations.id == location).first()
        if not location_name:
            return None
        ret["data"]["project_region"] = location_name.name
        tot_clinics = TotClinics()
        ret["data"]["clinic_num"] = tot_clinics.get(location)["total"]
        # Data on Hypertension and Diabebtes, there are two tables for each disease.
        # One for the age breakdown, and one for labs and complications.
        # For each table we have rows for each Region.
        # Each of these tables have a title key for the X-axis titles
        # Data is list of rows(dicts) with a title for the y-axis title and
        # a list of values that are the values for the row in the right order
        
        ret["hypertension"] = {"age": {}, "complications": {}, "email_summary": {}}
        ret["diabetes"] = {"age": {}, "complications": {}, "email_summary": {}}

        diabetes_id = "ncd_1"
        hypertension_id = "ncd_2"
        diseases = {"hypertension": hypertension_id,
                    "diabetes": diabetes_id}
        ids_to_include = {"hypertension": [("lab_4", "lab_3"), ("lab_5", "lab_3"), ("lab_2", "lab_1"), ("com_1", "tot"), ("smo_2", "smo_4"), ("lab_11", "lab_10")],
                          "diabetes": [("lab_4", "lab_3"), ("lab_5", "lab_3"), ("lab_7", "lab_6"), ("lab_9", "lab_8"), ("com_2", "tot"), ("smo_2", "smo_4"), ("lab_11", "lab_10")]
        }
  
        locations, ldid, regions, districts = all_location_data(db.session)
        v = Variables()
        ages = v.get("ncd_age")
        
        # Loop through diabetes and hypertension
        for disease in diseases.keys():
            # First sort out the titles
            ret[disease]["age"]["titles"] = [gettext("reg")]
            ret[disease]["age"]["data"] = []
            for age in sorted(ages.keys()):
                ret[disease]["age"]["titles"].append(ages[age]["name"])
            ret[disease]["age"]["titles"].insert(1,"Total")
            ret[disease]["complications"]["titles"] = ["reg",
                                                       "tot",
                                                       "gen_1",
                                                       "gen_2"]
  
            for i in ids_to_include[disease]:
                ret[disease]["complications"]["titles"].append(i[0])
            ret[disease]["complications"]["data"] = []


            # Loop through each region, we add [1] to include the whole country
            for i, region in enumerate( [1] + sorted(regions) ):

                d_id = diseases[disease]
                query_variable = QueryVariable()
                # get the age breakdown
                disease_age = query_variable.get(d_id, "ncd_age",
                                                 end_date=end_date_limit.isoformat(),
                                                 start_date=start_date.isoformat(),
                                                 only_loc=region,
                                                 use_ids=True)
                loc_name = locations[region].name
                if region == 1:
                    loc_name = gettext("Total")
                ret[disease]["age"]["data"].append(
                    {"title": loc_name, "values": []}
                )
                
                for age in sorted(ages.keys()):
                    ret[disease]["age"]["data"][i]["values"].append(disease_age[age]["total"])
                ret[disease]["age"]["data"][i]["values"].insert(0,sum( [a["total"] for a in disease_age.values()]))

                #Add whole country summary for email report
                if region == 1:
                  ret[disease]["email_summary"]["cases"]=ret[disease]["age"]["data"][i]["values"][0]

                # Get gender breakdown
                disease_gender = query_variable.get(d_id, "gender",
                                                    end_date=end_date_limit.isoformat(),
                                                    start_date=start_date.isoformat(),
                                                    only_loc=region)

                table_two_total = sum([disease_gender[gender]["total"] for gender in disease_gender])
                ret[disease]["complications"]["data"].append(
                    {
                        "title": loc_name,
                        "values": [table_two_total]
                    })
                if table_two_total == 0:
                    table_two_total = 1
                ret[disease]["complications"]["data"][i]["values"].append([disease_gender["Male"]["total"],  disease_gender["Male"]["total"] /table_two_total * 100])
                ret[disease]["complications"]["data"][i]["values"].append([disease_gender["Female"]["total"],  disease_gender["Female"]["total"] / table_two_total * 100])


                
                # Get the lab breakdown
                for new_id in ids_to_include[disease]:
                    if new_id[0]:
                        numerator = query_ids([d_id, new_id[0]], start_date, end_date_limit, only_loc=region)
                        if new_id[1] == "tot":
                            denominator = table_two_total
                        else:
                            denominator = query_ids([d_id, new_id[1]], start_date, end_date_limit, only_loc=region)
                        if denominator == 0:
                            denominator = 1
                        ret[disease]["complications"]["data"][i]["values"].append(
                            [numerator, numerator/ denominator * 100])

                        #control for email report for the whole country
                        if region == 1:
                          if disease == "diabetes" and new_id[0] == "lab_9":
                            ret[disease]["email_summary"]["control"] = numerator/ denominator * 100
                          elif disease == "hypertension" and new_id[0] == "lab_2":
                            ret[disease]["email_summary"]["control"] = numerator/ denominator * 100
                    else:
                        # We can N/A to the table if it includes data we are not collecting
                        ret[disease]["complications"]["data"][i]["values"].append("N/A")
            
        return ret


class CdReport(Resource):
    """
    Communicable Disease Report

    This report includes data on all disease that generate alerts. For each
    of these diseases we generate a timeline of the number of cases per week. 
    We split these cases into supected and confrimed cases and only show data
    for disease where there at least one case. 

    Args:\n
       location: Location to generate report for\n
       start_date: Start date of report\n
       end_date: End date of report\n
    Returns:\n
       report_data\n
    """
    decorators = [require_api_key]
    
    def get(self, location, start_date = None,end_date=None):
{'fraction_non_alphanumeric': 0.10038889633901033, 'fraction_numerical': 0.007268338473917125, 'mean_word_length': 2.0604941311663794, 'pattern_counts': {'":': 74, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 62, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1534591', 'n_tokens_mistral': 10608, 'n_tokens_neox': 10125, 'n_words': 3010}