<commit_before>import sys, os
sys.path.append('/Users/johndingee_seed/code/polio')
os.environ['DJANGO_SETTINGS_MODULE'] = 'polio.settings'
from django.conf import settings

from django.core.exceptions import ObjectDoesNotExist
from datapoints.models import Indicator, DataPoint, Region, Campaign
from odk_source.models import VCMSummaryNew

from dateutil import parser



import pprint as pp
import pandas as pd

def vcm_summary():

    inds = Indicator.objects.all()

    to_process = pd.DataFrame(list(VCMSummaryNew.objects.all().values())) # where processed = 0
    source_columns = VCMSummaryNew._meta.get_all_field_names()

    # map columns to indicators #
    column_to_indicator_map = {}
    for col in source_columns:
        if col in [ind.name for ind in inds]:
            column_to_indicator_map[col] = Indicator.objects.get(name=col).id

    # map rows to region / campaigns {<row_id>:(<region_id>,<campaign_id>)}  #
    row_to_region_campaign_map = {}

    meta_columns = ['id','Date_Implement','SettlementCode',]
    indicator_columns = [col for col,ind_id in column_to_indicator_map.iteritems()]
    slice_columns = meta_columns + indicator_columns

    sliced_df = to_process[slice_columns]

    for row in sliced_df.values:
        new_dp = proces_row(row,sliced_df.columns.tolist())


def proces_row(row,column_names):

    try:
        region_id = Region.objects.get(full_name=row[column_names.index \
            ('SettlementCode')]).id ## evaluate this by settlement code not name

    except ObjectDoesNotExist:
        return None

    try:
        the_date = parser.parse(row[column_names.index('Date_Implement')])
        campaign_id = Campaign.objects.get(start_date=the_date).id
        print campaign_id
    except ObjectDoesNotExist:
        return None







if __name__ == "__main__":
    vcm_summary()<commit_msg>insert datapoints from the source table<commit_after>import sys, os
sys.path.append('/Users/johndingee_seed/code/polio')
os.environ['DJANGO_SETTINGS_MODULE'] = 'polio.settings'
from django.conf import settings

from django.core.exceptions import ObjectDoesNotExist
from datapoints.models import Indicator, DataPoint, Region, Campaign
from odk_source.models import VCMSummaryNew

from dateutil import parser



import pprint as pp
import pandas as pd

def vcm_summary():

    inds = Indicator.objects.all()

    to_process = pd.DataFrame(list(VCMSummaryNew.objects.all().values())) # where processed = 0
    source_columns = VCMSummaryNew._meta.get_all_field_names()

    # map columns to indicators #
    column_to_indicator_map = {}
    for col in source_columns:
        if col in [ind.name for ind in inds]:
            column_to_indicator_map[col] = Indicator.objects.get(name=col).id

    # map rows to region / campaigns {<row_id>:(<region_id>,<campaign_id>)}  #
    row_to_region_campaign_map = {}

    meta_columns = ['id','Date_Implement','SettlementCode',]
    indicator_columns = [col for col,ind_id in column_to_indicator_map.iteritems()]
    slice_columns = meta_columns + indicator_columns

    sliced_df = to_process[slice_columns]
    column_list = sliced_df.columns.tolist()

    for row in sliced_df.values:
        new_dp = proces_row(row,column_list,column_to_indicator_map)


def proces_row(row,column_names,column_to_indicator_map):

    try:
        region_id = Region.objects.get(full_name=row[column_names.index \
            ('SettlementCode')]).id ## evaluate this by settlement code not name

    except ObjectDoesNotExist:
        return None

    try:
        the_date = parser.parse(row[column_names.index('Date_Implement')])
        campaign_id = Campaign.objects.get(start_date=the_date).id
        print campaign_id
    except ObjectDoesNotExist:
        return None

    for i, value in enumerate(row):
        try:
            indicator_id =  column_to_indicator_map[column_names[i]]
            datapoint_value = row[i]
            dp = DataPoint.objects.create(
                indicator_id = indicator_id, \
                region_id = region_id, \
                campaign_id = campaign_id, \
                value =  datapoint_value, \
                changed_by_id = 1  # FIX THIS! User should be "ODK ETL"
            )
        except KeyError:
            pass # means its one of the meta data columns









if __name__ == "__main__":
    vcm_summary()
{'fraction_non_alphanumeric': 0.09201748332183114, 'fraction_numerical': 0.0006901311249137336, 'mean_word_length': 3.3654618473895583, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '64204', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1354, 'n_words': 296}