<commit_before>#!/usr/bin/env python
import csv
import requests
import json

CSV_FN = 'waterpoints.csv'
REPORTS_URL = 'http://localhost:5000/reports'


class Waterpoint(object):

    def __init__(self, _id, lat, _long, functional):
        self.id = str(_id)
        self.lat = lat
        self.long = _long
        self.functional = functional

    def __repr__(self):
        return 'Waterpoint(%s, %s, %s, %s)' % (self.id, self.lat, self.long,
                                               self.functional)


def parse_csv(csv_fn):
    waterpoints = []
    _id = 0
    with open(csv_fn, 'r') as _file:
        reader = csv.reader(_file)
        header = reader.next()
        for row in reader:
            d = dict(zip(header, row))
            status = d['STATUS']
            if status not in ['Not functional', 'Functional']:
                continue
            functional = status == 'Functional'
            try:
                _lat, _long = float(d['LATITUDE']), float(d['LONGITUDE'])
            except ValueError:
                continue
            waterpoints.append(Waterpoint(_id, _lat, _long, functional))
            _id += 1
    return waterpoints


def send_report(wp):
    headers = {'content-type': 'application/json'}
    data = {'service_code': 'wp001',
            'data': {'functional': wp.functional,
                     'latitude': wp.lat,
                     'longitude': wp.long,
                     'waterpoint_id': wp.id,
                     }
            }

    requests.post(REPORTS_URL, data=json.dumps(data), headers=headers)


if __name__ == '__main__':
    waterpoints = parse_csv(CSV_FN)
    for w in waterpoints:
        print w
        send_report(w)<commit_msg>Waterpoint upload script creates matching service<commit_after>#!/usr/bin/env python
import csv
import requests
import json

CSV_FN = 'waterpoints.csv'
REPORTS_URL = 'http://localhost:5000/reports'
SERVICES_URL = 'http://localhost:5000/services'


#simple enum from http://stackoverflow.com/questions/36932/how-can-i-represent-an-enum-in-python
def enum(*sequential, **named):
    enums = dict(zip(sequential, range(len(sequential))), **named)
    reverse = dict((value, key) for key, value in enums.iteritems())
    enums['reverse_mapping'] = reverse
    return type('Enum', (), enums)


Status = enum(FUNCTIONAL='functional',
              NOT_FUNCTIONAL='not functional',
              IN_PROGRESS='in progress',
              UNKNOWN='unknown')


class Waterpoint(object):
    #TODO worth it to go through a class?
    def __init__(self, _id, region, lga, ward, village, tech, wptype, status, lat, lon):
        self.waterpoint_id = str(_id)
        self.region = region
        self.lga_name = lga
        self.ward = ward
        self.village = village
        self.technology_in_use = tech
        self.waterpoint = wptype
        self.status = status
        self.latitude = lat
        self.longitude = lon

    def __repr__(self):
        return 'Waterpoint(%s, %s, %s, %s)' % (self.waterpoint_id, self.latitude, self.longitude,
                                               self.status)


def resolve_status(status):
    if status.lower() == "functional":
        return Status.FUNCTIONAL
    elif status.lower() == "not functional":
        return Status.NOT_FUNCTIONAL
    elif status.lower() == "in progress":
        return Status.IN_PROGRESS
    else:
        return Status.UNKNOWN


def parse_csv(csv_fn):
    waterpoints = []
    with open(csv_fn, 'r') as _file:
        reader = csv.reader(_file)
        header = reader.next()
        for _id, row in enumerate(reader):
            d = dict(zip(header, row))

            status = resolve_status(d['STATUS'])

            try:
                lat, lon = float(d['LATITUDE']), float(d['LONGITUDE'])
            except ValueError:
                #TODO log error
                continue

            wp = Waterpoint(_id, *(row[:-3] + [status, lat, lon]))
            waterpoints.append(wp)

    return waterpoints


def create_wp_service():
    headers = {'content-type': 'application/json'}
    data = {'name': 'WaterpointService',
            'fields': {'waterpoint_id': {'type': 'StringField', 'required': True},
                       'region': {'type': 'StringField', 'required': True},
                       'lga_name': {'type': 'StringField', 'required': True},
                       'ward': {'type': 'StringField', 'required': True},
                       'village': {'type': 'StringField', 'required': True},
                       'technology_in_use': {'type': 'StringField', 'required': True},
                       'waterpoint': {'type': 'StringField', 'required': True},
                       'status': {'type': 'StringField', 'required': True},
                       'latitude': {'type': 'FloatField', 'required': True},
                       'longitude': {'type': 'FloatField', 'required': True},
                       },
            'group': 'location based reports',
            'keywords': ['waterpoints'],
            'protocol_type': '',
            'service_name': '',
            'service_code': 'wp1'
            }
    requests.post(SERVICES_URL, data=json.dumps(data), headers=headers)


def send_report(wp):
    headers = {'content-type': 'application/json'}
    data = {'service_code': 'wp1', 'data': wp.__dict__}

    requests.post(REPORTS_URL, data=json.dumps(data), headers=headers)


if __name__ == '__main__':
    create_wp_service()
    waterpoints = parse_csv(CSV_FN)
    for w in waterpoints:
        print w
        send_report(w)
{'fraction_non_alphanumeric': 0.1344976593446165, 'fraction_numerical': 0.004501260352898812, 'mean_word_length': 2.143746462931522, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3484899', 'n_tokens_mistral': 1669, 'n_tokens_neox': 1590, 'n_words': 385}