<commit_before>#!/usr/bin/env python
"""Parse Ushahidi reports download (csv) and look for discrepancies.

%InsertOptionParserUsage%
"""

# TODO
# Look for locations that are close to each other but with different names
# Calculate consensus location
# List report ids of outliers
# Add support for getting data from the ushahidi web site via api or download form
# testing from ipython: sys.argv = ['parse.py', '-d', '/home/neal/info/ushahidi/libyacrisismap-reports-1301079295.csv']

import os
import sys
import optparse
from optparse import make_option
import logging
import csv
import math
from datetime import datetime

"""
from root import settings
from django.core.management import setup_environ
setup_environ(settings)
from django.db import transaction
import electionaudits.models as models
"""

__author__ = "<NAME> <http://neal.mcburnett.org/>"
__copyright__ = "Copyright (c) 2011 <NAME>"
__license__ = "MIT"


usage = """Usage: parse.py [options] [file]....

Example:
 geoaudit/parse.py tests/ushahidi-report-test.csv"""

option_list = (
    make_option("-d", "--debug",
                  action="store_true", default=False,
                  help="turn on debugging output"),
)

parser = optparse.OptionParser(prog="parse", usage=usage, option_list=option_list)

# incorporate OptionParser usage documentation into our docstring
__doc__ = __doc__.replace("%InsertOptionParserUsage%\n", parser.format_help())

def set_options(args):
    """Return options for parser given specified arguments.
    E.g. options = set_options(["-c", "-s"])
    """

    (options, args) = parser.parse_args(args)
    return options

class Location(object):
    """
    A group of geographic points that have been identified as a single unique location.
    """
    
    def __init__(self):
        """
        Create an empty Location.
        """

        self.ids = []
        self.names = None
        self.lats = []
        self.lons = []

    def merge(self, report):
        """
        Merge in a new report
        """
        self.ids.append(report['#'])
        if self.names:
            if self.names != report['LOCATION']:
                raise ValueError()
        else:
            self.names = report['LOCATION']

        self.lats.append(float(report['LATITUDE']))
        self.lons.append(float(report['LONGITUDE']))

    def median(self):
        """
        Return the point specified by the median of latitudes and median of longitudes.
        Note this is different from the Geometric Median which is harder to calculate:
          http://en.wikipedia.org/wiki/Geometric_median
        """

        
        
    def __str__(self):
        return "L%s: (%f,%f)\t%s" % (self.ids[0], self.lats[0], self.lons[0], self.names[0])

def main(parser):
    """Parse files and do audit"""

    (options, args) = parser.parse_args()

    if options.debug:
        loglevel = logging.DEBUG
    else:
        loglevel = logging.INFO

    logging.basicConfig(level=loglevel) # format='%(message)s'

    logging.debug("options = %s, args = %s" % (options, list(args)))

    if len(args) == 0:
        args.append(os.path.join(os.path.dirname(__file__), '../tests/ushahidi-report-test.csv'))
        logging.debug("using test file: " + args[0])

    reports = parse(args, options)

    logging.debug("%s" % ([r for r in sorted(reports)]))

    locations = merge_by_name(reports)

    analyze(locations)

    return locations<commit_msg>add Location.outliers()<commit_after>#!/usr/bin/env python
"""Parse Ushahidi reports download (csv) and look for discrepancies.

%InsertOptionParserUsage%
"""

# TODO
# Look for locations that are close to each other but with different names
# Calculate consensus location
# List report ids of outliers
# Add support for getting data from the ushahidi web site via api or download form
# testing from ipython: import sys; sys.argv = ['parse.py', '-d', '/home/neal/info/ushahidi/libyacrisismap-reports-1301079295.csv']
# Use real spherical coordinates for distance calculations, etc.

import os
import sys
import optparse
from optparse import make_option
import logging
import csv
import math
from datetime import datetime

"""
from root import settings
from django.core.management import setup_environ
setup_environ(settings)
from django.db import transaction
import electionaudits.models as models
"""

__author__ = "<NAME> <http://neal.mcburnett.org/>"
__copyright__ = "Copyright (c) 2011 <NAME>"
__license__ = "MIT"


usage = """Usage: parse.py [options] [file]....

Example:
 geoaudit/parse.py tests/ushahidi-report-test.csv"""

option_list = (
    make_option("-d", "--debug",
                  action="store_true", default=False,
                  help="turn on debugging output"),
)

parser = optparse.OptionParser(prog="parse", usage=usage, option_list=option_list)

# incorporate OptionParser usage documentation into our docstring
__doc__ = __doc__.replace("%InsertOptionParserUsage%\n", parser.format_help())

def set_options(args):
    """Return options for parser given specified arguments.
    E.g. options = set_options(["-c", "-s"])
    """

    (options, args) = parser.parse_args(args)
    return options

class Location(object):
    """
    A group of geographic points that have been identified as a single unique location.
    """
    
    def __init__(self):
        """
        Create an empty Location.
        """

        self.ids = []
        self.names = None
        self.lats = []
        self.lons = []

    def merge(self, report):
        """
        Merge in a new report
        """
        self.ids.append(report['#'])
        if self.names:
            if self.names != report['LOCATION']:
                raise ValueError()
        else:
            self.names = report['LOCATION']

        self.lats.append(float(report['LATITUDE']))
        self.lons.append(float(report['LONGITUDE']))

    def median(self):
        """
        Return the point specified by the median of latitudes and median of longitudes.
        Note this is different from the Geometric Median which is harder to calculate:
          http://en.wikipedia.org/wiki/Geometric_median
        """

        lats = sorted(self.lats)
        lons = sorted(self.lons)
        num = len(lats)
        # Median is the midpoint if odd, or average of 2 points if even
        if num % 2:
            return(lats[num/2], lons[num/2])
        else:
            return ( (lats[num/2 - 1] + lats[num/2]) / 2.0, (lons[num/2 - 1] + lons[num/2]) / 2.0)
        
    def outliers(self, max_distance = .02):
        """
        return points in this Location that are further than "max_distance" from the median of this location.
        """
        
        median = self.median()

        outs = []

        for n in range(len(self.lats)):
            point = (self.lats[n], self.lons[n])
            if distance(median, point) > max_distance:
                outs.append(point)

        return outs

    def __str__(self):
        return "L%s: (%f,%f)\t%s" % (self.ids[0], self.lats[0], self.lons[0], self.names[0])

def distance(pointa, pointb):
    """
    return the distance between pointa and pointb
    """
    return math.sqrt( (pointa[0]-pointb[0]) ** 2 + (pointa[1]-pointb[1]) ** 2)

def main(parser):
    """Parse files and do audit"""

    (options, args) = parser.parse_args()

    if options.debug:
        loglevel = logging.DEBUG
    else:
        loglevel = logging.INFO

    logging.basicConfig(level=loglevel) # format='%(message)s'

    logging.debug("options = %s, args = %s" % (options, list(args)))

    if len(args) == 0:
        args.append(os.path.join(os.path.dirname(__file__), '../tests/ushahidi-report-test.csv'))
        logging.debug("using test file: " + args[0])

    reports = parse(args, options)

    logging.debug("%s" % ([r for r in sorted(reports)]))

    locations = merge_by_name(reports)

    analyze(locations)

    return locations
{'fraction_non_alphanumeric': 0.12767303555611792, 'fraction_numerical': 0.007845122105529546, 'mean_word_length': 2.8073217726396917, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2923655', 'n_tokens_mistral': 2450, 'n_tokens_neox': 2328, 'n_words': 669}