<commit_before>"""
Created on Tue Sep 23 17:15:47 2014

@author: Natural Solutions (Thomas)
"""

from pyramid.view import view_config
from sqlalchemy import desc, select
from ecorelevesensor.models import DBSession, DataGsm
from ecorelevesensor.utils.distance import haversine

import pandas as pd
import numpy as np

prefix = 'dataGsm/'

@view_config(route_name=prefix + 'unchecked', renderer='json')
def data_gsm_unchecked(request):
    '''Get the unchecked GSM data.
    '''
    gsm = int(request.matchdict['id'])
    
    if request.GET['format'] == 'geojson':
        # Query
        query = select([
            DataGsm.id.label('id'),
            DataGsm.lat,
            DataGsm.lon,
            DataGsm.date_
        ]).where(DataGsm.gsm == gsm).where(DataGsm.checked == False).order_by(desc(DataGsm.date_))
        # Create list of features from query result
        features = [
            {
                'type':'Feature',
                'properties':{'date':str(date)},
                'geometry':{'type':'Point', 'coordinates':[float(lon),float(lat)]},
                'id':id_
            }
        for id_, lat, lon, date in DBSession.execute(query).fetchall()]
        result = {'type':'FeatureCollection', 'features':features}
        return result
        
    elif request.GET['format'] == 'json':
        # Query
        query = select([
            DataGsm.id.label('id'),
            DataGsm.lat,
            DataGsm.lon,
            DataGsm.date_
        ]).where(DataGsm.gsm == gsm).where(DataGsm.checked == False).order_by(desc(DataGsm.date_))
        data = DBSession.execute(query).fetchall()
        # Load data from the DB then
        # compute the distance between 2 consecutive points.
        df = pd.DataFrame.from_records(data, columns=data[0].keys(), coerce_float=True)
        X1 = df.ix[:,['lat', 'lon']].values[:-1,:]
        X2 = df.ix[1:,['lat', 'lon']].values
        dist = pd.Series(np.append(haversine(X1, X2).round(3), 0), name='dist')
        impo = pd.Series([False]*len(data), name='import')
        df = pd.concat([df['id'], df['date_'].apply(lambda x: str(x)), df['lat'], df['lon'], dist, impo], axis=1)
        return df.to_dict('records')<commit_msg>View data_gsm: add import auto check.<commit_after>"""
Created on Tue Sep 23 17:15:47 2014

@author: Natural Solutions (Thomas)
"""

from pyramid.view import view_config
from sqlalchemy import desc, select
from ecorelevesensor.models import DBSession, DataGsm
from ecorelevesensor.utils.distance import haversine

import pandas as pd
import numpy as np

prefix = 'dataGsm/'

@view_config(route_name=prefix + 'unchecked', renderer='json')
def data_gsm_unchecked(request):
    '''Get the unchecked GSM data.
    '''
    gsm = int(request.matchdict['id'])
    
    if request.GET['format'] == 'geojson':
        # Query
        query = select([
            DataGsm.id.label('id'),
            DataGsm.lat,
            DataGsm.lon,
            DataGsm.date_
        ]).where(DataGsm.gsm == gsm).where(DataGsm.checked == False).order_by(desc(DataGsm.date_))
        # Create list of features from query result
        features = [
            {
                'type':'Feature',
                'properties':{'date':str(date)},
                'geometry':{'type':'Point', 'coordinates':[float(lon),float(lat)]},
                'id':id_
            }
        for id_, lat, lon, date in DBSession.execute(query).fetchall()]
        result = {'type':'FeatureCollection', 'features':features}
        return result
        
    elif request.GET['format'] == 'json':
        # Query
        query = select([
            DataGsm.id.label('id'),
            DataGsm.lat,
            DataGsm.lon,
            DataGsm.date_
        ]).where(DataGsm.gsm == gsm).where(DataGsm.checked == False).order_by(desc(DataGsm.date_))
        data = DBSession.execute(query).fetchall()
        # Load data from the DB then
        # compute the distance between 2 consecutive points.
        df = pd.DataFrame.from_records(data, columns=data[0].keys(), coerce_float=True)
        X1 = df.ix[:,['lat', 'lon']].values[:-1,:]
        X2 = df.ix[1:,['lat', 'lon']].values
        df['dist'] = np.append(haversine(X1, X2).round(3), 0)
        df['import'] = [False]*len(df.index)
        ids = df.set_index('date_').resample('1H', how='first').dropna().id
        df['import'].ix[df.id[ids]] = True
        return df.to_dict('records')
{'fraction_non_alphanumeric': 0.15409018808067074, 'fraction_numerical': 0.010423748017221844, 'mean_word_length': 2.5312, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3201999', 'n_tokens_mistral': 1432, 'n_tokens_neox': 1383, 'n_words': 293}