from __future__ import division, print_function
import datetime
from collections import *

def describe( data ):

    import types

    if isinstance( data, types.GeneratorType ):
        data = list( data )

    from timeline import module_timeline

    if len(data) == 0:
        print( "Dataset empty." )
        return

    print( "Entries together", len(data) )
    print( "Number of different authors", len( set( map( lambda d: d['creator'], filter( lambda d: d['creator'] is not '', data ) ) ) ) )

    ## remove dates which can not be true
    date_ok = filter( lambda d: d['timestamp'] is not '', data )
    date_ok = filter( lambda d: d['timestamp'] > datetime.datetime(1970,1,1,23,59), date_ok )

    print( "First post", min( map( lambda d: d['timestamp'], date_ok ) ) )
    print( "Last post", max( map( lambda d: d['timestamp'], date_ok  ) ) )

    print("Data sources")

    ## todo: reimplement?
    counter = defaultdict( int )

    for post in data:
        counter[ post['source_detail'] ] += 1

    for name, count in counter.items():
        print( '-', name, count )

    return module_timeline.create_timeline( datasets = [date_ok] )

if __name__ == '__main__':

    for function_name in dir( data_loader ):

        if 'load_' in function_name:

            print( function_name )
            f =  getattr( data_loader, function_name )
            data = f()
            describe( data )

from __future__ import division, print_function
import datetime
from collections import *

import pprint
pp = pprint.PrettyPrinter(indent=1)

def describe( data ):

    import types

    if isinstance( data, types.GeneratorType ):
        data = list( data )

    from timeline import module_timeline

    if len(data) == 0:
        print( "Dataset empty." )
        return

    print( "Entries together", len(data) )
    print( "Number of different authors", len( set( map( lambda d: d['creator'], filter( lambda d: d['creator'] is not '', data ) ) ) ) )

    ## remove dates which can not be true
    date_ok = filter( lambda d: d['timestamp'] is not '', data )
    date_ok = filter( lambda d: d['timestamp'] > datetime.datetime(1970,1,1,23,59), date_ok )

    print( "First post", min( map( lambda d: d['timestamp'], date_ok ) ) )
    print( "Last post", max( map( lambda d: d['timestamp'], date_ok  ) ) )

    print("Data sources")

    ## todo: reimplement?
    counter = defaultdict( int )

    for post in data:
        counter[ post['source_detail'] ] += 1

    for name, count in counter.items():
        print( '-', name, count )

    print( 'Data structure' )
    pp.pprint( keys( data ) )

    return module_timeline.create_timeline( datasets = [date_ok] )

def keys( data ):

    res = []

    ## fake a bit, as different data entries can have different keys
    for entry in data[0:50]:
        res.append( _keys( entry ) )

    ## search for the longest string representation as it most likely has most information

    return sorted( res, key = lambda x : len(str(x)), reverse = True)[0]

def _keys( entry ):

    if isinstance( entry, list ):
        if len( entry ) > 0:
            return [ _keys( entry[0] ) ]
        return []

    if isinstance( entry, dict ):
        out = {}

        for k, v in entry.items():
            out[str(k)] = _keys( v )

        return out

    if isinstance( entry, str) or isinstance( entry, unicode):
        return 'text'

    if isinstance( entry, float) or isinstance( entry, int):
        return 'number'

    if isinstance( entry, datetime.datetime ):
        return 'time'

    return str( type( entry ) )

Print the keys and data types when data described. Fixes #19.