<commit_before>#!/usr/bin/env python

import sys
import json
import logging

import urllib3
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
from elasticsearch.helpers import streaming_bulk
import boto
from boto.s3.key import Key
from pymongo import MongoClient
import happybase
from pykafka import KafkaClient
# from addict import Dict

# disable annoying SSL certificate warnings
urllib3.disable_warnings()

# set up a logger
logging.basicConfig(level=logging.INFO)


class JsonPort(object):
    """ Parses out a JSON iterator object."""

    def __init__(self, jsonlist):
        self.jsonlist = jsonlist

    def parse(self):
        """ Returns an JSON iterator object if input is valid JSON, else
            it returns an empty dictionary.
        """
        for idx, i in enumerate(self.jsonlist):
            try:
                yield json.loads(i)
            except ValueError as ve:
                logging.warning('line {0}:  {1}'.format(idx, ve))
                logging.debug('line {0}:  {1}'.format(idx, i))

    def inspect(self):
        """ Output the serialized JSON object one line at a time.  To
            continue, press any key.  To end, Ctrl+d.
        """
        for i in self.parse():
            print json.dumps(i, indent=2)
            try:
                raw_input('\n--Press any key to continue--\n')
            except EOFError:
                sys.exit(0)


class ElasticPort(object):
    """ Class to handle Elastic Search actions. """

    def __str__(self):
        return 'ElasticPort'

    def __init__(self, host, ssl, logger=None):
        self.es = Elasticsearch(host, set_ssl=ssl)

        self.logger = logging.getLogger(__name__)
        # ch = logging.StreamHandler()
        # ch.setLevel(logging.INFO)
        # self.logger.addHandler(ch)
        # self.logger.setLevel(logging.INFO)

    def query(self):
        """ Query Elastic Search """
        pass

    def index(self, jsonit, iname, dtype):
        """ Data input is a JSON generator.  If using the command-line tool,
            this is handled via the JsonPort method which creates a
            JSON generator from lines read in from files.
        """

        # Create a generator of JSON objects for bulk indexing
        def bulkgen(jsongen):
            for idx, jobj in enumerate(jsongen):
                bulkr = dict()
                bulkr['_index'] = iname
                bulkr['_type'] = dtype
                bulkr['_source'] = jobj
                if 'id' in jobj:
                    bulkr['_id'] = jobj['id']
                self.logger.debug('done with index %s' % idx)
                yield bulkr


        r = bulk(client=self.es, actions=bulkgen(jsonit), stats_only=True)
        self.logger.info('INDEX: successful: %s; failed: %s' % (r[0], r[1]))

    def map(self, iname, dtype, mapping):<commit_msg>Print indexing statistics as part of user interface<commit_after>#!/usr/bin/env python

import sys
import json
import logging

import urllib3
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
from elasticsearch.helpers import streaming_bulk
import boto
from boto.s3.key import Key
from pymongo import MongoClient
import happybase
from pykafka import KafkaClient
# from addict import Dict

# disable annoying SSL certificate warnings
urllib3.disable_warnings()

# set up a logger
logging.basicConfig(level=logging.INFO)


class JsonPort(object):
    """ Parses out a JSON iterator object."""

    def __init__(self, jsonlist):
        self.jsonlist = jsonlist

    def parse(self):
        """ Returns an JSON iterator object if input is valid JSON, else
            it returns an empty dictionary.
        """
        for idx, i in enumerate(self.jsonlist):
            try:
                yield json.loads(i)
            except ValueError as ve:
                logging.warning('line {0}:  {1}'.format(idx, ve))
                logging.debug('line {0}:  {1}'.format(idx, i))

    def inspect(self):
        """ Output the serialized JSON object one line at a time.  To
            continue, press any key.  To end, Ctrl+d.
        """
        for i in self.parse():
            print json.dumps(i, indent=2)
            try:
                raw_input('\n--Press any key to continue--\n')
            except EOFError:
                sys.exit(0)


class ElasticPort(object):
    """ Class to handle Elastic Search actions. """

    def __str__(self):
        return 'ElasticPort'

    def __init__(self, host, ssl, logger=None):
        self.es = Elasticsearch(host, set_ssl=ssl)

        self.logger = logging.getLogger(__name__)
        # ch = logging.StreamHandler()
        # ch.setLevel(logging.INFO)
        # self.logger.addHandler(ch)
        # self.logger.setLevel(logging.INFO)

    def query(self):
        """ Query Elastic Search """
        pass

    def index(self, jsonit, iname, dtype):
        """ Data input is a JSON generator.  If using the command-line tool,
            this is handled via the JsonPort method which creates a
            JSON generator from lines read in from files.
        """

        # Create a generator of JSON objects for bulk indexing
        def bulkgen(jsongen):
            for idx, jobj in enumerate(jsongen):
                bulkr = dict()
                bulkr['_index'] = iname
                bulkr['_type'] = dtype
                bulkr['_source'] = jobj
                if 'id' in jobj:
                    bulkr['_id'] = jobj['id']
                self.logger.debug('done with index %s' % idx)
                yield bulkr


        r = bulk(client=self.es, actions=bulkgen(jsonit), stats_only=True)
        print 'INDEX: successful: %s; failed: %s' % (r[0], r[1])

    def map(self, iname, dtype, mapping):
{'fraction_non_alphanumeric': 0.09958362248438585, 'fraction_numerical': 0.003816793893129771, 'mean_word_length': 2.1416893732970026, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6969514', 'n_tokens_mistral': 1657, 'n_tokens_neox': 1601, 'n_words': 527}