<commit_before>"""
Load data from json file to ElasticSearch
Read an array of json objects, indexing each json object
"""

__author__ = 'saggu'

import json
from elasticsearch import Elasticsearch
from sys import stderr
import sys
import argparse

def loadDatainES(filename, index, doctype,dataFileType,hostname="localhost",port=9200):
    try:
        print "Connecting to " + hostname + " at port:" + str(port) 
        es = Elasticsearch([{'host': hostname, 'port': port}])
        
        if dataFileType=="1":
            with open(filename) as f:
                d = json.load(f)
                for wp in d:
                  res = es.index(index=index,doc_type=doctype,body=wp)
                  print "indexing id: " + res["_id"] + " for uri: " + wp["uri"]
        elif dataFileType == "0":
            with open(filename) as f:
                lines = f.readlines()

                for line in lines:
                    if line.strip() != "":
                        jsonurlobj = json.loads(line.strip())
                        objkey = jsonurlobj['uri']
                        res = es.index(index=index,doc_type=doctype,body=line)
                        print "indexing id: " + res["_id"] + " for uri: " + objkey
                 
    except Exception, e:
        print >> stderr.write('ERROR: %s\n' % str(e))
        



if __name__ == '__main__':
    
    argp = argparse.ArgumentParser()
    argp.add_argument("-hostname",help="Elastic Search Server hostname, defaults to 'localhost'",default="localhost")
    argp.add_argument("-port",type=int, help="Elastic Search Server port,defaults to 9200",default=9200)
    argp.add_argument("filepath",help="json file to be loaded in ElasticSearch")
    argp.add_argument("indexname",help="desired name of the index in ElasticSearch")
    argp.add_argument("doctype",help="type of the document to be indexed")
    argp.add_argument("dataFileType", help="Specify '0' if every line in the data file is different json object  or '1' otherwise")
    arguments = argp.parse_args()
    
    if arguments.hostname and arguments.port:
        loadDatainES(arguments.filepath, arguments.indexname, arguments.doctype,arguments.dataFileType,arguments.hostname, arguments.port)
    else:
        loadDatainES(arguments.filepath, arguments.indexname, arguments.doctype,arguments.dataFileType)
    
    print "Thats all folks!"
<commit_msg>specify mapping file at the command line<commit_after>"""
Load data from json file to ElasticSearch
Read an array of json objects, indexing each json object
"""

__author__ = 'saggu'

import json
from elasticsearch import Elasticsearch
from sys import stderr
import sys
import argparse

def loadDatainES(filename, index, doctype,dataFileType,hostname="localhost",port=9200,mappingFilePath=None):
    try:
        print "Connecting to " + hostname + " at port:" + str(port) 
        es = Elasticsearch([{'host': hostname, 'port': port}])
        
        if mappingFilePath:
            with open(mappingFilePath) as m:
                mapping = m.read()
                #print "Mapping file:" + mapping
                es.indices.create(index=index,  body=mapping,ignore=400)
                
        if dataFileType=="1":
            with open(filename) as f:   
                d = json.load(f)
                for wp in d:
                  res = es.index(index=index,doc_type=doctype,body=wp)
                  print "indexing id: " + res["_id"] + " for uri: " + wp["uri"]
        elif dataFileType == "0":
            with open(filename) as f:
                lines = f.readlines()

                for line in lines:
                    if line.strip() != "":
                        jsonurlobj = json.loads(line.strip())
                        objkey = jsonurlobj['uri']
                        res = es.index(index=index,doc_type=doctype,body=line)
                        print "indexing id: " + res["_id"] + " for uri: " + objkey
                 
    except Exception, e:
        print >> stderr.write('ERROR: %s\n' % str(e))
        



if __name__ == '__main__':
    
    argp = argparse.ArgumentParser()
    argp.add_argument("-hostname",help="Elastic Search Server hostname, defaults to 'localhost'",default="localhost")
    argp.add_argument("-port",type=int, help="Elastic Search Server port,defaults to 9200",default=9200)
    argp.add_argument("-mappingFilePath", help="mapping/setting file for the index")
    argp.add_argument("filepath",help="json file to be loaded in ElasticSearch")
    argp.add_argument("indexname",help="desired name of the index in ElasticSearch")
    argp.add_argument("doctype",help="type of the document to be indexed")
    argp.add_argument("dataFileType", help="Specify '0' if every line in the data file is different json object  or '1' otherwise")
    arguments = argp.parse_args()
    
    if arguments.hostname and arguments.port:
        loadDatainES(arguments.filepath, arguments.indexname, arguments.doctype,arguments.dataFileType,arguments.hostname, arguments.port,arguments.mappingFilePath)
    else:
        loadDatainES(arguments.filepath, arguments.indexname, arguments.doctype,arguments.dataFileType,mappingFilePath)
    
    print "Thats all folks!"

{'fraction_non_alphanumeric': 0.11224293676724967, 'fraction_numerical': 0.0067268883336536614, 'mean_word_length': 2.616400277970813, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4931264', 'n_tokens_mistral': 1525, 'n_tokens_neox': 1466, 'n_words': 390}