<commit_before>#!/usr/bin/python
from datetime import datetime
from elasticsearch import Elasticsearch
import pprint
import time

pp = pprint.PrettyPrinter(indent=4)
es = Elasticsearch()

res = es.search(size=50, index="*", body={"query": {"match_all": {}}})
print("\"_time\",\"_raw\"")
for hit in res['hits']['hits']:
   hit["_source"]["message"] = hit["_source"]["message"].replace('"',' ');
   epochTimestamp = hit['_source']['@timestamp'];
   date_time = '2014-12-21T16:11:18.419Z'
   pattern = '%Y-%m-%dT%H:%M:%S.%fZ'
   hit['_source']['_epoch'] = int(time.mktime(time.strptime(epochTimestamp, pattern)))
   print("%(_epoch)s,\"%(message)s\" " % hit["_source"] )<commit_msg>added cli arg index = lets you specify the index to search = default=* query = does not work yet<commit_after>#!/usr/bin/python
from datetime import datetime
from elasticsearch import Elasticsearch
import pprint
import time
import sys
 
total = len(sys.argv)
cmdargs = str(sys.argv)
for i in xrange(total):
   opt = str (sys.argv[i]);
   if (  opt.startswith("index") ):
      index= str (sys.argv[i]).split("=")[1];
   elif (  opt.startswith("query") ):
      query= str (sys.argv[i]).split("=")[1];
   
pp = pprint.PrettyPrinter(indent=4)
es = Elasticsearch()
res = es.search(size=50, index=index, body={"query": {"match_all": {}} })
print("\"_time\",\"_raw\",\"index\",\"type\"")
#pp.pprint(res);
for hit in res['hits']['hits']:
   hit["_source"]["message"] = hit["_source"]["message"].replace('"',' ');
   epochTimestamp = hit['_source']['@timestamp'];
   date_time = '2014-12-21T16:11:18.419Z'
   pattern = '%Y-%m-%dT%H:%M:%S.%fZ'
   hit['_source']['_epoch'] = int(time.mktime(time.strptime(epochTimestamp, pattern)))
   print("%(_epoch)s,\"%(message)s\"," % hit["_source"] + 
         "\"%(_index)s\",\"%(_type)s\"" % hit
         )
{'fraction_non_alphanumeric': 0.23280132085855806, 'fraction_numerical': 0.023115024766097964, 'mean_word_length': 5.0, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '108831', 'n_tokens_mistral': 680, 'n_tokens_neox': 668, 'n_words': 122}