<commit_before>
from elasticsearch import Elasticsearch
import argparse
import codecs



def scanandscroll(index, doctype, query, hostname="localhost", port=9200, username = None, password = None):

    #query = {"query" : {"match_all" : {}}}
    #query = {"query": {"filtered": {"query": {"match_phrase": {"url": "http://ieeexplore.ieee.org"}},"filter": {"and": {"filters": [{ "term": {"team": "hyperion-gray"}}]}}}}}

    # Initialize the scroll

    if username and password:
        es = Elasticsearch(['https://' + username + ':' + password + '@' + hostname + ":" + str(port)],show_ssl_warnings=False)
    else:
        es = Elasticsearch(['https://' + hostname + ":" + str(port)],show_ssl_warnings=False)

    page = es.search(index = index,doc_type = doctype,scroll = '20m',search_type = 'scan',size = 5,body = query)
    sid = page['_scroll_id']
    scroll_size = page['hits']['total']
    #print "Total hits:" + str(scroll_size)

    with codecs.open("els_results.json", "w", "utf-8") as f:
        with codecs.open("scroll_id.txt","w","utf-8") as sf:
            # Start scrolling
            while scroll_size > 0:
                try:
                    #print "Scrolling..."
                    page = es.scroll(scroll_id=sid, scroll='20m')
                    # Update the scroll ID
                    sid = page['_scroll_id']
                    sf.write(str(sid) + '\n')
                    # Get the number of results that we returned in the last scroll
                    scroll_size = len(page['hits']['hits'])
                    #print page['hits']['hits']

                    for i in range(len(page['hits']['hits'])):
                        #print page['hits']['hits'][i]
                        f.write(str(page['hits']['hits'][i]) + '\n')
                except Exception as e:
                    pass
       
            #print "scroll size: " + str(scroll_size)
            # Do something with the obtained page
    f.close()
    sf.close()




if __name__ == '__main__':

    argp = argparse.ArgumentParser()
    argp.add_argument("-hostname",help="Elastic Search Server hostname, defaults to 'localhost'",default="localhost")
    argp.add_argument("-port",type=int, help="Elastic Search Server port,defaults to 9200",default=9200)
    argp.add_argument("-username", help = "username for Elasticsearch cluster", default = None)
    argp.add_argument("-password", help = "password for Elasticsearch cluster", default=None)
    argp.add_argument("indexname",help="desired name of the index in ElasticSearch")
    argp.add_argument("doctype",help="type of the document to be indexed")
    argp.add_argument("query",help="query to the elasticsearch")
    arguments = argp.parse_args()

    if arguments.hostname and arguments.port:
        scanandscroll(arguments.indexname, arguments.doctype, arguments.query, arguments.hostname, arguments.port, arguments.username, arguments.password)
    else:
        scanandscroll(arguments.indexname, arguments.doctype, arguments.query, arguments.username, arguments.password)

    #print "Thats all folks!"<commit_msg>output as seq file<commit_after>
from elasticsearch import Elasticsearch
import argparse
import codecs



def scanandscroll(index, doctype, query, hostname="localhost", port=9200, username = None, password = None):

    #query = {"query" : {"match_all" : {}}}
    #query = {"query": {"filtered": {"query": {"match_phrase": {"url": "http://ieeexplore.ieee.org"}},"filter": {"and": {"filters": [{ "term": {"team": "hyperion-gray"}}]}}}}}

    # Initialize the scroll

    if username and password:
        es = Elasticsearch(['https://' + username + ':' + password + '@' + hostname + ":" + str(port)],show_ssl_warnings=False)
    else:
        es = Elasticsearch(['https://' + hostname + ":" + str(port)],show_ssl_warnings=False)

    page = es.search(index = index,doc_type = doctype,scroll = '20m',search_type = 'scan',size = 5,body = query)
    sid = page['_scroll_id']
    scroll_size = page['hits']['total']
    #print "Total hits:" + str(scroll_size)

    with codecs.open("els_results.json", "w", "utf-8") as f:
        with codecs.open("scroll_id.txt","w","utf-8") as sf:
            # Start scrolling
            while scroll_size > 0:
                try:
                    #print "Scrolling..."
                    page = es.scroll(scroll_id=sid, scroll='20m')
                    # Update the scroll ID
                    sid = page['_scroll_id']
                    sf.write(str(sid) + '\n')
                    # Get the number of results that we returned in the last scroll
                    scroll_size = len(page['hits']['hits'])
                    #print page['hits']['hits']

                    for i in range(len(page['hits']['hits'])):
                        #print page['hits']['hits'][i]
                        f.write(page['hits']['hits'][i]['_source']['url'] + "\t" + str(page['hits']['hits'][i]) + '\n')
                except Exception as e:
                    pass

            #print "scroll size: " + str(scroll_size)
            # Do something with the obtained page
    f.close()
    sf.close()




if __name__ == '__main__':

    argp = argparse.ArgumentParser()
    argp.add_argument("-hostname",help="Elastic Search Server hostname, defaults to 'localhost'",default="localhost")
    argp.add_argument("-port",type=int, help="Elastic Search Server port,defaults to 9200",default=9200)
    argp.add_argument("-username", help = "username for Elasticsearch cluster", default = None)
    argp.add_argument("-password", help = "password for Elasticsearch cluster", default=None)
    argp.add_argument("indexname",help="desired name of the index in ElasticSearch")
    argp.add_argument("doctype",help="type of the document to be indexed")
    argp.add_argument("query",help="query to the elasticsearch")
    arguments = argp.parse_args()

    if arguments.hostname and arguments.port:
        scanandscroll(arguments.indexname, arguments.doctype, arguments.query, arguments.hostname, arguments.port, arguments.username, arguments.password)
    else:
        scanandscroll(arguments.indexname, arguments.doctype, arguments.query, arguments.username, arguments.password)

    #print "Thats all folks!"
{'fraction_non_alphanumeric': 0.15116465863453815, 'fraction_numerical': 0.00642570281124498, 'mean_word_length': 2.8574969021065675, 'pattern_counts': {'":': 24, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4929116', 'n_tokens_mistral': 1861, 'n_tokens_neox': 1792, 'n_words': 430}