<commit_before>import sys
from elasticsearch import Elasticsearch, exceptions

source_query = ''

for logstash_line in sys.stdin:
    source_query = str(logstash_line).strip()
    #with open("/var/test.txt", "w") as text_file:
    #    text_file.write("Pipe Output was: {}".format(source_query))
    break

# update all documents matching the source and running state
query = {
     "script": {
        "inline": "ctx._source.spider_state='hefna_debug'",
        "lang": "painless"
     },
     "query": {
        "bool": {
          "must": [
            {
              "match" : {
                "spider_state": "test_debug777"
              }
            },
            {
              "match" : {
                "source": "{}".format(source_query)
              }
            }<commit_msg>replace dummy spider_state values with real ones<commit_after>import sys
from elasticsearch import Elasticsearch, exceptions

source_query = ''

for logstash_line in sys.stdin:
    source_query = str(logstash_line).strip()
    #with open("/var/test.txt", "w") as text_file:
    #    text_file.write("Pipe Output was: {}".format(source_query))
    break

# update all documents matching the source and running state
query = {
     "script": {
        "inline": "ctx._source.spider_state='debug'",
        "lang": "painless"
     },
     "query": {
        "bool": {
          "must": [
            {
              "match" : {
                "spider_state": "running"
              }
            },
            {
              "match" : {
                "source": "{}".format(source_query)
              }
            }
{'fraction_non_alphanumeric': 0.13126934984520125, 'fraction_numerical': 0.0018575851393188853, 'mean_word_length': 1.7910189982728844, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1680653', 'n_tokens_mistral': 481, 'n_tokens_neox': 474, 'n_words': 102}