<commit_before>#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Module that polls the skia-telemetry AppEngine WebApp."""


import json
import os
import subprocess
import sys
import tempfile
import time
import urllib

import appengine_constants


SLEEP_BETWEEN_POLLS_SECS = 60

ENCOUNTERED_KEYS = {}


class Poller(object):

  def Poll(self):

    while True:

      # Get all tasks in the queue
      # TODO(rmistry): Make urls to open a tuple??
      get_tasks_page = urllib.urlopen(
          appengine_constants.SKIA_TELEMETRY_WEBAPP +
          appengine_constants.GET_LUA_TASKS_SUBPATH)
      pending_tasks = json.loads(
          get_tasks_page.read().replace('\r\n', '\\r\\n'))
      for key in sorted(pending_tasks.keys()):
        task = pending_tasks[key]
        task_key = task['key']
        if task_key in ENCOUNTERED_KEYS:
          '%s is already being processed' % task_key
          continue
        ENCOUNTERED_KEYS[task_key] = 1
        # Create a run id.
        run_id = '%s-%s' % (task['username'].split('@')[0], int(time.time()))
        lua_file = os.path.join(tempfile.gettempdir(), '%s.lua' % run_id)
        f = open(lua_file, 'w')
        f.write(task['lua_script'])
        f.close()
        # Now call the script!!!
        log_file = os.path.join(tempfile.gettempdir(), '%s.output' % run_id)
        cmd = 'bash vm_run_lua_on_slaves.sh %s %s %s %s' % (<commit_msg>Guard against parallel GCE runs.<commit_after>#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Module that polls the skia-telemetry AppEngine WebApp."""


import json
import os
import subprocess
import sys
import tempfile
import time
import urllib

import appengine_constants


SLEEP_BETWEEN_POLLS_SECS = 60

ENCOUNTERED_KEYS = {}


class Poller(object):

  def Poll(self):

    while True:

      # Get all tasks in the queue
      # TODO(rmistry): Make urls to open a tuple??
      get_tasks_page = urllib.urlopen(
          appengine_constants.SKIA_TELEMETRY_WEBAPP +
          appengine_constants.GET_LUA_TASKS_SUBPATH)
      pending_tasks = json.loads(
          get_tasks_page.read().replace('\r\n', '\\r\\n'))
      for key in sorted(pending_tasks.keys()):
        task = pending_tasks[key]
        task_key = task['key']
        if task_key in ENCOUNTERED_KEYS:
          '%s is already being processed' % task_key
          continue
        ENCOUNTERED_KEYS[task_key] = 1
        # Create a run id.
        run_id = '%s-%s' % (task['username'].split('@')[0], time.time())
        lua_file = os.path.join(tempfile.gettempdir(), '%s.lua' % run_id)
        f = open(lua_file, 'w')
        f.write(task['lua_script'])
        f.close()
        # Now call the script!!!
        log_file = os.path.join(tempfile.gettempdir(), '%s.output' % run_id)
        cmd = 'bash vm_run_lua_on_slaves.sh %s %s %s %s' % (
{'fraction_non_alphanumeric': 0.11603102779573368, 'fraction_numerical': 0.005171299288946348, 'mean_word_length': 2.8304455445544554, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '832508', 'n_tokens_mistral': 1072, 'n_tokens_neox': 1044, 'n_words': 276}