<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# run_slavelastic.py: Runs a test based off of a slavelastic manifest file.

from __future__ import with_statement
import BaseHTTPServer
import cStringIO
import json  # pylint: disable=F0401
import optparse
import os
import platform
import random
import SimpleHTTPServer
import SocketServer
import socket
import subprocess
import sys
import threading
import time
import urllib
import zipfile


DESCRIPTION = """This script takes a slavelastic manifest file, packages it,
and sends a swarm manifest file to the swarm server.  This is expected to be
called as a build step with the cwd as the parent of the src/ directory.
"""

# A global list of shards, used to hold the stdout buffer for each shard.
# This is declared globally so that the HttpHandler would be able to access it.
g_shards = []

class ThreadedHTTPServer(SocketServer.ThreadingMixIn,
                         BaseHTTPServer.HTTPServer):
  pass

class HttpHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
  """Handler that is called to process incoming POST connections
  from swarm slaves.
  """
  def log_message(self, _format, *args):
    pass

  def do_POST(self):
    global g_shards  # pylint: disable=W0602
    self.send_response(200)
    self.send_header('Content-type', 'text/html')
    self.end_headers()
    data = self.rfile.read()

    # Decode headers
    headers = {}
    for pair in data.split('&'):
      key, value = pair.split('=')
      headers[urllib.unquote(key)] = urllib.unquote_plus(value)

    # Verbose names for headers
    test_case_name = headers['n']
    config_name = headers['c']
    shard_number = int(headers['i'])
    output = headers['r']
    status = headers['s']

    assert shard_number < len(g_shards), 'Invalid shard # %d' % shard_number
    shard = g_shards[shard_number]
    assert test_case_name == shard.test_case_name
    assert config_name == shard.os_image

    if not shard.hostname:
      shard.hostname = self.client_address[0]

    # Take care of 'result_url'
    if 'result' in self.path.split('/'):
      exit_codes = [int(code) if code else 0 for
          code in headers['x'].split(',')]
      code = max(exit_codes)

      # Need to write to shard before calling passed() or failed(), otherwise
      # a race condition is created and the output may not print.
      if shard.verbose:
        data = data.replace('No output!', '')
        shard.write('\nSwarm test completed, swarm results:\n')
        shard.write(output)

      if status == 'True':
        shard.passed()
      else:
        shard.failed(code)
      return

    # Finally, write to stdout
    shard.write(output)

class Manifest(object):
  def __init__(self, filename, switches):
    """Populates a manifest object.
      Args:
        name - Name of the running test.
        files - A list of files to zip up and transfer over.
    """
    platform_mapping =  {
      'win32': 'Windows',
      'cygwin': 'Windows',
      'linux2': 'Linux',
      'darwin': 'Mac'
      }

    # This can cause problems when
    # |current_platform| != |switches_dict['os_image']|
    # crbug.com/117442
    current_platform = platform_mapping[sys.platform]
    switches_dict = {
      'num_shards': switches.num_shards,
      'os_image': current_platform,
    }
    # Parse manifest file
    data = json.load(open(filename))
    self.name = filename
    self.files = data['files']
    self.command = data['command']
    self.relative_cwd = data['relative_cwd']
    self.hashtable_dir = os.path.join('src', 'out', 'Release')

    self.g_shards = switches.num_shards
    # Random name for the output zip file
    self.zipfile_name = 'swarm_tempfile_%s.zip' % ''.join(random.choice(
        'abcdefghijklmnopqrstuvwxyz0123456789') for x in range(10))
    # Port to listen to stdout coming from the swarm slave, will be set later
    # once the HTTP server is initiated and a free port is found
    self.port = None
    self.tasks = []
    self.current_platform = current_platform
    self.target_platform = switches_dict['os_image']
    self.working_dir = switches.working_dir
    self.block_size = switches.block_size
    self.data_directory = 'data'

  def add_task(self, task_name, actions):
    """Appends a new task to the swarm manifest file."""
    self.tasks.append({
          'test_name': task_name,
          'action': actions,
    })

  def zip(self):
    """Zip up all the files in self.files"""
    start_time = time.time()

    zip_file = zipfile.ZipFile(self.zipfile_name, 'w')
    for file_path, info in self.files.iteritems():
      zip_file.write(os.path.join(self.hashtable_dir, info['sha-1']),
                     os.path.join(self.data_directory, file_path))
    zip_file.close()

    print 'Zipping completed, time elapsed: %f' % (time.time() - start_time)

  def cleanup(self):
    os.remove(self.zipfile_name)

  def to_json(self):
    """Export the current configuration into a swarm-readable manifest file"""
    hostname = socket.gethostbyname(socket.gethostname())
    # pylint: disable=E1103
    filepath = os.path.relpath(self.zipfile_name, '../..').replace('\\', '/')

    # The first path is the python location and doesn't need to be adjusted.
    path_adjusted_command = [self.command[0]]
    for path in self.command[1:]:
      if os.path.isabs(path):
        raise Exception('Given an absolute path, unable to convert for swarm '
                        ' bot:\n' + path)
      else:
        adjusted_path = os.path.join(self.relative_cwd, path)
        adjusted_path = os.path.normpath(adjusted_path)
        adjusted_path = os.path.join(self.data_directory, adjusted_path)
        path_adjusted_command.append(adjusted_path)

    # TODO(csharp) file attributes should be set from the manifest file.
    # http://crbug.com/116251
    if self.current_platform == 'Linux' or self.current_platform == 'Mac':
      self.add_task('Change permissions',
                    ['chmod', '+x', '-R', self.data_directory])

    self.add_task('Run Test', path_adjusted_command)

    # Clean up
    if self.current_platform == 'Linux' or self.current_platform == 'Mac':
      cleanup_commands = ['rm', '-rf']
    elif self.current_platform == 'Windows':
      cleanup_commands = ['del']
    self.add_task('Clean Up',
                  cleanup_commands + [self.zipfile_name, self.data_directory])

    # Call kill_processes.py if on windows
    if self.target_platform == 'Windows':
      self.add_task('Kill Processes',
          [sys.executable, '..\\b\\build\\scripts\\slave\\kill_processes.py'])

    # Construct test case
    test_case = {
      'test_case_name': self.name,
      'data': [
        'http://%s/%s' % (hostname, filepath),
      ],
      'tests': self.tasks,
      'env_vars': {
        'GTEST_TOTAL_SHARDS': '%(num_instances)s',
        'GTEST_SHARD_INDEX': '%(instance_index)s',
      },
      'store_result': 'fail',
      'configurations': [
        {
          'min_instances': self.g_shards,
          'max_instances': self.g_shards,
          'config_name': self.target_platform,
          'dimensions': {
            'os': self.target_platform,<commit_msg>Modify Slavelastic to use tree_creator.<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# run_slavelastic.py: Runs a test based off of a slavelastic manifest file.

from __future__ import with_statement
import BaseHTTPServer
import cStringIO
import json  # pylint: disable=F0401
import optparse
import os
import platform
import random
import SimpleHTTPServer
import SocketServer
import socket
import subprocess
import sys
import threading
import time
import urllib
import zipfile


DESCRIPTION = """This script takes a slavelastic manifest file, packages it,
and sends a swarm manifest file to the swarm server.  This is expected to be
called as a build step with the cwd as the parent of the src/ directory.
"""

# A global list of shards, used to hold the stdout buffer for each shard.
# This is declared globally so that the HttpHandler would be able to access it.
g_shards = []

class ThreadedHTTPServer(SocketServer.ThreadingMixIn,
                         BaseHTTPServer.HTTPServer):
  pass

class HttpHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
  """Handler that is called to process incoming POST connections
  from swarm slaves.
  """
  def log_message(self, _format, *args):
    pass

  def do_POST(self):
    global g_shards  # pylint: disable=W0602
    self.send_response(200)
    self.send_header('Content-type', 'text/html')
    self.end_headers()
    data = self.rfile.read()

    # Decode headers
    headers = {}
    for pair in data.split('&'):
      key, value = pair.split('=')
      headers[urllib.unquote(key)] = urllib.unquote_plus(value)

    # Verbose names for headers
    test_case_name = headers['n']
    config_name = headers['c']
    shard_number = int(headers['i'])
    output = headers['r']
    status = headers['s']

    assert shard_number < len(g_shards), 'Invalid shard # %d' % shard_number
    shard = g_shards[shard_number]
    assert test_case_name == shard.test_case_name
    assert config_name == shard.os_image

    if not shard.hostname:
      shard.hostname = self.client_address[0]

    # Take care of 'result_url'
    if 'result' in self.path.split('/'):
      exit_codes = [int(code) if code else 0 for
          code in headers['x'].split(',')]
      code = max(exit_codes)

      # Need to write to shard before calling passed() or failed(), otherwise
      # a race condition is created and the output may not print.
      if shard.verbose:
        data = data.replace('No output!', '')
        shard.write('\nSwarm test completed, swarm results:\n')
        shard.write(output)

      if status == 'True':
        shard.passed()
      else:
        shard.failed(code)
      return

    # Finally, write to stdout
    shard.write(output)

class Manifest(object):
  tree_creator_path = os.path.join('src', 'tools', 'isolate', 'tree_creator.py')

  def __init__(self, filename, switches):
    """Populates a manifest object.
      Args:
        name - Name of the running test.
        files - A list of files to zip up and transfer over.
    """
    platform_mapping =  {
      'win32': 'Windows',
      'cygwin': 'Windows',
      'linux2': 'Linux',
      'darwin': 'Mac'
      }

    # This can cause problems when
    # |current_platform| != |switches_dict['os_image']|
    # crbug.com/117442
    current_platform = platform_mapping[sys.platform]
    switches_dict = {
      'num_shards': switches.num_shards,
      'os_image': current_platform,
    }
    self.name = filename

    self.g_shards = switches.num_shards
    # Random name for the output zip file
    self.zipfile_name = 'swarm_tempfile_%s.zip' % ''.join(random.choice(
        'abcdefghijklmnopqrstuvwxyz0123456789') for x in range(10))
    # Port to listen to stdout coming from the swarm slave, will be set later
    # once the HTTP server is initiated and a free port is found
    self.port = None
    self.tasks = []
    self.current_platform = current_platform
    self.target_platform = switches_dict['os_image']
    self.working_dir = switches.working_dir
    self.block_size = switches.block_size

  def add_task(self, task_name, actions):
    """Appends a new task to the swarm manifest file."""
    self.tasks.append({
          'test_name': task_name,
          'action': actions,
    })

  def zip(self):
    """Zip up all the files in self.files"""
    start_time = time.time()

    zip_file = zipfile.ZipFile(self.zipfile_name, 'w')
    zip_file.write(self.name)
    zip_file.write(self.tree_creator_path)
    zip_file.close()

    print 'Zipping completed, time elapsed: %f' % (time.time() - start_time)

  def cleanup(self):
    os.remove(self.zipfile_name)

  def to_json(self):
    """Export the current configuration into a swarm-readable manifest file"""
    hostname = socket.gethostbyname(socket.gethostname())
    # pylint: disable=E1103
    filepath = os.path.relpath(self.zipfile_name, '../..').replace('\\', '/')

    url = 'http://%s/' % hostname
    self.add_task(
        'Run Test',
        ['python', self.tree_creator_path, '-m', self.name, '-r', url])

    # Clean up
    if self.current_platform == 'Linux' or self.current_platform == 'Mac':
      cleanup_commands = ['rm', '-rf']
    elif self.current_platform == 'Windows':
      cleanup_commands = ['del']
    self.add_task('Clean Up', cleanup_commands + [self.zipfile_name])

    # Call kill_processes.py if on windows
    if self.target_platform == 'Windows':
      self.add_task('Kill Processes',
          [sys.executable, '..\\b\\build\\scripts\\slave\\kill_processes.py'])

    # Construct test case
    test_case = {
      'test_case_name': self.name,
      'data': [
        'http://%s/%s' % (hostname, filepath),
      ],
      'tests': self.tasks,
      'env_vars': {
        'GTEST_TOTAL_SHARDS': '%(num_instances)s',
        'GTEST_SHARD_INDEX': '%(instance_index)s',
      },
      'store_result': 'fail',
      'configurations': [
        {
          'min_instances': self.g_shards,
          'max_instances': self.g_shards,
          'config_name': self.target_platform,
          'dimensions': {
            'os': self.target_platform,
{'fraction_non_alphanumeric': 0.11488037923116806, 'fraction_numerical': 0.006888378638619362, 'mean_word_length': 3.0005925925925925, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '224625', 'n_tokens_mistral': 4328, 'n_tokens_neox': 4175, 'n_words': 1168}