<commit_before># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from datetime import datetime
import json

import command_processor
import http_server
import logging
import spectator_client


def millis_to_time(millis):
  """Convert milliseconds to a time string."""
  return datetime.fromtimestamp(millis / 1000).isoformat('T') + 'Z'


def strip_non_html_params(options):
  """Return a copy of options with only those that are query parameters.

  This is to propagate options in web response URLs.
  """
  params = {}
  for key in ['tagNameRegex', 'tagValueRegex', 'metricNameRegex']:
    if key in options:
      params[key] = options[key]
  return params


class BaseSpectatorCommandHandler(command_processor.CommandHandler):
  def add_argparser(self, subparsers):
    parser = super(BaseSpectatorCommandHandler, self).add_argparser(subparsers)
    parser.add_argument('--by', default='service',
                        help='Organize by "service" or by "metric" name.')
    spectator_client.SpectatorClient.add_standard_parser_arguments(parser)
    return parser

  def _get_data_map(self, catalog, options):
    restrict_services = options.get('services', None)
    if restrict_services:
      catalog = {service: config
                 for service, config in catalog.items()
                 if service in restrict_services.split(',')}
    spectator = spectator_client.SpectatorClient(options)

    by = options.get('by', 'service')
    if by == 'service':
      data_map = spectator.scan_by_service(catalog, params=options)
    else:
      data_map = spectator.scan_by_type(catalog, params=options)
    return data_map


class DumpMetricsHandler(BaseSpectatorCommandHandler):
  def process_commandline_request(self, options):
    catalog = spectator_client.get_source_catalog(options)
    data_map = self._get_data_map(catalog, options)
    json_text = json.JSONEncoder(indent=2).encode(data_map)
    self.output(options, json_text)

  def process_web_request(self, request, path, params, fragment):
    options = dict(command_processor.get_global_options())
    options.update(params)
    catalog = spectator_client.get_source_catalog(options)
    param_services = params.get('services', 'all').split(',')
    if param_services == ['all']:
      restricted_catalog = catalog
    else:
      restricted_catalog = {key: value
                            for key, value in catalog.items()
                            if key in param_services}
    data_map = self._get_data_map(restricted_catalog, options)
    body = json.JSONEncoder(indent=2).encode(data_map)
    request.respond(200, {'ContentType': 'application/json'}, body)


class ExploreCustomDescriptorsHandler(BaseSpectatorCommandHandler):
  """Show all the current descriptors in use, and who is using them."""

  def __init__(self, *pos_args, **kwargs):
    self.__service_map = None
    self.__raw_spectator = None
    self.__filtering_spectator = None
    self.__transforming_spectator = None
    super(ExploreCustomDescriptorsHandler, self).__init__(*pos_args, **kwargs)

  def __get_xform_info(self, type_map):
    """Derive transformation information from a type_map.

    This is to take the original raw results and produce the transformed
    results so that we can depict what the original was and what the final is.
 
    Returns:
      3-tuple:
         dict mapping source names to transformed names,
         type_map using transformation
         service_tag_map using transformation
    """
    response_processor = self.__transforming_spectator.response_processor
    xform_name_map = {}
    xform_type_map = {}
    for meter_name, service_map in type_map.items():
      for service, meter_responses in service_map.items():
        transformer = response_processor.determine_service_metric_transformer(
            service)
        if not transformer:
          continue
        for meter_response in meter_responses:
          xform_response = transformer.process_response(
              {meter_name: meter_response})
          if not xform_response:
            continue
          for xform_name, response_part in xform_response.items():
            if meter_name in xform_name_map:
              xform_name_map[meter_name].append(xform_name)
            else:
              xform_name_map[meter_name] = [xform_name]
            if not xform_name in xform_type_map:
              xform_type_map[xform_name] = {}
            xform_type_map[xform_name].update({service: [response_part]})

    return xform_name_map, xform_type_map, self.to_service_tag_map(xform_type_map)[0]

  def __get_type_and_tag_map_and_active_services(self, catalog, options):
    self.__transforming_spectator = spectator_client.SpectatorClient(options)

    filtering_options = dict(options)
    filtering_options['disable_metric_filter'] = True
    self.__filtering_spectator = spectator_client.SpectatorClient(
        filtering_options)

    scan_options = dict(filtering_options)
    scan_options['disable_metric_transform'] = True
    raw_spectator = spectator_client.SpectatorClient(scan_options)

    self.__raw_spectator = raw_spectator
    self.__service_map = raw_spectator.scan_by_service(<commit_msg>fix(daemon): Dont replicate transformations in explorer<commit_after># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from datetime import datetime
import json

import command_processor
import http_server
import logging
import spectator_client


def millis_to_time(millis):
  """Convert milliseconds to a time string."""
  return datetime.fromtimestamp(millis / 1000).isoformat('T') + 'Z'


def strip_non_html_params(options):
  """Return a copy of options with only those that are query parameters.

  This is to propagate options in web response URLs.
  """
  params = {}
  for key in ['tagNameRegex', 'tagValueRegex', 'metricNameRegex']:
    if key in options:
      params[key] = options[key]
  return params


class BaseSpectatorCommandHandler(command_processor.CommandHandler):
  def add_argparser(self, subparsers):
    parser = super(BaseSpectatorCommandHandler, self).add_argparser(subparsers)
    parser.add_argument('--by', default='service',
                        help='Organize by "service" or by "metric" name.')
    spectator_client.SpectatorClient.add_standard_parser_arguments(parser)
    return parser

  def _get_data_map(self, catalog, options):
    restrict_services = options.get('services', None)
    if restrict_services:
      catalog = {service: config
                 for service, config in catalog.items()
                 if service in restrict_services.split(',')}
    spectator = spectator_client.SpectatorClient(options)

    by = options.get('by', 'service')
    if by == 'service':
      data_map = spectator.scan_by_service(catalog, params=options)
    else:
      data_map = spectator.scan_by_type(catalog, params=options)
    return data_map


class DumpMetricsHandler(BaseSpectatorCommandHandler):
  def process_commandline_request(self, options):
    catalog = spectator_client.get_source_catalog(options)
    data_map = self._get_data_map(catalog, options)
    json_text = json.JSONEncoder(indent=2).encode(data_map)
    self.output(options, json_text)

  def process_web_request(self, request, path, params, fragment):
    options = dict(command_processor.get_global_options())
    options.update(params)
    catalog = spectator_client.get_source_catalog(options)
    param_services = params.get('services', 'all').split(',')
    if param_services == ['all']:
      restricted_catalog = catalog
    else:
      restricted_catalog = {key: value
                            for key, value in catalog.items()
                            if key in param_services}
    data_map = self._get_data_map(restricted_catalog, options)
    body = json.JSONEncoder(indent=2).encode(data_map)
    request.respond(200, {'ContentType': 'application/json'}, body)


class ExploreCustomDescriptorsHandler(BaseSpectatorCommandHandler):
  """Show all the current descriptors in use, and who is using them."""

  def __init__(self, *pos_args, **kwargs):
    self.__service_map = None
    self.__raw_spectator = None
    self.__filtering_spectator = None
    self.__transforming_spectator = None
    super(ExploreCustomDescriptorsHandler, self).__init__(*pos_args, **kwargs)

  def __get_xform_info(self, type_map):
    """Derive transformation information from a type_map.

    This is to take the original raw results and produce the transformed
    results so that we can depict what the original was and what the final is.
 
    Returns:
      3-tuple:
         dict mapping source names to transformed names,
         type_map using transformation
         service_tag_map using transformation
    """
    response_processor = self.__transforming_spectator.response_processor
    xform_name_map = {}
    xform_type_map = {}
    for meter_name, service_map in type_map.items():
      for service, meter_responses in service_map.items():
        transformer = response_processor.determine_service_metric_transformer(
            service)
        if not transformer:
          continue
        for meter_response in meter_responses:
          xform_response = transformer.process_response(
              {meter_name: meter_response})
          if not xform_response:
            continue
          for xform_name, response_part in xform_response.items():
            if meter_name in xform_name_map:
              have_names = xform_name_map[meter_name]
              if xform_name not in have_names:
                have_names.append(xform_name)
            else:
              xform_name_map[meter_name] = [xform_name]
            if not xform_name in xform_type_map:
              xform_type_map[xform_name] = {}
            xform_type_map[xform_name].update({service: [response_part]})

    return xform_name_map, xform_type_map, self.to_service_tag_map(xform_type_map)[0]

  def __get_type_and_tag_map_and_active_services(self, catalog, options):
    self.__transforming_spectator = spectator_client.SpectatorClient(options)

    filtering_options = dict(options)
    filtering_options['disable_metric_filter'] = True
    self.__filtering_spectator = spectator_client.SpectatorClient(
        filtering_options)

    scan_options = dict(filtering_options)
    scan_options['disable_metric_transform'] = True
    raw_spectator = spectator_client.SpectatorClient(scan_options)

    self.__raw_spectator = raw_spectator
    self.__service_map = raw_spectator.scan_by_service(
{'fraction_non_alphanumeric': 0.08083340537736665, 'fraction_numerical': 0.003285207919080142, 'mean_word_length': 3.6986190089358244, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4045883', 'n_tokens_mistral': 3312, 'n_tokens_neox': 3204, 'n_words': 859}