<commit_before># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import datetime
import glob
import heapq
import logging
import os
import os.path
import random
import re
import shutil
import subprocess as subprocess
import sys
import tempfile
import time

import py_utils
from py_utils import cloud_storage  # pylint: disable=import-error
import dependency_manager  # pylint: disable=import-error

from telemetry.internal.util import binary_manager
from telemetry.core import exceptions
from telemetry.internal.backends.chrome import chrome_browser_backend
from telemetry.internal.util import path


DEVTOOLS_ACTIVE_PORT_FILE = 'DevToolsActivePort'


def ParseCrashpadDateTime(date_time_str):
  # Python strptime does not support time zone parsing, strip it.
  date_time_parts = date_time_str.split()
  if len(date_time_parts) >= 3:
    date_time_str = ' '.join(date_time_parts[:2])
  return datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S')


def GetSymbolBinaries(minidump, arch_name, os_name):
  # Returns binary file where symbols are located.
  minidump_dump = binary_manager.FetchPath('minidump_dump', arch_name, os_name)
  assert minidump_dump

  symbol_binaries = []

  minidump_cmd = [minidump_dump, minidump]
  try:
    with open(os.devnull, 'wb') as dev_null:
      minidump_output = subprocess.check_output(minidump_cmd, stderr=dev_null)
  except subprocess.CalledProcessError as e:
    # For some reason minidump_dump always fails despite successful dumping.
    minidump_output = e.output

  minidump_binary_re = re.compile(r'\W+\(code_file\)\W+=\W\"(.*)\"')
  for minidump_line in minidump_output.splitlines():
    line_match = minidump_binary_re.match(minidump_line)
    if line_match:
      binary_path = line_match.group(1)
      if not os.path.isfile(binary_path):
        continue

      # Filter out system binaries.
      if (binary_path.startswith('/usr/lib/') or
          binary_path.startswith('/System/Library/') or
          binary_path.startswith('/lib/')):
        continue

      # Filter out other binary file types which have no symbols.
      if (binary_path.endswith('.pak') or
          binary_path.endswith('.bin') or
          binary_path.endswith('.dat') or
          binary_path.endswith('.ttf')):
        continue

      symbol_binaries.append(binary_path)
  return symbol_binaries


def GenerateBreakpadSymbols(minidump, arch, os_name, symbols_dir, browser_dir):
  logging.info('Dumping breakpad symbols.')
  generate_breakpad_symbols_command = binary_manager.FetchPath(
      'generate_breakpad_symbols', arch, os_name)
  if not generate_breakpad_symbols_command:
    return

  for binary_path in GetSymbolBinaries(minidump, arch, os_name):
    cmd = [
        sys.executable,
        generate_breakpad_symbols_command,
        '--binary=%s' % binary_path,
        '--symbols-dir=%s' % symbols_dir,
        '--build-dir=%s' % browser_dir,
        ]

    try:
      subprocess.check_call(cmd, stderr=open(os.devnull, 'w'))
    except subprocess.CalledProcessError:
      logging.warning('Failed to execute "%s"', ' '.join(cmd))
      return


class DesktopBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
  """The backend for controlling a locally-executed browser instance, on Linux,
  Mac or Windows.
  """
  def __init__(self, desktop_platform_backend, browser_options,
               browser_directory, profile_directory,
               executable, flash_path, is_content_shell):
    super(DesktopBrowserBackend, self).__init__(
        desktop_platform_backend,
        browser_options=browser_options,
        browser_directory=browser_directory,
        profile_directory=profile_directory,
        supports_extensions=not is_content_shell,<commit_msg>Don't suppress stderr when calling generate_breakpad_symbols.py<commit_after># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import datetime
import glob
import heapq
import logging
import os
import os.path
import random
import re
import shutil
import subprocess as subprocess
import sys
import tempfile
import time

import py_utils
from py_utils import cloud_storage  # pylint: disable=import-error
import dependency_manager  # pylint: disable=import-error

from telemetry.internal.util import binary_manager
from telemetry.core import exceptions
from telemetry.internal.backends.chrome import chrome_browser_backend
from telemetry.internal.util import path


DEVTOOLS_ACTIVE_PORT_FILE = 'DevToolsActivePort'


def ParseCrashpadDateTime(date_time_str):
  # Python strptime does not support time zone parsing, strip it.
  date_time_parts = date_time_str.split()
  if len(date_time_parts) >= 3:
    date_time_str = ' '.join(date_time_parts[:2])
  return datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S')


def GetSymbolBinaries(minidump, arch_name, os_name):
  # Returns binary file where symbols are located.
  minidump_dump = binary_manager.FetchPath('minidump_dump', arch_name, os_name)
  assert minidump_dump

  symbol_binaries = []

  minidump_cmd = [minidump_dump, minidump]
  try:
    with open(os.devnull, 'wb') as dev_null:
      minidump_output = subprocess.check_output(minidump_cmd, stderr=dev_null)
  except subprocess.CalledProcessError as e:
    # For some reason minidump_dump always fails despite successful dumping.
    minidump_output = e.output

  minidump_binary_re = re.compile(r'\W+\(code_file\)\W+=\W\"(.*)\"')
  for minidump_line in minidump_output.splitlines():
    line_match = minidump_binary_re.match(minidump_line)
    if line_match:
      binary_path = line_match.group(1)
      if not os.path.isfile(binary_path):
        continue

      # Filter out system binaries.
      if (binary_path.startswith('/usr/lib/') or
          binary_path.startswith('/System/Library/') or
          binary_path.startswith('/lib/')):
        continue

      # Filter out other binary file types which have no symbols.
      if (binary_path.endswith('.pak') or
          binary_path.endswith('.bin') or
          binary_path.endswith('.dat') or
          binary_path.endswith('.ttf')):
        continue

      symbol_binaries.append(binary_path)
  return symbol_binaries


def GenerateBreakpadSymbols(minidump, arch, os_name, symbols_dir, browser_dir):
  logging.info('Dumping breakpad symbols.')
  generate_breakpad_symbols_command = binary_manager.FetchPath(
      'generate_breakpad_symbols', arch, os_name)
  if not generate_breakpad_symbols_command:
    return

  for binary_path in GetSymbolBinaries(minidump, arch, os_name):
    cmd = [
        sys.executable,
        generate_breakpad_symbols_command,
        '--binary=%s' % binary_path,
        '--symbols-dir=%s' % symbols_dir,
        '--build-dir=%s' % browser_dir,
        ]

    try:
      subprocess.check_call(cmd)
    except subprocess.CalledProcessError:
      logging.warning('Failed to execute "%s"', ' '.join(cmd))
      return


class DesktopBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
  """The backend for controlling a locally-executed browser instance, on Linux,
  Mac or Windows.
  """
  def __init__(self, desktop_platform_backend, browser_options,
               browser_directory, profile_directory,
               executable, flash_path, is_content_shell):
    super(DesktopBrowserBackend, self).__init__(
        desktop_platform_backend,
        browser_options=browser_options,
        browser_directory=browser_directory,
        profile_directory=profile_directory,
        supports_extensions=not is_content_shell,
{'fraction_non_alphanumeric': 0.08755520914523253, 'fraction_numerical': 0.0018186541958950377, 'mean_word_length': 3.9735142118863047, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7152185', 'n_tokens_mistral': 2384, 'n_tokens_neox': 2330, 'n_words': 562}