<commit_before># Copyright 2016 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.

from __future__ import print_function

import argparse
import json
import logging
import os
import shutil
import subprocess
import sys

from .internal import simple_cfg
from .internal.autoroll_impl.candidate_algorithm import get_roll_candidates

from google.protobuf import json_format as jsonpb


LOGGER = logging.getLogger(__name__)

ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')

IS_WIN = sys.platform.startswith(('win', 'cygwin'))
VPYTHON = 'vpython' + ('.bat' if IS_WIN else '')
GIT = 'git' + ('.bat' if IS_WIN else '')

def write_global_files_to_main_repo(recipe_deps, spec):
  """Writes the recipes.cfg and recipes.py scripts to the main repo on disk.

  This pulls `recipes.py` from the current 'recipe_engine' dep in recipe_deps.

  Args:
    * recipe_deps (RecipeDeps) - The loaded recipe dependencies; the destination
      repo is `recipe_deps.main_repo`.
    * spec (proto message RepoSpec) - The RepoSpec proto to write to
      recipes.cfg.
  """
  main_repo = recipe_deps.main_repo
  if spec.project_id:
    spec.repo_name = spec.project_id
  out = jsonpb.MessageToJson(spec, preserving_proto_field_name=True)
  LOGGER.info('writing: %s', out)

  cfg_path = os.path.join(main_repo.path, simple_cfg.RECIPES_CFG_LOCATION_REL)
  with open(cfg_path, 'wb') as f:
    f.write(out)

  engine = recipe_deps.repos['recipe_engine']
  recipes_py_path = os.path.join(main_repo.recipes_root_path, 'recipes.py')
  with open(recipes_py_path, 'wb') as f:
    f.write(
      engine.backend.cat_file(
        spec.deps['recipe_engine'].revision, 'recipes.py'))


def run_simulation_test(repo, *additional_args):
  """Runs the recipe simulation test for given repo.

  Returns a tuple of exit code and output.
  """
  p = subprocess.Popen([
    VPYTHON, os.path.join(repo.recipes_root_path, 'recipes.py'), 'test',
  ] + list(additional_args), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  output, _ = p.communicate()
  rc = p.returncode
  return rc, output
<commit_msg>[autoroll] Make recipes.cfg deterministically generated.<commit_after># Copyright 2016 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.

from __future__ import print_function

import argparse
import json
import logging
import os
import shutil
import subprocess
import sys

from .internal import simple_cfg
from .internal.autoroll_impl.candidate_algorithm import get_roll_candidates

from google.protobuf import json_format as jsonpb


LOGGER = logging.getLogger(__name__)

ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')

IS_WIN = sys.platform.startswith(('win', 'cygwin'))
VPYTHON = 'vpython' + ('.bat' if IS_WIN else '')
GIT = 'git' + ('.bat' if IS_WIN else '')

def write_global_files_to_main_repo(recipe_deps, spec):
  """Writes the recipes.cfg and recipes.py scripts to the main repo on disk.

  This pulls `recipes.py` from the current 'recipe_engine' dep in recipe_deps.

  Args:
    * recipe_deps (RecipeDeps) - The loaded recipe dependencies; the destination
      repo is `recipe_deps.main_repo`.
    * spec (proto message RepoSpec) - The RepoSpec proto to write to
      recipes.cfg.
  """
  main_repo = recipe_deps.main_repo
  if spec.project_id:
    spec.repo_name = spec.project_id
  # Format recipes.cfg nicely and make it deterministic.
  out = json.dumps(
      jsonpb.MessageToDict(spec, preserving_proto_field_name=True),
      indent=2, sort_keys=True).replace(' \n', '\n') + '\n'
  LOGGER.info('writing: %s', out)

  cfg_path = os.path.join(main_repo.path, simple_cfg.RECIPES_CFG_LOCATION_REL)
  with open(cfg_path, 'wb') as f:
    f.write(out)

  engine = recipe_deps.repos['recipe_engine']
  recipes_py_path = os.path.join(main_repo.recipes_root_path, 'recipes.py')
  with open(recipes_py_path, 'wb') as f:
    f.write(
      engine.backend.cat_file(
        spec.deps['recipe_engine'].revision, 'recipes.py'))


def run_simulation_test(repo, *additional_args):
  """Runs the recipe simulation test for given repo.

  Returns a tuple of exit code and output.
  """
  p = subprocess.Popen([
    VPYTHON, os.path.join(repo.recipes_root_path, 'recipes.py'), 'test',
  ] + list(additional_args), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  output, _ = p.communicate()
  rc = p.returncode
  return rc, output

{'fraction_non_alphanumeric': 0.10884353741496598, 'fraction_numerical': 0.0028527540048277375, 'mean_word_length': 4.36866902237927, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5077703', 'n_tokens_mistral': 1578, 'n_tokens_neox': 1546, 'n_words': 415}