<commit_before># -*- python -*-
# ex: set filetype=python:
#
# Copyright 2018 The Kythe Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from os import path, listdir
import json
import urllib.request

from buildbot.changes.github import GitHubPullrequestPoller
from buildbot.plugins import *
from twisted.internet import defer

c = BuildmasterConfig = {}
c['buildbotNetUsageData'] = None

# https://github.com/buildbot/buildbot/issues/3627
# db_url and a few other places don't support Interpolate, so do it manually.
secrets_dir = path.join(path.dirname(__file__), "../secrets")
secrets_dict = {p: open(path.join(secrets_dir, p)).read().rstrip("\n")
                for p in listdir(secrets_dir)}

c['secretsProviders'] = [secrets.SecretInAFile(dirname=secrets_dir)]

####### WORKERS

# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.
c['workers'] = [
    worker.Worker("local-worker", secrets_dict["worker_password"]),
]

# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c['protocols'] = {'pb': {'port': 9989}}

####### CHANGESOURCES

def pullRequestFilter(pr):
  # Blacklist anything that explicitly says 'nobuildbot'
  for label in pr['labels']:
    if label['name'] == 'nobuildbot':
      return False

  # Allow all Kythe maintainer Pull Requests
  url = 'https://api.github.com/teams/2988296/memberships/'+pr['user']['login']
  req = urllib.request.Request(
      url, headers={
          'User-Agent': 'Buildbot',
          'Authorization': 'token {github_token}'.format(**secrets_dict)
      })
  try:
    with urllib.request.urlopen(req) as f:
      userMembership = json.loads(f.read().decode('utf-8'))
      if userMembership['state'] == 'active' and userMembership['role'] == 'maintainer':
        return True
  except:
    # User is not a member of the Kythe team
    pass

  # Allow explicitly labeled PRs by non-maintainers if they signed the CLA
  signed_cla = False
  requested = False
  for label in pr['labels']:
    if label['name'] == 'cla: yes':
      signed_cla = True
    elif label['name'] == 'buildbot':
      requested = True
  return signed_cla and requested

c['change_source'] = []

def add_github_change_sources(repo, owner='kythe'):
  # Track master
  c['change_source'].append(changes.GitPoller(
      'git://github.com/%s/%s.git' % (owner, repo),
      project='%s/%s' % (owner, repo),
      workdir='%s-workdir' % repo, branches=['master'],
      category='master', pollAtLaunch=True, pollInterval=300))
  # Track pull requests
  c['change_source'].append(GitHubPullrequestPoller(
      owner=owner, repo=repo, repository_type='git',
      pullrequest_filter=pullRequestFilter,
      category='pull', magic_link=True,
      pollAtLaunch=True, pollInterval=60,
      token=secrets_dict['github_token']))

add_github_change_sources('kythe')
add_github_change_sources('lang-proto')

####### SCHEDULERS

bazelMinVersion = '0.22.0'
bazelMaxVersion = '0.24.0'

bazelBuilders = [
    'bazel-'+bazelMaxVersion,
]
if bazelMinVersion != bazelMaxVersion:
    bazelBuilders += ['bazel-'+bazelMinVersion]

goBuilders = [
    'go-1.11-gopath',
    'go-1.11-module'
]

releaseBuilders = [
    'bazel-release',
]

allBuilders = bazelBuilders + goBuilders + releaseBuilders

c['schedulers'] = []
c['schedulers'].append(schedulers.SingleBranchScheduler(
    name="bazel-master",
    change_filter=util.ChangeFilter(
        category='master',
        branch='master',
        project=['kythe/kythe', 'kythe/lang-proto'],
    ),
    builderNames=bazelBuilders))
c['schedulers'].append(schedulers.SingleBranchScheduler(
    name="bazel-pull-requests",
    change_filter=util.ChangeFilter(
        category='pull',
        project=['kythe/kythe', 'kythe/lang-proto'],
    ),<commit_msg>build: remove kythe/lang-proto repo from buildbot (#3659)<commit_after># -*- python -*-
# ex: set filetype=python:
#
# Copyright 2018 The Kythe Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from os import path, listdir
import json
import urllib.request

from buildbot.changes.github import GitHubPullrequestPoller
from buildbot.plugins import *
from twisted.internet import defer

c = BuildmasterConfig = {}
c['buildbotNetUsageData'] = None

# https://github.com/buildbot/buildbot/issues/3627
# db_url and a few other places don't support Interpolate, so do it manually.
secrets_dir = path.join(path.dirname(__file__), "../secrets")
secrets_dict = {p: open(path.join(secrets_dir, p)).read().rstrip("\n")
                for p in listdir(secrets_dir)}

c['secretsProviders'] = [secrets.SecretInAFile(dirname=secrets_dir)]

####### WORKERS

# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.
c['workers'] = [
    worker.Worker("local-worker", secrets_dict["worker_password"]),
]

# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c['protocols'] = {'pb': {'port': 9989}}

####### CHANGESOURCES

def pullRequestFilter(pr):
  # Blacklist anything that explicitly says 'nobuildbot'
  for label in pr['labels']:
    if label['name'] == 'nobuildbot':
      return False

  # Allow all Kythe maintainer Pull Requests
  url = 'https://api.github.com/teams/2988296/memberships/'+pr['user']['login']
  req = urllib.request.Request(
      url, headers={
          'User-Agent': 'Buildbot',
          'Authorization': 'token {github_token}'.format(**secrets_dict)
      })
  try:
    with urllib.request.urlopen(req) as f:
      userMembership = json.loads(f.read().decode('utf-8'))
      if userMembership['state'] == 'active' and userMembership['role'] == 'maintainer':
        return True
  except:
    # User is not a member of the Kythe team
    pass

  # Allow explicitly labeled PRs by non-maintainers if they signed the CLA
  signed_cla = False
  requested = False
  for label in pr['labels']:
    if label['name'] == 'cla: yes':
      signed_cla = True
    elif label['name'] == 'buildbot':
      requested = True
  return signed_cla and requested

c['change_source'] = []

def add_github_change_sources(repo, owner='kythe'):
  # Track master
  c['change_source'].append(changes.GitPoller(
      'git://github.com/%s/%s.git' % (owner, repo),
      project='%s/%s' % (owner, repo),
      workdir='%s-workdir' % repo, branches=['master'],
      category='master', pollAtLaunch=True, pollInterval=300))
  # Track pull requests
  c['change_source'].append(GitHubPullrequestPoller(
      owner=owner, repo=repo, repository_type='git',
      pullrequest_filter=pullRequestFilter,
      category='pull', magic_link=True,
      pollAtLaunch=True, pollInterval=60,
      token=secrets_dict['github_token']))

add_github_change_sources('kythe')

####### SCHEDULERS

bazelMinVersion = '0.22.0'
bazelMaxVersion = '0.24.0'

bazelBuilders = [
    'bazel-'+bazelMaxVersion,
]
if bazelMinVersion != bazelMaxVersion:
    bazelBuilders += ['bazel-'+bazelMinVersion]

goBuilders = [
    'go-1.11-gopath',
    'go-1.11-module'
]

releaseBuilders = [
    'bazel-release',
]

allBuilders = bazelBuilders + goBuilders + releaseBuilders

c['schedulers'] = []
c['schedulers'].append(schedulers.SingleBranchScheduler(
    name="bazel-master",
    change_filter=util.ChangeFilter(
        category='master',
        branch='master',
        project=['kythe/kythe'],
    ),
    builderNames=bazelBuilders))
c['schedulers'].append(schedulers.SingleBranchScheduler(
    name="bazel-pull-requests",
    change_filter=util.ChangeFilter(
        category='pull',
        project=['kythe/kythe'],
    ),
{'fraction_non_alphanumeric': 0.1286518069681887, 'fraction_numerical': 0.009738151915169876, 'mean_word_length': 4.137854363535298, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1685764', 'n_tokens_mistral': 2958, 'n_tokens_neox': 2828, 'n_words': 787}