<commit_before>#!/usr/bin/env python3
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Receive push events for new builds and upload rows to BigQuery."""



import argparse
import json
import os
import pprint
import socket
import sys
import traceback
import time

import multiprocessing.pool
import ruamel.yaml as yaml

try:
    from google.api_core import exceptions as api_exceptions
    from google.cloud import bigquery
    from google.cloud import pubsub_v1
    import google.cloud.exceptions
except ImportError:
    print('WARNING: unable to load google cloud (test environment?)')
    traceback.print_exc()

import model
import make_db
import make_json


MAX_ROW_UPLOAD = 10 # See https://github.com/googleapis/google-cloud-go/issues/2855


def should_exclude(objectID, bucketID, buckets):
   # Objects of form a/b/c/<jobname>/<hash>/<objectFile>'
    job_name = objectID.rsplit('/')[-3] # See tests for refernce
    return job_name in buckets[bucketID].get('exclude_jobs', [])


def process_changes(results, buckets):
    """Split GCS change events into trivial ack_ids and builds to further process."""
    ack_ids = []  # pubsub rec_message ids to acknowledge
    todo = []  # (id, job, build) of builds to grab
    # process results, find finished builds to process
    for rec_message in results:
        eventType = rec_message.message.attributes['eventType']
        objectID = rec_message.message.attributes['objectId']<commit_msg>Apply suggestions from code review<commit_after>#!/usr/bin/env python3
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Receive push events for new builds and upload rows to BigQuery."""



import argparse
import json
import os
import pprint
import socket
import sys
import traceback
import time

import multiprocessing.pool
import ruamel.yaml as yaml

try:
    from google.api_core import exceptions as api_exceptions
    from google.cloud import bigquery
    from google.cloud import pubsub_v1
    import google.cloud.exceptions
except ImportError:
    print('WARNING: unable to load google cloud (test environment?)')
    traceback.print_exc()

import model
import make_db
import make_json


MAX_ROW_UPLOAD = 10 # See https://github.com/googleapis/google-cloud-go/issues/2855


def should_exclude(objectID, bucketID, buckets):
   # Objects of form a/b/c/<jobname>/<hash>/<objectFile>'
    job_name = objectID.rsplit('/')[-3] # See tests for reference
    return job_name in buckets[bucketID].get('exclude_jobs', [])


def process_changes(results, buckets):
    """Split GCS change events into trivial ack_ids and builds to further process."""
    ack_ids = []  # pubsub rec_message ids to acknowledge
    todo = []  # (id, job, build) of builds to grab
    # process results, find finished builds to process
    for rec_message in results:
        eventType = rec_message.message.attributes['eventType']
        objectID = rec_message.message.attributes['objectId']
{'fraction_non_alphanumeric': 0.08204102051025512, 'fraction_numerical': 0.008504252126063032, 'mean_word_length': 4.186770428015564, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '867337', 'n_tokens_mistral': 1137, 'n_tokens_neox': 1086, 'n_words': 439}