<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import warnings

import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions

from mappers.ads_ssd_hashing_mapper import AdsSSDHashingMapper
from mappers.ads_user_list_pii_hashing_mapper import AdsUserListPIIHashingMapper
from sources.filter_load_and_group_data import FilterLoadAndGroupData
from sources.spreadsheet_execution_source import SpreadsheetExecutionSource
from sources.transactional_events_bq_api_dofn import TransactionalEventsBigQueryApiDoFn
# from uploaders.appsflyer_s2s_uploader import AppsFlyerS2SUploaderDoFn
from uploaders.appsflyer_s2s_uploader_async import AppsFlyerS2SUploaderDoFn
from uploaders.campaign_manager_conversion_uploader import CampaignManagerConversionUploaderDoFn
from uploaders.google_ads_customer_match.contact_info_uploader import GoogleAdsCustomerMatchContactInfoUploaderDoFn
from uploaders.google_ads_customer_match.mobile_uploader import GoogleAdsCustomerMatchMobileUploaderDoFn
from uploaders.google_ads_customer_match.user_id_uploader import GoogleAdsCustomerMatchUserIdUploaderDoFn
from uploaders.google_ads_offline_conversions_uploader import GoogleAdsOfflineUploaderDoFn
from uploaders.google_ads_ssd_uploader import GoogleAdsSSDUploaderDoFn
from uploaders.google_analytics_data_import_uploader import GoogleAnalyticsDataImportUploaderDoFn
from uploaders.google_analytics_measurement_protocol import GoogleAnalyticsMeasurementProtocolUploaderDoFn
from uploaders.google_analytics_user_list_uploader import GoogleAnalyticsUserListUploaderDoFn
from utils.execution import DestinationType, Execution
from utils.google_analytics_data_import_eraser import GoogleAnalyticsDataImportEraser
from utils.oauth_credentials import OAuthCredentials
from utils.options import DataflowOptions
from utils.sheets_config import SheetsConfig
from utils.transactional_events_results_writer import TransactionalEventsResultsWriter

warnings.filterwarnings("ignore", "Your application has authenticated using end user credentials")


def filter_by_action(execution: Execution, destination_type: DestinationType):
  return execution.destination.destination_type is destination_type


def run(argv=None):
  pipeline_options = PipelineOptions()
  dataflow_options = pipeline_options.view_as(DataflowOptions)
  oauth_credentials = OAuthCredentials(dataflow_options.client_id, dataflow_options.client_secret,
                                       dataflow_options.developer_token, dataflow_options.refresh_token)

  sheets_config = SheetsConfig(oauth_credentials)
  # conversion_plus_mapper = ConversionPlusMapper(
  #   sheets_config, dataflow_options.cp_sheet_id, dataflow_options.cp_sheet_range)
  user_list_hasher = AdsUserListPIIHashingMapper()

  with beam.Pipeline(options=pipeline_options) as pipeline:
    executions = (pipeline | 'Load executions' >> beam.io.Read(
      SpreadsheetExecutionSource(sheets_config, dataflow_options.setup_sheet_id)))

    _add_google_ads_user_list_upload(executions, user_list_hasher, oauth_credentials, dataflow_options)
    _add_google_ads_offline_conversion(executions, None, oauth_credentials, dataflow_options)
    _add_google_ads_ssd(executions, AdsSSDHashingMapper(), oauth_credentials, dataflow_options)
    _add_ga_user_list(executions, oauth_credentials)
    _add_ga_data_import(executions, oauth_credentials)
    _add_ga_measurement_protocol(executions, dataflow_options)
    _add_cm_conversion(executions, oauth_credentials)
    _add_appsflyer_s2s_events(executions, dataflow_options)

    # todo: update trix at the end


def _add_google_ads_user_list_upload(pipeline, hasher, oauth_credentials, dataflow_options):<commit_msg>Updates main file.<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import warnings

import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions

from mappers.ads_ssd_hashing_mapper import AdsSSDHashingMapper
from mappers.ads_user_list_pii_hashing_mapper import AdsUserListPIIHashingMapper
from sources.filter_load_and_group_data import FilterLoadAndGroupData
from sources.spreadsheet_execution_source import SpreadsheetExecutionSource
from sources.transactional_events_bq_api_dofn import TransactionalEventsBigQueryApiDoFn
from uploaders.appsflyer_s2s_uploader_async import AppsFlyerS2SUploaderDoFn
from uploaders.campaign_manager_conversion_uploader import CampaignManagerConversionUploaderDoFn
from uploaders.google_ads_customer_match.contact_info_uploader import GoogleAdsCustomerMatchContactInfoUploaderDoFn
from uploaders.google_ads_customer_match.mobile_uploader import GoogleAdsCustomerMatchMobileUploaderDoFn
from uploaders.google_ads_customer_match.user_id_uploader import GoogleAdsCustomerMatchUserIdUploaderDoFn
from uploaders.google_ads_offline_conversions_uploader import GoogleAdsOfflineUploaderDoFn
from uploaders.google_ads_ssd_uploader import GoogleAdsSSDUploaderDoFn
from uploaders.google_analytics_data_import_uploader import GoogleAnalyticsDataImportUploaderDoFn
from uploaders.google_analytics_measurement_protocol import GoogleAnalyticsMeasurementProtocolUploaderDoFn
from uploaders.google_analytics_user_list_uploader import GoogleAnalyticsUserListUploaderDoFn
from utils.execution import DestinationType
from utils.execution import Execution
from utils.google_analytics_data_import_eraser import GoogleAnalyticsDataImportEraser
from utils.oauth_credentials import OAuthCredentials
from utils.options import DataflowOptions
from utils.sheets_config import SheetsConfig
from utils.transactional_events_results_writer import TransactionalEventsResultsWriter

warnings.filterwarnings(
    "ignore", "Your application has authenticated using end user credentials")


def filter_by_action(execution: Execution, destination_type: DestinationType):
  return execution.destination.destination_type is destination_type


def run(argv=None):
  pipeline_options = PipelineOptions()
  dataflow_options = pipeline_options.view_as(DataflowOptions)
  oauth_credentials = OAuthCredentials(
      dataflow_options.client_id,
      dataflow_options.client_secret,
      dataflow_options.developer_token,
      dataflow_options.refresh_token)

  sheets_config = SheetsConfig(oauth_credentials)
  # conversion_plus_mapper = ConversionPlusMapper(
  #   sheets_config, dataflow_options.cp_sheet_id, dataflow_options.cp_sheet_range)
  user_list_hasher = AdsUserListPIIHashingMapper()

  with beam.Pipeline(options=pipeline_options) as pipeline:
    executions = (pipeline | 'Load executions' >> beam.io.Read(
      SpreadsheetExecutionSource(sheets_config, dataflow_options.setup_sheet_id)))

    _add_google_ads_user_list_upload(executions, user_list_hasher, oauth_credentials, dataflow_options)
    _add_google_ads_offline_conversion(executions, None, oauth_credentials, dataflow_options)
    _add_google_ads_ssd(executions, AdsSSDHashingMapper(), oauth_credentials, dataflow_options)
    _add_ga_user_list(executions, oauth_credentials)
    _add_ga_data_import(executions, oauth_credentials)
    _add_ga_measurement_protocol(executions, dataflow_options)
    _add_cm_conversion(executions, oauth_credentials)
    _add_appsflyer_s2s_events(executions, dataflow_options)

    # todo: update trix at the end


def _add_google_ads_user_list_upload(pipeline, hasher, oauth_credentials, dataflow_options):
{'fraction_non_alphanumeric': 0.04306560683355084, 'fraction_numerical': 0.002847312848499229, 'mean_word_length': 8.113513513513514, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5799344', 'n_tokens_mistral': 2457, 'n_tokens_neox': 2423, 'n_words': 534}