<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for fetching account structure using Google Ads reporting API."""

import json
from google.ads.google_ads.client import GoogleAdsClient


class RowsIterator(object):
  """Streamed report results iterator."""

  def __init__(self, response):
    self._response = response
    self._results = None

  def _next_batch(self):
    self._batch = next(self._response)
    self._results = iter(self._batch.results)

  def __iter__(self):
    return self

  def __next__(self):
    if self._results is None:
      self._next_batch()
    try:
      return next(self._results)
    except StopIteration:
      self._next_batch()
      return next(self._results)


class StructureBuilder(object):
  """Abstract structure builder class."""

  def __init__(self, client, customer_id):
    self._service = client.get_service('GoogleAdsService', version='v4')
    self._customer_id = customer_id
    self._enums = {
        'type': client.get_type('AssetTypeEnum').AssetType,
        'field_type': client.get_type('AssetFieldTypeEnum').AssetFieldType,
    }

  def _get_rows(self, query, account=None):
    if account:
      cid = account
    else:
      cid = str((self._customer_id))

    response = self._service.search_stream(cid, query)
    return RowsIterator(response)

  def _build_asset(self, row):
    asset_type = self._enums['type'].Name(row.asset.type)
    asset = {
        'id': row.asset.id.value,
        'name': row.asset.name.value,
        'type': asset_type,
        'stats':{
          'clicks': row.metrics.clicks.value,
          'all_conversions' : row.metrics.all_conversions.value,
          'impressions': row.metrics.impressions.value,
          'cost':row.metrics.cost_micros.value / 1000000
        }
    }
    if asset_type == 'IMAGE':
      asset['image_url'] = row.asset.image_asset.full_size.url.value
      asset['file_size'] = row.asset.image_asset.file_size.value
      asset['image_height'] = row.asset.image_asset.full_size.height_pixels.value
      asset['image_width'] = row.asset.image_asset.full_size.width_pixels.value
    elif asset_type == 'TEXT':
      text_type = self._enums['field_type'].Name(
          row.ad_group_ad_asset_view.field_type)
      asset['text_type'] = text_type.lower() + 's'
      asset['asset_text'] = row.asset.text_asset.text.value
    elif asset_type == 'YOUTUBE_VIDEO':
      video_id = row.asset.youtube_video_asset.youtube_video_id.value
      asset['video_id'] = video_id
      asset['link'] = f'https://www.youtube.com/watch?v={video_id}'
      asset['image_url'] = f'https://img.youtube.com/vi/{video_id}/1.jpg'
    return asset

  def build(self):
    return None


class AdGroupAssetsStructureBuilder(StructureBuilder):
  """Ad group assets structure builder class."""

  def build(self, ad_group_id, account):
    rows = self._get_rows(f'''
        SELECT
          ad_group.id,
          asset.id,
          asset.name,
          asset.type,
          asset.image_asset.full_size.url,
          ad_group_ad_asset_view.field_type,
          asset.text_asset.text,
          asset.youtube_video_asset.youtube_video_id
        FROM
          ad_group_ad_asset_view
        WHERE
          ad_group.id = {ad_group_id}
    ''')
    return [self._build_asset(row) for row in rows]


class AccountAssetsBuilder(StructureBuilder):
  """All assets under an account structure builder."""

  def build(self,account, mcc_struct_file):
    rows = self._get_rows(f'''
    SELECT 
      asset.name,
      asset.id,
      asset.image_asset.file_size, 
      asset.image_asset.full_size.url, 
      asset.text_asset.text, 
      asset.image_asset.full_size.height_pixels,
      asset.image_asset.full_size.width_pixels,
      asset.youtube_video_asset.youtube_video_id,
      asset.type
    FROM
      asset
      ''', account)

    account_assets = {}
    for row in rows:
      asset = self._build_asset(row)
      account_assets[str(asset['id'])] = asset
      

    with open(mcc_struct_file, 'r') as f:
      struc = json.load(f)

    for acc in struc:
      if acc['id'] == int(account):
        account_campaigns_handler = acc['campaigns']
        break

    if account_campaigns_handler:
      for campaign in account_campaigns_handler:
        for adgroup in campaign['adgroups']:
          for asset in adgroup['assets']:
            asset_id = asset['id']
            for k in asset['stats']:
              account_assets[str(asset_id)]['stats'][k] += asset['stats'][k]
            
    return list(account_assets.values())


class AccountStructureBuilder(StructureBuilder):
  """Account structure builder class."""

  _CAMPAIGN_FILTER = '''
      campaign.advertising_channel_sub_type IN(
        'APP_CAMPAIGN',
        'APP_CAMPAIGN_FOR_ENGAGEMENT',
        'SEARCH_MOBILE_APP',
        'DISPLAY_MOBILE_APP'
      )
  '''

  def __init__(self, client, customer_id, name):
    super().__init__(client, customer_id)
    self._name = name

  def _populate_campaigns_and_ad_groups(self):
    self._campaigns = []
    self._ad_groups = {}
    campaigns = {}

    rows = self._get_rows(f'''
        SELECT
          campaign.id,
          campaign.name
        FROM
          campaign
        WHERE
          {self._CAMPAIGN_FILTER}
    ''')
    for row in rows:
      campaign = {
          'id': row.campaign.id.value,
          'campaign_name': row.campaign.name.value,
          'adgroups': [],
      }
      self._campaigns.append(campaign)
      campaigns[campaign['id']] = campaign

    rows = self._get_rows(f'''
        SELECT
          campaign.id,
          ad_group.id,
          ad_group.name
        FROM
          ad_group
        WHERE
          {self._CAMPAIGN_FILTER}
    ''')
    for row in rows:
      ad_group = {
          'id': row.ad_group.id.value,
          'name': row.ad_group.name.value,
          'assets': [],
      }
      campaigns[row.campaign.id.value]['adgroups'].append(ad_group)
      self._ad_groups[ad_group['id']] = ad_group

  def build(self):
    structure = {
        'id': self._customer_id,
        'name': self._name,
    }
    self._populate_campaigns_and_ad_groups()
    rows = self._get_rows(f'''
        SELECT
          ad_group.id,
          asset.id,
          asset.name,
          asset.type,
          asset.image_asset.full_size.url,
          asset.image_asset.file_size,
          asset.image_asset.full_size.height_pixels,
          asset.image_asset.full_size.width_pixels,
          ad_group_ad_asset_view.field_type,
          asset.text_asset.text,
          asset.youtube_video_asset.youtube_video_id,
          metrics.all_conversions,
          metrics.impressions,
          metrics.clicks,
          metrics.cost_micros
        FROM
          ad_group_ad_asset_view
        WHERE
          {self._CAMPAIGN_FILTER}
    ''')
    for row in rows:
      asset = self._build_asset(row)
      self._ad_groups[row.ad_group.id.value]['assets'].append(asset)
    structure['campaigns'] = self._campaigns
    return structure

<commit_msg>add adgroup and campaign status to structure<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for fetching account structure using Google Ads reporting API."""

import json
from google.ads.google_ads.client import GoogleAdsClient


class RowsIterator(object):
  """Streamed report results iterator."""

  def __init__(self, response):
    self._response = response
    self._results = None

  def _next_batch(self):
    self._batch = next(self._response)
    self._results = iter(self._batch.results)

  def __iter__(self):
    return self

  def __next__(self):
    if self._results is None:
      self._next_batch()
    try:
      return next(self._results)
    except StopIteration:
      self._next_batch()
      return next(self._results)


class StructureBuilder(object):
  """Abstract structure builder class."""

  def __init__(self, client, customer_id):
    self._service = client.get_service('GoogleAdsService', version='v4')
    self._customer_id = customer_id
    self._enums = {
        'type': client.get_type('AssetTypeEnum').AssetType,
        'field_type': client.get_type('AssetFieldTypeEnum').AssetFieldType,
    }

  def _get_rows(self, query, account=None):
    if account:
      cid = account
    else:
      cid = str((self._customer_id))

    response = self._service.search_stream(cid, query)
    return RowsIterator(response)

  def _build_asset(self, row):
    asset_type = self._enums['type'].Name(row.asset.type)
    asset = {
        'id': row.asset.id.value,
        'name': row.asset.name.value,
        'type': asset_type,
        'stats':{
          'clicks': row.metrics.clicks.value,
          'all_conversions' : row.metrics.all_conversions.value,
          'impressions': row.metrics.impressions.value,
          'cost':row.metrics.cost_micros.value / 1000000
        }
    }
    if asset_type == 'IMAGE':
      asset['image_url'] = row.asset.image_asset.full_size.url.value
      asset['file_size'] = row.asset.image_asset.file_size.value
      asset['image_height'] = row.asset.image_asset.full_size.height_pixels.value
      asset['image_width'] = row.asset.image_asset.full_size.width_pixels.value
    elif asset_type == 'TEXT':
      text_type = self._enums['field_type'].Name(
          row.ad_group_ad_asset_view.field_type)
      asset['text_type'] = text_type.lower() + 's'
      asset['asset_text'] = row.asset.text_asset.text.value
    elif asset_type == 'YOUTUBE_VIDEO':
      video_id = row.asset.youtube_video_asset.youtube_video_id.value
      asset['video_id'] = video_id
      asset['link'] = f'https://www.youtube.com/watch?v={video_id}'
      asset['image_url'] = f'https://img.youtube.com/vi/{video_id}/1.jpg'
    return asset

  def build(self):
    return None


class AdGroupAssetsStructureBuilder(StructureBuilder):
  """Ad group assets structure builder class."""

  def build(self, ad_group_id, account):
    rows = self._get_rows(f'''
        SELECT
          ad_group.id,
          asset.id,
          asset.name,
          asset.type,
          asset.image_asset.full_size.url,
          ad_group_ad_asset_view.field_type,
          asset.text_asset.text,
          asset.youtube_video_asset.youtube_video_id
        FROM
          ad_group_ad_asset_view
        WHERE
          ad_group.id = {ad_group_id}
    ''')
    return [self._build_asset(row) for row in rows]


class AccountAssetsBuilder(StructureBuilder):
  """All assets under an account structure builder."""

  def build(self,account, mcc_struct_file):
    rows = self._get_rows(f'''
    SELECT 
      asset.name,
      asset.id,
      asset.image_asset.file_size, 
      asset.image_asset.full_size.url, 
      asset.text_asset.text, 
      asset.image_asset.full_size.height_pixels,
      asset.image_asset.full_size.width_pixels,
      asset.youtube_video_asset.youtube_video_id,
      asset.type
    FROM
      asset
      ''', account)

    account_assets = {}
    for row in rows:
      asset = self._build_asset(row)
      account_assets[str(asset['id'])] = asset
      

    with open(mcc_struct_file, 'r') as f:
      struc = json.load(f)

    for acc in struc:
      if acc['id'] == int(account):
        account_campaigns_handler = acc['campaigns']
        break

    if account_campaigns_handler:
      for campaign in account_campaigns_handler:
        for adgroup in campaign['adgroups']:
          for asset in adgroup['assets']:
            asset_id = asset['id']
            for k in asset['stats']:
              account_assets[str(asset_id)]['stats'][k] += asset['stats'][k]
            
    return list(account_assets.values())


class AccountStructureBuilder(StructureBuilder):
  """Account structure builder class."""

  _CAMPAIGN_FILTER = '''
      campaign.advertising_channel_sub_type IN(
        'APP_CAMPAIGN',
        'APP_CAMPAIGN_FOR_ENGAGEMENT',
        'SEARCH_MOBILE_APP',
        'DISPLAY_MOBILE_APP'
      )
      AND campaign.status NOT IN('REMOVED')
  '''

  _AD_GROUP_FILTER = '''
  ad_group.status NOT IN ('REMOVED')
  '''

  def __init__(self, client, customer_id, name):
    super().__init__(client, customer_id)
    self._name = name
    self._status_enums = {
      "adgroup_status": client.get_type('AdGroupStatusEnum').AdGroupStatus,
      "campaign_status" : client.get_type('CampaignStatusEnum').CampaignStatus,
    }

  def _populate_campaigns_and_ad_groups(self):
    self._campaigns = []
    self._ad_groups = {}
    campaigns = {}

    rows = self._get_rows(f'''
        SELECT
          campaign.id,
          campaign.name,
          campaign.status
        FROM
          campaign
        WHERE
          {self._CAMPAIGN_FILTER}
    ''')
    for row in rows:
      campaign_status = self._status_enums['campaign_status'].Name(row.campaign.status)
      campaign = {
          'id': row.campaign.id.value,
          'campaign_name': row.campaign.name.value,
          'status': campaign_status,
          'adgroups': [],
      }
      self._campaigns.append(campaign)
      campaigns[campaign['id']] = campaign

    rows = self._get_rows(f'''
        SELECT
          campaign.id,
          ad_group.id,
          ad_group.name,
          ad_group.status
        FROM
          ad_group
        WHERE
          {self._CAMPAIGN_FILTER}
        AND
          {self._AD_GROUP_FILTER}
    ''')
    for row in rows:
      adgroup_status = self._status_enums['adgroup_status'].Name(row.ad_group.status)
      ad_group = {
          'id': row.ad_group.id.value,
          'name': row.ad_group.name.value,
          'status': adgroup_status,
          'assets': [],
      }
      campaigns[row.campaign.id.value]['adgroups'].append(ad_group)
      self._ad_groups[ad_group['id']] = ad_group

  def build(self):
    structure = {
        'id': self._customer_id,
        'name': self._name,
    }
    self._populate_campaigns_and_ad_groups()
    rows = self._get_rows(f'''
        SELECT
          ad_group.id,
          asset.id,
          asset.name,
          asset.type,
          asset.image_asset.full_size.url,
          asset.image_asset.file_size,
          asset.image_asset.full_size.height_pixels,
          asset.image_asset.full_size.width_pixels,
          ad_group_ad_asset_view.field_type,
          asset.text_asset.text,
          asset.youtube_video_asset.youtube_video_id,
          metrics.all_conversions,
          metrics.impressions,
          metrics.clicks,
          metrics.cost_micros
        FROM
          ad_group_ad_asset_view
        WHERE
          {self._CAMPAIGN_FILTER}
        AND
          {self._AD_GROUP_FILTER}
    ''')
    for row in rows:
      asset = self._build_asset(row)
      self._ad_groups[row.ad_group.id.value]['assets'].append(asset)
    structure['campaigns'] = self._campaigns
    return structure


{'fraction_non_alphanumeric': 0.11103396035854059, 'fraction_numerical': 0.0021461936624163616, 'mean_word_length': 2.612175102599179, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6477034', 'n_tokens_mistral': 5002, 'n_tokens_neox': 4870, 'n_words': 948}