<commit_before>"""
Helper classes and methods.
"""

import requests

WIKIPEDIA_API_URL = 'https://en.wikipedia.org/w/api.php'


def fetch_wikipedia_pages_info(page_ids, database):
  """Fetched page information such as title, URL, and image thumbnail URL for the provided page IDs.

  Args:
    page_title: The page title to validate.

  Returns:
    None

  Raises:
    ValueError: If the provided page title is invalid.
  """
  pages_info = {}

  current_page_ids_index = 0
  while current_page_ids_index < len(page_ids):
    # Query at most 50 pages per request (given WikiMedia API limits)
    end_page_ids_index = min(current_page_ids_index + 50, len(page_ids))

    query_params = {
        'action': 'query',
        'format': 'json',
        'pageids': '|'.join(page_ids[current_page_ids_index:end_page_ids_index]),
        'prop': 'info|pageimages|pageterms',
        'inprop': 'url|displaytitle',
        'piprop': 'thumbnail',
        'pithumbsize': 160,
        'pilimit': 50,
        'wbptterms': 'description',
    }

    current_page_ids_index = end_page_ids_index

    # Identify this client as per Wikipedia API guidelines.
    # https://www.mediawiki.org/wiki/API:Main_page#Identifying_your_client
    headers = {
        'User-Agent': 'Six Degrees of Wikipedia/1.0 (https://www.sixdegreesofwikipedia.com/; <EMAIL>)',
    }

    req = requests.get(WIKIPEDIA_API_URL, params=query_params, headers=headers)

    pages_result = req.json().get('query', {}).get('pages')
<commit_msg>Add more logging for when MediaWiki API fails<commit_after>"""
Helper classes and methods.
"""

import logging
import requests


WIKIPEDIA_API_URL = 'https://en.wikipedia.org/w/api.php'


def fetch_wikipedia_pages_info(page_ids, database):
  """Fetched page information such as title, URL, and image thumbnail URL for the provided page IDs.

  Args:
    page_title: The page title to validate.

  Returns:
    None

  Raises:
    ValueError: If the provided page title is invalid.
  """
  pages_info = {}

  current_page_ids_index = 0
  while current_page_ids_index < len(page_ids):
    # Query at most 50 pages per request (given WikiMedia API limits)
    end_page_ids_index = min(current_page_ids_index + 50, len(page_ids))

    query_params = {
        'action': 'query',
        'format': 'json',
        'pageids': '|'.join(page_ids[current_page_ids_index:end_page_ids_index]),
        'prop': 'info|pageimages|pageterms',
        'inprop': 'url|displaytitle',
        'piprop': 'thumbnail',
        'pithumbsize': 160,
        'pilimit': 50,
        'wbptterms': 'description',
    }

    current_page_ids_index = end_page_ids_index

    # Identify this client as per Wikipedia API guidelines.
    # https://www.mediawiki.org/wiki/API:Main_page#Identifying_your_client
    headers = {
        'User-Agent': 'Six Degrees of Wikipedia/1.0 (https://www.sixdegreesofwikipedia.com/; <EMAIL>)',
    }

    req = requests.get(WIKIPEDIA_API_URL, params=query_params, headers=headers)

    try:
      pages_result = req.json().get('query', {}).get('pages')
    except Exception as error:
      # Log and re-raise the exception.
      logging.exception({
          'error': 'Failed to decode MediaWiki API response: "{0}"'.format(error),
          'status_code': req.status_code,
          'response_text': req.text,
      })
      raise error

{'fraction_non_alphanumeric': 0.1199400299850075, 'fraction_numerical': 0.0074962518740629685, 'mean_word_length': 3.3550913838120104, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4690524', 'n_tokens_mistral': 1124, 'n_tokens_neox': 1068, 'n_words': 258}