<commit_before>module Docs
  class Drupal < UrlScraper
    self.type = 'drupal'
    self.version = '7.38'
    self.base_url = 'https://api.drupal.org/api/drupal/'
    self.initial_paths = %w(groups groups?page=1)
    self.links = {
      home: 'https://www.drupal.org/'
    }

    html_filters.push 'drupal/entries', 'drupal/clean_html', 'title'

    options[:decode_and_clean_paths] = true
    options[:container] = '#page-inner'
    options[:title] = false
    options[:root_title] = 'Drupal'

    options[:only_patterns] = [
      /\/class\/[^\/]+/,
      /\/group\/[^\/]+/,
      /\/function\/[^\/]+/]

    options[:skip_link] = ->(link) { link['href'] =~ /[\?&]order/ }

    options[:skip_patterns] = [
      /\/group\/updates\-7/,
      /\/group\/updates\-6/,
      /_update_[0-9]{4}/,               # Skip update functions
      /\/[4-6](\.[0-9])*$/,             # Skip previous versions
      /\/[8-9](\.[0-9])*$/,             # Skip future versions
      /\/class\/hierarchy\//,           # Skip class hierarchy listings
      /\/function\/calls\//,            # Skip function calls listings
      /\/function\/invokes\//,          # Skip function invokations listings<commit_msg>Update Drupal documentation (7.40)<commit_after>module Docs
  class Drupal < UrlScraper
    self.type = 'drupal'
    self.version = '7.40'
    self.base_url = 'https://api.drupal.org/api/drupal/'
    self.initial_paths = %w(groups groups?page=1)
    self.links = {
      home: 'https://www.drupal.org/'
    }

    html_filters.push 'drupal/entries', 'drupal/clean_html', 'title'

    options[:decode_and_clean_paths] = true
    options[:container] = '#page-inner'
    options[:title] = false
    options[:root_title] = 'Drupal'

    options[:only_patterns] = [
      /\/class\/[^\/]+/,
      /\/group\/[^\/]+/,
      /\/function\/[^\/]+/]

    options[:skip_link] = ->(link) { link['href'] =~ /[\?&]order/ }

    options[:skip_patterns] = [
      /\/group\/updates\-7/,
      /\/group\/updates\-6/,
      /_update_[0-9]{4}/,               # Skip update functions
      /\/[4-6](\.[0-9])*$/,             # Skip previous versions
      /\/[8-9](\.[0-9])*$/,             # Skip future versions
      /\/class\/hierarchy\//,           # Skip class hierarchy listings
      /\/function\/calls\//,            # Skip function calls listings
      /\/function\/invokes\//,          # Skip function invokations listings
{'fraction_non_alphanumeric': 0.20383493122134222, 'fraction_numerical': 0.015423092955398083, 'mean_word_length': 1.952029520295203, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7362044', 'n_tokens_mistral': 829, 'n_tokens_neox': 803, 'n_words': 136}