<commit_before>module.exports = {
  siteMetadata: {
    title: `AJ Fisher Website`,
    description: `Personal website of AJFisher - technologist`,
    author: `@ajfisher`,
    siteURL: 'https://ajfisher.me'
  },
  plugins: [
    `gatsby-plugin-react-helmet`,
    {
      resolve: `gatsby-source-filesystem`,
      options: {
        name: `image`,
        path: `${__dirname}/src/img`,
      },
    },
    {
      resolve: `gatsby-source-filesystem`,
      options: {
        name: `content`,
        path: `${__dirname}/src/content`,
        ignore: [`**/\.*`],
      },
    },
    `gatsby-transformer-sharp`,
    `gatsby-plugin-sharp`,
    {
      resolve: `gatsby-transformer-remark`,
      options: {
        plugins: [
          `gatsby-transformer-remark-tags`,
          `gatsby-remark-reading-time`,
          {
            resolve: `gatsby-remark-relative-images`
          },
          {
            resolve: `gatsby-remark-images`,
            options: {
              maxWidth: 1000,
              backgroundColor: `none`,
              quality: 90,
              withWebp: { quality: 90 },
              showCaptions: true,
              disableBgImage: true
            },
          },
          {
            resolve: `gatsby-remark-responsive-image`,
            options: {
              maxWidth: 1000,
              backgroundColor: `none`,
              quality: 90,
              withWebp: { quality: 90 },
              showCaptions: true,
              disableBgImage: true
            }
          }
        ]
      }
    },
    {
      resolve: `gatsby-plugin-manifest`,
      options: {
        name: `gatsby-starter-default`,
        short_name: `starter`,
        start_url: `/`,
        background_color: `#663399`,
        theme_color: `#663399`,
        display: `minimal-ui`,
        icon: `src/images/gatsby-icon.png`, // This path is relative to the root of the site.
      },
    },
    // this (optional) plugin enables Progressive Web App + Offline functionality
    // To learn more, visit: https://gatsby.dev/offline
    // `gatsby-plugin-offline`,
  ],
}
<commit_msg>feat: add robots.txt file and configuration. #27<commit_after>module.exports = {
  siteMetadata: {
    title: `AJ Fisher | Blog | Portfolio`,
    description: `Website of AJFisher - technologist`,
    author: `@ajfisher`,
    siteURL: 'https://ajfisher.me'
  },
  plugins: [
    `gatsby-plugin-react-helmet`,
    {
      resolve: `gatsby-source-filesystem`,
      options: {
        name: `image`,
        path: `${__dirname}/src/img`,
      },
    },
    {
      resolve: `gatsby-source-filesystem`,
      options: {
        name: `content`,
        path: `${__dirname}/src/content`,
        ignore: [`**/\.*`],
      },
    },
    `gatsby-transformer-sharp`,
    `gatsby-plugin-sharp`,
    {
      resolve: `gatsby-transformer-remark`,
      options: {
        plugins: [
          `gatsby-transformer-remark-tags`,
          `gatsby-remark-reading-time`,
          {
            resolve: `gatsby-remark-relative-images`
          },
          {
            resolve: `gatsby-remark-images`,
            options: {
              maxWidth: 1000,
              backgroundColor: `none`,
              quality: 90,
              withWebp: { quality: 90 },
              showCaptions: true,
              disableBgImage: true
            },
          },
          {
            resolve: `gatsby-remark-responsive-image`,
            options: {
              maxWidth: 1000,
              backgroundColor: `none`,
              quality: 90,
              withWebp: { quality: 90 },
              showCaptions: true,
              disableBgImage: true
            }
          }
        ]
      }
    },
    {
      resolve: `gatsby-plugin-manifest`,
      options: {
        name: `gatsby-starter-default`,
        short_name: `starter`,
        start_url: `/`,
        background_color: `#663399`,
        theme_color: `#663399`,
        display: `minimal-ui`,
        icon: `src/images/gatsby-icon.png`, // This path is relative to the root of the site.
      },
    },
    `gatsby-plugin-robots-txt`
    // this (optional) plugin enables Progressive Web App + Offline functionality
    // To learn more, visit: https://gatsby.dev/offline
    // `gatsby-plugin-offline`,
  ],
}

{'fraction_non_alphanumeric': 0.1352803738317757, 'fraction_numerical': 0.013551401869158878, 'mean_word_length': 1.571171171171171, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7422456', 'n_tokens_mistral': 1351, 'n_tokens_neox': 1278, 'n_words': 239}