<commit_before>{ fabricate } = require 'antigravity'
sinon = require 'sinon'
Backbone = require 'backbone'
rewire = require 'rewire'
routes = rewire '../routes'
moment = require 'moment'
PartnerFeaturedCities = require '../../../collections/partner_featured_cities'

describe 'Sitemaps', ->

  beforeEach ->
    sinon.stub Backbone, 'sync'
    @req = {}
    @res =
      render: sinon.stub()
      set: sinon.stub()
      send: sinon.stub()

  afterEach ->
    Backbone.sync.restore()

  describe '#robots', ->
    beforeEach ->
      @res = set: sinon.stub(), send: sinon.stub()

    afterEach ->
      routes.__set__ 'ENABLE_WEB_CRAWLING', false

    it 'renders a noindex when ENABLE_WEB_CRAWLING is false', ->
      routes.__set__ 'ENABLE_WEB_CRAWLING', false
      routes.robots null, @res
      @res.send.args[0][0]
        .should.equal  'User-agent: *\nDisallow: /'

    describe 'when ENABLE_WEB_CRAWLING is true', ->
      beforeEach ->
        routes.__set__ ENABLE_WEB_CRAWLING: true, APP_URL: 'https://www.artsy.net'
        routes.robots null, @res
      it 'renders robots.txt', ->
        @res.send.args[0][0]
          .should.eql """
User-agent: *
Noindex: ?sort=
Noindex: ?dimension_range=
Disallow: ?dns_source=
Disallow: ?microsite=
Disallow: ?from-show-guide=
Sitemap: https://www.artsy.net/sitemap-articles.xml
Sitemap: https://www.artsy.net/sitemap-artists.xml
Sitemap: https://www.artsy.net/sitemap-artist-images.xml
Sitemap: https://www.artsy.net/sitemap-artworks.xml
Sitemap: https://www.artsy.net/sitemap-cities.xml
Sitemap: https://www.artsy.net/sitemap-collect.xml
Sitemap: https://www.artsy.net/sitemap-fairs.xml
Sitemap: https://www.artsy.net/sitemap-features.xml
Sitemap: https://www.artsy.net/sitemap-genes.xml
Sitemap: https://www.artsy.net/sitemap-images.xml<commit_msg>Update robots test to match return<commit_after>{ fabricate } = require 'antigravity'
sinon = require 'sinon'
Backbone = require 'backbone'
rewire = require 'rewire'
routes = rewire '../routes'
moment = require 'moment'
PartnerFeaturedCities = require '../../../collections/partner_featured_cities'

describe 'Sitemaps', ->

  beforeEach ->
    sinon.stub Backbone, 'sync'
    @req = {}
    @res =
      render: sinon.stub()
      set: sinon.stub()
      send: sinon.stub()

  afterEach ->
    Backbone.sync.restore()

  describe '#robots', ->
    beforeEach ->
      @res = set: sinon.stub(), send: sinon.stub()

    afterEach ->
      routes.__set__ 'ENABLE_WEB_CRAWLING', false

    it 'renders a noindex when ENABLE_WEB_CRAWLING is false', ->
      routes.__set__ 'ENABLE_WEB_CRAWLING', false
      routes.robots null, @res
      @res.send.args[0][0]
        .should.equal  'User-agent: *\nDisallow: /'

    describe 'when ENABLE_WEB_CRAWLING is true', ->
      beforeEach ->
        routes.__set__ ENABLE_WEB_CRAWLING: true, APP_URL: 'https://www.artsy.net'
        routes.robots null, @res
      it 'renders robots.txt', ->
        @res.send.args[0][0]
          .should.eql """
User-agent: *
Noindex: ?sort=
Noindex: ?dimension_range=
Disallow: /*?dns_source=
Disallow: /*?microsite=
Disallow: /*?from-show-guide=
Sitemap: https://www.artsy.net/sitemap-articles.xml
Sitemap: https://www.artsy.net/sitemap-artists.xml
Sitemap: https://www.artsy.net/sitemap-artist-images.xml
Sitemap: https://www.artsy.net/sitemap-artworks.xml
Sitemap: https://www.artsy.net/sitemap-cities.xml
Sitemap: https://www.artsy.net/sitemap-collect.xml
Sitemap: https://www.artsy.net/sitemap-fairs.xml
Sitemap: https://www.artsy.net/sitemap-features.xml
Sitemap: https://www.artsy.net/sitemap-genes.xml
Sitemap: https://www.artsy.net/sitemap-images.xml
{'fraction_non_alphanumeric': 0.1544380324264908, 'fraction_numerical': 0.0021984061555372354, 'mean_word_length': 3.328180737217598, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 22, 'lorem ipsum': 0, 'www.': 22, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3424293', 'n_tokens_mistral': 1374, 'n_tokens_neox': 1370, 'n_words': 240}