<commit_before>{EventEmitter} = require 'events'

jraphical = require 'jraphical'
{Base, race, dash, daisy} = require "bongo"
{CronJob} = require 'cron'
_ = require "underscore"

NAMEPERPAGE = 50000
GROUPPERPAGE = 5
module.exports = class SitemapGeneratorWorker extends EventEmitter
  constructor: (@bongo, @options = {}) ->

  generateSitemapString: (urls)->
     # sitemap.XML beginning and ending parts
    sitemap = '<?xml version="1.0" encoding="UTF-8"?>
      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" 
      xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" 
      xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">'
    sitemapFooter = '</urlset>'

    for url in urls
      sitemap += "<url><loc>#{@options.uri.address}/#{url}</loc></url>"
    sitemap += sitemapFooter

  generateSitemapName: (skip)->
    return  skip + "_" + (skip + NAMEPERPAGE)

  saveSitemap: (name, content)->
    {JSitemap} = @bongo.models
    JSitemap.update {name}, $set : {content}, {upsert : yes}, (err)-> 
      console.log err if err
  generateMainSitemap: (sitemapNames)=>
    name = "main"
    content = @generateSitemapString sitemapNames
    @saveSitemap name, content
  generate:=>
    {JName, JGroup, JSitemap} = @bongo.models

    sitemapNames = []


    groupSelector = {
      privacy:'public'
    }

    JGroup.count groupSelector, (err, groupCount)=>
      numberOfGroupPages = Math.ceil(groupCount / GROUPPERPAGE)
      for groupPageNumber in [1..numberOfGroupPages]
        groupSkip = (groupPageNumber - 1) * GROUPPERPAGE
        groupOptions = {
          limit : GROUPPERPAGE,
          skip  : groupSkip
        }
        JGroup.some groupSelector, groupOptions, (err, jGroups)=>
          publicGroups = []

          # We behave 'koding' as a public group, even though
          # its privacy is marked as "private" in the model.
          publicGroups.push 'koding'

          for group in jGroups
            publicGroups.push group.slug


          selector = {
            'slugs.group': { $in: publicGroups }
          }

          JName.count selector, (err, count)=>
            numberOfNamePages = Math.ceil(count / NAMEPERPAGE)

            queue = [1..numberOfNamePages].map (pageNumber)=>=>
                skip = (pageNumber - 1) * NAMEPERPAGE
                option = {
                  limit : NAMEPERPAGE,
                  skip  : skip
                }
                JName.some selector, option, (err, names)=>
                  urls = (name.name for name in names) 

                  sitemapName =  @generateSitemapName skip
                  content = @generateSitemapString urls
                  @saveSitemap sitemapName, content

                  sitemapNames.push sitemapName + ".xml"
                  queue.next()
            queue.push => @generateMainSitemap sitemapNames

            daisy queue


  init:-><commit_msg>Fixed multiple fetching of koding group bug.<commit_after>{EventEmitter} = require 'events'

jraphical = require 'jraphical'
{Base, race, dash, daisy} = require "bongo"
{CronJob} = require 'cron'
_ = require "underscore"

NAMEPERPAGE = 50000
GROUPPERPAGE = 5
module.exports = class SitemapGeneratorWorker extends EventEmitter
  constructor: (@bongo, @options = {}) ->

  generateSitemapString: (urls)->
     # sitemap.XML beginning and ending parts
    sitemap = '<?xml version="1.0" encoding="UTF-8"?>
      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" 
      xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" 
      xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">'
    sitemapFooter = '</urlset>'

    for url in urls
      sitemap += "<url><loc>#{@options.uri.address}/#{url}</loc></url>"
    sitemap += sitemapFooter

  generateSitemapName: (skip)->
    return  skip + "_" + (skip + NAMEPERPAGE)

  saveSitemap: (name, content)->
    {JSitemap} = @bongo.models
    JSitemap.update {name}, $set : {content}, {upsert : yes}, (err)-> 
      console.log err if err
  generateMainSitemap: (sitemapNames)=>
    name = "main"
    content = @generateSitemapString sitemapNames
    @saveSitemap name, content
  generate:=>
    {JName, JGroup, JSitemap} = @bongo.models

    sitemapNames = []

    groupSelector = {
      privacy:'public'
    }

    JGroup.count groupSelector, (err, groupCount)=>
      numberOfGroupPages = Math.ceil(groupCount / GROUPPERPAGE)

      publicGroups = []

      # We behave 'koding' as a public group, even though
      # its privacy is marked as "private" in the model.
      # We add 'koding' group to publicGroups selector for the first cycle.
      publicGroups.push 'koding'

      for groupPageNumber in [1..numberOfGroupPages]
        groupSkip = (groupPageNumber - 1) * GROUPPERPAGE
        groupOptions = {
          limit : GROUPPERPAGE,
          skip  : groupSkip
        }
        JGroup.some groupSelector, groupOptions, (err, jGroups)=>

          for group in jGroups
            publicGroups.push group.slug


          selector = {
            'slugs.group': { $in: publicGroups }
          }

          # Empty the group, it'll be filled in next cycle.
          publicGroups = []

          JName.count selector, (err, count)=>
            numberOfNamePages = Math.ceil(count / NAMEPERPAGE)

            queue = [1..numberOfNamePages].map (pageNumber)=>=>
              skip = (pageNumber - 1) * NAMEPERPAGE
              option = {
                limit : NAMEPERPAGE,
                skip  : skip
              }
              JName.some selector, option, (err, names)=>
                urls = (name.name for name in names) 

                sitemapName =  @generateSitemapName skip
                content = @generateSitemapString urls
                @saveSitemap sitemapName, content

                sitemapNames.push sitemapName + ".xml"
                queue.next()
            queue.push => @generateMainSitemap sitemapNames

            daisy queue


  init:->
{'fraction_non_alphanumeric': 0.11612686692398054, 'fraction_numerical': 0.006376908877328411, 'mean_word_length': 2.2550518842162752, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 2, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6442495', 'n_tokens_mistral': 1897, 'n_tokens_neox': 1808, 'n_words': 424}