<commit_before>import generator from './utils/page-generator'
import { path as rootPath } from 'app-root-path'
import * as path from 'path'
import * as ssr from './utils/ssr-renderer'
import config from 'config'
import themeRoot from '../build/theme-path'
import { search } from './utils/catalog-client'
import bodybuilder from 'bodybuilder'
const resolve = file => path.resolve(rootPath, file)

const compileOptions = {
  escape: /{{([^{][\s\S]+?[^}])}}/g,
  interpolate: /{{{([\s\S]+?)}}}/g
}
const templatesCache = ssr.initTemplatesCache(config, compileOptions)

// In production: create server renderer using server bundle and index HTML
// template from real fs.
// The server bundle is generated by vue-ssr-webpack-plugin.
const clientManifest = require(resolve('dist/vue-ssr-client-manifest.json'))
const bundle = require(resolve('dist/vue-ssr-bundle.json'))
// src/index.template.html is processed by html-webpack-plugin to inject
// build assets and output as dist/index.html.
// TODO: Add dynamic templates loading from (config based?) list
const renderer = ssr.createRenderer(bundle, clientManifest)
const destPath = resolve(config.staticPages.destPath)

generator.clearAll(destPath)
generator.saveScripts(resolve(''), destPath)
generator.saveSW(resolve(''), destPath)
generator.saveAssets(themeRoot, destPath)

/* search({
    size: 1000,
    from: 0,
    sort: 'id:desc',
    type: 'category',
    searchQuery: bodybuilder().query('term', 'is_active', true)
}, config, config).then(results => {
    if (results.hits) {
        results.hits.hits.forEach(category => {
            console.log(`Generating static page for category ${category._source.url_path}`)
            const urlToRender = category._source.url_path
            const res = { redirect: (url) => {}}
            const req = { url: urlToRender }
            const context = ssr.initSSRRequestContext(null, req, res, config)
            renderer.renderToString(context).then(output => {
                output = ssr.applyAdvancedOutputProcessing(context, output, templatesCache, true);
                generator.saveRenderedPage(path.join(destPath, urlToRender), output)
            }).catch(err => {
                console.error(err)
            })
        });
    }
}) */

search({
  size: 50,
  from: 0,
  sort: 'id:desc',
  type: 'product',
  searchQuery: bodybuilder().query('status', [1], true).andQuery('visibility', [3, 4])
}, config, config /* TODO: add support for different storeviews */).then(results => {
  if (results.hits) {
    results.hits.hits.forEach(category => {
      console.log(`Generating static page for product ${category._source.url_path}`)
      const urlToRender = category._source.url_path
      const res = { redirect: (url) => {} }
      const req = { url: urlToRender }
      const context = ssr.initSSRRequestContext(null, req, res, config)
      renderer.renderToString(context).then(output => {
        output = ssr.applyAdvancedOutputProcessing(context, output, templatesCache, true);
        generator.saveRenderedPage(path.join(destPath, urlToRender), output)
      }).catch(err => {
        console.error(err)
      })
    });
  }
})<commit_msg>Added: paging, categories, products<commit_after>import generator from './utils/page-generator'
import { path as rootPath } from 'app-root-path'
import * as path from 'path'
import * as ssr from './utils/ssr-renderer'
import config from 'config'
import themeRoot from '../build/theme-path'
import { search } from './utils/catalog-client'
import bodybuilder from 'bodybuilder'
const resolve = file => path.resolve(rootPath, file)
import program from 'commander'

const { renderer, templatesCache, destPath } = _prepareRenderer();

program
  .command('products')
  .option('-f|--from <from>', 'from - starting record', 0)
  .option('-s|--size <size>', 'size - batch size', 4)
  .action(async (cmd) => {

    const getProductsPage =  (from, size) => search({
      size: size,
      from: from,
      sort: 'id:desc',
      type: 'product',
      searchQuery: bodybuilder().filter('terms', 'visibility', [2, 3, 4]).andFilter('term', 'status', 1).build()
    }, config, config /* TODO: add support for different storeviews */)
    let pageFrom = parseInt(cmd.from)
    let pageSize = parseInt(cmd.size)

    _renderItems(getProductsPage, pageFrom, pageSize)
    
})

program
  .command('categories')
  .option('-f|--from <from>', 'from - starting record', 0)
  .option('-s|--size <size>', 'size - batch size', 4)
  .option('-t|--tag <tag>', 'tag name, available tags: ' + config.server.availableCacheTags.join(', '), '*')
  .action(async (cmd) => {

    const getCategoriesPage =  (from, size) => search({
      size: size,
      from: from,
      sort: 'id:desc',
      type: 'category',
      searchQuery: bodybuilder().filter('term', 'is_active', true).build()
    }, config, config /* TODO: add support for different storeviews */)
    let pageFrom = parseInt(cmd.from)
    let pageSize = parseInt(cmd.size)

    _renderItems(getCategoriesPage, pageFrom, pageSize)
    
})


program
  .command('cms')
  .option('-f|--from <from>', 'from - starting record', 0)
  .option('-s|--size <size>', 'size - batch size', 4)
  .option('-t|--tag <tag>', 'tag name, available tags: ' + config.server.availableCacheTags.join(', '), '*')
  .action(async (cmd) => {

    const getCmsPage =  (from, size) => search({
      size: size,
      from: from,
      sort: 'id:desc',
      type: 'cms_page',
      searchQuery: bodybuilder().build()
    }, config, config /* TODO: add support for different storeviews */).then(results => {
        if (results.hits && results.hits.hits.length > 0) {
            results.hits.hits.map(page => { page._source.url_path = `/i/${page._source.identifier}` })
        }
        return results
    })
    let pageFrom = parseInt(cmd.from)
    let pageSize = parseInt(cmd.size)

    _renderItems(getCmsPage, pageFrom, pageSize)
    
})

async function _renderItems(itemsSource, pageFrom, pageSize) {
    let recordsProcessed = 0
    let results = null
    do {
        results = await itemsSource(pageFrom, pageSize)
        console.log(`Processing records - pageSize: ${pageSize} from: ${pageFrom}`)
        if (results.hits && results.hits.hits.length > 0) {
            results.hits.hits.forEach(product => {
            console.log(`Generating static page for ${product._source.url_path}`)
            const urlToRender = product._source.url_path
            const res = { redirect: (url) => {} }
            const req = { url: urlToRender }
            const context = ssr.initSSRRequestContext(null, req, res, config)
            renderer.renderToString(context).then(output => {
                output = ssr.applyAdvancedOutputProcessing(context, output, templatesCache, true);
                generator.saveRenderedPage(path.join(destPath, urlToRender), output)
            }).catch(err => {
                console.error(`Error rendering product: ${product._source.name} - ${product._source.sku}: ${err}`)
            })
        });
        recordsProcessed += results.hits.hits.length
        } else {
            console.log(`Done! Total number of items processed: ${recordsProcessed}`)
        }
        pageFrom = pageFrom + pageSize
    } while (results.hits && results.hits.hits.length > 0)    
}

function _prepareRenderer() {
    const compileOptions = {
        escape: /{{([^{][\s\S]+?[^}])}}/g,
        interpolate: /{{{([\s\S]+?)}}}/g
    };
    const templatesCache = ssr.initTemplatesCache(config, compileOptions);
    // In production: create server renderer using server bundle and index HTML
    // template from real fs.
    // The server bundle is generated by vue-ssr-webpack-plugin.
    const clientManifest = require(resolve('dist/vue-ssr-client-manifest.json'));
    const bundle = require(resolve('dist/vue-ssr-bundle.json'));
    // src/index.template.html is processed by html-webpack-plugin to inject
    // build assets and output as dist/index.html.
    // TODO: Add dynamic templates loading from (config based?) list
    const renderer = ssr.createRenderer(bundle, clientManifest);
    const destPath = resolve(config.staticPages.destPath);
    generator.clearAll(destPath);
    generator.saveScripts(resolve(''), destPath);
    generator.saveSW(resolve(''), destPath);
    generator.saveAssets(themeRoot, destPath);
    return { renderer, templatesCache, destPath };
}

program.parse(process.argv)
{'fraction_non_alphanumeric': 0.1481525484139242, 'fraction_numerical': 0.0029701793988356896, 'mean_word_length': 3.2132132132132134, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3793917', 'n_tokens_mistral': 2533, 'n_tokens_neox': 2499, 'n_words': 620}