<commit_before>// - stripping out handles
// - stripping out styles
// - stripping out frontend js
// - fixing paths in all of those different embedded blocks
// - creating distinct bundles for styles and frontend js
// - reading beard templates (without blocks) into memory cache
// - scoping css on scoped styles block
// --- TODO
// - prune unused styles
// - prune unused js


const fs = require('fs');
const fse = require('fs-extra');
const traversy = require('traversy');
const { basename, extname, resolve, dirname, relative } = require('path');
const { cleanWhitespace, hash } = require('./utils');
const cheerio = require('cheerio');
const XRegExp = require('xregexp');
const exts = '(.beard$)';
const regex = new RegExp('(.beard$)', 'g');
const normalizeSelector = require('normalize-selector');

const blockTypes = {
  ssjs: {
    type: 'ssjs',
    tag: 'script[handle]',
    pathsRegex: /(import|require)[^'"`]+['"`]([\.\/][^'"`]+)['"`]/gmi,
    ext: 'ssjs.js'
  },
  css: {
    type: 'css',
    tag: 'style',
    pathsRegex: /(@import|url)\s*["'\(]*([^'"\)]+)/gmi,
    importStatement: (path) => `@import './${path}';`,
    ext: 'scss'
  },
  js: {
    type: 'js',
    tag: 'script:not(script[handle]):not(script[inline]):not(script[src])',
    pathsRegex: /(import|require)[^'"`]+['"`]([\.\/][^'"`]+)['"`]/gmi,
    importStatement: (path) => `import './${path}';`,
    ext: 'js'
  }
};

const combinators = ['>', '+', '~'];
const deepCombinator = '>>>'; // this is our custom deep combinator for decendant scoping
const psuedoElements = /::after|:after|::backdrop|::after|:after|::backdrop|::before|:before|::cue|:cue|::first-letter|:first-letter|::first-line|:first-line|::grammar-error|::marker|::part\(.*?\)|::placeholder|::selection|::slotted\(.*?\)|::spelling-error/;

let root;
let beardDir;
const handles = {};
const templates = {};


exports.bundle = (opts) => {
  root = opts.root;
  beardDir = `${root}/../.beard`;

  fse.ensureDirSync(beardDir);

  blockTypes.css.bundles = {
    entry: []
  };

  blockTypes.js.bundles = {
    entry: []
  };

  traversy(root, exts, (path) => {
    const key = path.replace(regex, '');
    const contents = fs.readFileSync(path, 'utf8');
    const template = /<template>.*?<\/template>/.test(contents)
      ? contents
      : `<template>${contents}</template>`;

    const $ = cheerio.load(template, {
      withDomLvl1: false,
      normalizeWhitespace: false,
      xmlMode: false,
      decodeEntities: false,
      lowerCaseAttributeNames: false
    });

    const blocks = parseBlocks($, path);
    const body = $('template').html().replace(/=\"=(=?)\"/gm, '==$1');

    writeBlockFiles(blocks);

    if (process.env.NODE_ENV !== 'production') {
      fs.writeFileSync(`${beardDir}/${getHashedPath(path, 'beard')}`, body);
    }

    if (opts.loadHandles && blocks.ssjs) {
      handles[key] = require(`${beardDir}/${blocks.ssjs.file}`);
    }

    templates[key] = cleanWhitespace(body);
  });

  writeEntryFile('css');
  writeEntryFile('js');

  return {
    templates,
    handles
  };
}


function parseBlocks($, path) {
  const blocks = extractBlocks($, path);

  Object.entries(blocks).forEach(([type, block]) => {
    const blockType = blockTypes[type];
    const { importStatement, ext, pathsRegex } = blockType;

    if (block.hasOwnProperty('scoped')) {
      block.content = scopeCSS(path, block.content, $);
    }

    block.content = fixPaths(path, block.content, pathsRegex);
    block.file = getHashedPath(path, block.lang || ext);

    let { bundle } = block;

    if (type !== 'ssjs') {
      if (!bundle) {
        bundle = 'entry';
      }

      if (!blockType.bundles[bundle]) {
        blockType.bundles[bundle] = [];
      }

      blockType.bundles[bundle].push(importStatement(block.file));
    }
  });

  return blocks;
}


function extractBlocks($) {
  const blocks = {};

  Object.entries(blockTypes).forEach(([type, blockType]) => {
    const { tag } = blockType;

    $(tag).each((i, el) => {
      const block = { ...{ content: $(el).get()[0].children[0].data }, ...el.attribs };
      blocks[type] = block;
      $(el).remove();
    });
  });

  return blocks;
}

function writeBlockFiles(blocks) {
  Object.entries(blocks).forEach(([key, block]) => {
    fs.writeFileSync(`${beardDir}/${block.file}`, block.content);<commit_msg>Add support for multi-bundle css and js blocks<commit_after>// - stripping out handles
// - stripping out styles
// - stripping out frontend js
// - fixing paths in all of those different embedded blocks
// - creating distinct bundles for styles and frontend js
// - reading beard templates (without blocks) into memory cache
// - scoping css on scoped styles block
// --- TODO
// - prune unused styles
// - prune unused js


const fs = require('fs');
const fse = require('fs-extra');
const traversy = require('traversy');
const { basename, extname, resolve, dirname, relative } = require('path');
const { cleanWhitespace, hash } = require('./utils');
const cheerio = require('cheerio');
const XRegExp = require('xregexp');
const exts = '(.beard$)';
const regex = new RegExp('(.beard$)', 'g');
const normalizeSelector = require('normalize-selector');

const blockTypes = {
  ssjs: {
    type: 'ssjs',
    tag: 'script[handle]',
    pathsRegex: /(import|require)[^'"`]+['"`]([\.\/][^'"`]+)['"`]/gmi,
    ext: 'ssjs.js'
  },
  css: {
    type: 'css',
    tag: 'style',
    pathsRegex: /(@import|url)\s*["'\(]*([^'"\)]+)/gmi,
    importStatement: (path) => `@import './${path}';`,
    ext: 'scss'
  },
  js: {
    type: 'js',
    tag: 'script:not(script[handle]):not(script[inline]):not(script[src])',
    pathsRegex: /(import|require)[^'"`]+['"`]([\.\/][^'"`]+)['"`]/gmi,
    importStatement: (path) => `import './${path}';`,
    ext: 'js'
  }
};

const combinators = ['>', '+', '~'];
const deepCombinator = '>>>'; // this is our custom deep combinator for decendant scoping
const psuedoElements = /::after|:after|::backdrop|::after|:after|::backdrop|::before|:before|::cue|:cue|::first-letter|:first-letter|::first-line|:first-line|::grammar-error|::marker|::part\(.*?\)|::placeholder|::selection|::slotted\(.*?\)|::spelling-error/;

let root;
let beardDir;
const handles = {};
const templates = {};


exports.bundle = (opts) => {
  root = opts.root;
  beardDir = `${root}/../.beard`;

  fse.ensureDirSync(beardDir);

  blockTypes.css.bundles = {
    entry: []
  };

  blockTypes.js.bundles = {
    entry: []
  };

  traversy(root, exts, (path) => {
    const key = path.replace(regex, '');
    const contents = fs.readFileSync(path, 'utf8');
    const template = /<template>.*?<\/template>/.test(contents)
      ? contents
      : `<template>${contents}</template>`;

    const $ = cheerio.load(template, {
      withDomLvl1: false,
      normalizeWhitespace: false,
      xmlMode: false,
      decodeEntities: false,
      lowerCaseAttributeNames: false
    });

    const blocks = parseBlocks($, path);
    const body = $('template').html().replace(/=\"=(=?)\"/gm, '==$1');

    writeBlockFiles(blocks);

    if (process.env.NODE_ENV !== 'production') {
      fs.writeFileSync(`${beardDir}/${getHashedPath(path, 'beard')}`, body);
    }

    if (opts.loadHandles && blocks.ssjs) {
      handles[key] = require(`${beardDir}/${blocks.ssjs.file}`);
    }

    templates[key] = cleanWhitespace(body);
  });

  writeEntryFile('css');
  writeEntryFile('js');

  return {
    templates,
    handles
  };
}


function parseBlocks($, path) {
  const blocks = extractBlocks($, path);

  Object.entries(blocks).forEach(([type, block]) => {
    const blockType = blockTypes[type];
    const { importStatement, ext, pathsRegex } = blockType;

    if (block.hasOwnProperty('scoped')) {
      block.content = scopeCSS(path, block.content, $);
    }

    block.content = fixPaths(path, block.content, pathsRegex);
    block.file = getHashedPath(path, block.lang || ext);

    let { bundle } = block;

    if (type !== 'ssjs') {
      if (!bundle) {
        bundle = ['entry'];
      } else {
        bundle = bundle.split(',').filter(b => b).map(b => b.trim());
      }

      bundle.forEach((b) => {
        if (!blockType.bundles[b]) {
          blockType.bundles[b] = [];
        }

        blockType.bundles[b].push(importStatement(block.file));
      });
      // if (!blockType.bundles[bundle]) {
      //   blockType.bundles[bundle] = [];
      // }

      // blockType.bundles[bundle].push(importStatement(block.file));
    }
  });

  return blocks;
}


function extractBlocks($) {
  const blocks = {};

  Object.entries(blockTypes).forEach(([type, blockType]) => {
    const { tag } = blockType;

    $(tag).each((i, el) => {
      const block = { ...{ content: $(el).get()[0].children[0].data }, ...el.attribs };
      blocks[type] = block;
      $(el).remove();
    });
  });

  return blocks;
}

function writeBlockFiles(blocks) {
  Object.entries(blocks).forEach(([key, block]) => {
    fs.writeFileSync(`${beardDir}/${block.file}`, block.content);
{'fraction_non_alphanumeric': 0.209405995765073, 'fraction_numerical': 0.0011144544745347153, 'mean_word_length': 3.3732943469785575, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5952656', 'n_tokens_mistral': 3059, 'n_tokens_neox': 2979, 'n_words': 586}