<commit_before>'use strict';

const globby = require('globby');
const Promise = require('bluebird');
const fs = require('fs');
const path = require('path');
const express = require('express');
const app = express();
const adaro = require('adaro');

const htmlFiles = {
  // Used in getContent() function
  all: [
    'consumables/html/**/*.html',
    '!consumables/html/**/body.html',
    '!consumables/html/**/header.html',
    '!consumables/html/components/detail-page-header/**/*.html',
  ],
  header: 'consumables/html/**/header.html',
  detailHeader: 'consumables/html/components/detail-page-header/**/*.html',
  baseElements: 'consumables/html/base-elements/**/*.html',
  components: 'consumables/html/components/**/*.html',
  global: 'consumables/html/global/**/*.html'
};

const directoryOrder = [
  // Used for allLinks() function
  'consumables/html/base-elements/**/*.html',
  'consumables/html/components/**/*.html',
  'consumables/html/global/**/*.html'
];

app.engine('dust', adaro.dust());
app.set('view engine', 'dust');
app.set('views', path.resolve(__dirname, 'demo/views'));
app.use(express.static('demo'));
app.use(express.static('consumables'));

const getContent = (glob) => {
  return globby(glob)
    .then(paths => {
      return paths
        .map(file => {
          return fs.readFileSync(file, { 'encoding': 'utf8' });
        })
        .reduce((a, b) => {
          return a.concat(b);
        });
    });
}

const allLinks = globby(directoryOrder)
  .then(paths => {
    return paths
      .map(path => {
        const indices = [];
        for (let i = 0; i < path.length; i++) {
          if (path[i] === '/') {
            indices.push(i);
          }
        }

        return path.slice(0, indices[1]);
      })
      .filter((path, index, paths) => {
        return paths.indexOf(path) === index;
      })
      .map(path => {
        return {
          url: path
        }
      })
  });


app.get('/', (req, res) => {
  Promise.all([
    getContent(htmlFiles.header),
    getContent(htmlFiles.detailHeader),
    getContent(htmlFiles.all)
  ])
  .then(results => {
    res.render('demo-all', {
      header: results[0],
      detailHeader: results[1],
      html: results[2]
    });
  });
});

app.get('/components/:component', (req, res) => {
  const glob = `consumables/html/components/${req.params.component}/**/*.html`;

  Promise.all([getContent(glob), allLinks])
    .then(results => {
      res.render('demo-all', {
        html: results[0],
        links: results[1]
      });
    });
});

app.get('/base-elements/:component', (req, res) => {
  const glob = `consumables/html/base-elements/${req.params.component}/**/*.html`;

  Promise.all([getContent(glob), allLinks])
    .then(results => {
      res.render('demo-all', {
        html: results[0],
        links: results[1]
      });
    });
});

app.get('/global/:component', (req, res) => {
  const glob = `consumables/html/global/${req.params.component}/**/*.html`;

  Promise.all([getContent(glob), allLinks])
    .then(results => {
      res.render('demo-all', {
        html: results[0],
        links: results[1]
      });
    });
});

app.listen(8080);
<commit_msg>Return 404 for content not found, 500 for erros thrown in promise, fixes #1450. (#1451)<commit_after>'use strict';

const globby = require('globby');
const Promise = require('bluebird');
const fs = require('fs');
const path = require('path');
const express = require('express');
const app = express();
const adaro = require('adaro');

const htmlFiles = {
  // Used in getContent() function
  all: [
    'consumables/html/**/*.html',
    '!consumables/html/**/body.html',
    '!consumables/html/**/header.html',
    '!consumables/html/components/detail-page-header/**/*.html',
  ],
  header: 'consumables/html/**/header.html',
  detailHeader: 'consumables/html/components/detail-page-header/**/*.html',
  baseElements: 'consumables/html/base-elements/**/*.html',
  components: 'consumables/html/components/**/*.html',
  global: 'consumables/html/global/**/*.html'
};

const directoryOrder = [
  // Used for allLinks() function
  'consumables/html/base-elements/**/*.html',
  'consumables/html/components/**/*.html',
  'consumables/html/global/**/*.html'
];

app.engine('dust', adaro.dust());
app.set('view engine', 'dust');
app.set('views', path.resolve(__dirname, 'demo/views'));
app.use(express.static('demo'));
app.use(express.static('consumables'));

const getContent = (glob) => {
  return globby(glob)
    .then(paths => {
      return paths.length === 0 ? undefined : paths
        .map(file => {
          return fs.readFileSync(file, { 'encoding': 'utf8' });
        })
        .reduce((a, b) => {
          return a.concat(b);
        });
    });
}

const allLinks = globby(directoryOrder)
  .then(paths => {
    return paths
      .map(path => {
        const indices = [];
        for (let i = 0; i < path.length; i++) {
          if (path[i] === '/') {
            indices.push(i);
          }
        }

        return path.slice(0, indices[1]);
      })
      .filter((path, index, paths) => {
        return paths.indexOf(path) === index;
      })
      .map(path => {
        return {
          url: path
        }
      })
  });


app.get('/', (req, res) => {
  Promise.all([
    getContent(htmlFiles.header),
    getContent(htmlFiles.detailHeader),
    getContent(htmlFiles.all)
  ])
  .then(results => {
    res.render('demo-all', {
      header: results[0],
      detailHeader: results[1],
      html: results[2]
    });
  })
  .catch((error) => {
    console.error(error.stack); // eslint-disable-line no-console
    res.status(500).end();
  });
});

app.get('/components/:component', (req, res) => {
  const glob = `consumables/html/components/${req.params.component}/**/*.html`;

  Promise.all([getContent(glob), allLinks])
    .then(results => {
      if (typeof results[0] === 'undefined') {
        res.status(404).end();
      } else {
        res.render('demo-all', {
          html: results[0],
          links: results[1]
        });
      }
    })
    .catch((error) => {
      console.error(error.stack); // eslint-disable-line no-console
      res.status(500).end();
    });
});

app.get('/base-elements/:component', (req, res) => {
  const glob = `consumables/html/base-elements/${req.params.component}/**/*.html`;

  Promise.all([getContent(glob), allLinks])
    .then(results => {
      if (typeof results[0] === 'undefined') {
        res.status(404).end();
      } else {
        res.render('demo-all', {
          html: results[0],
          links: results[1]
        });
      }
    })
    .catch((error) => {
      console.error(error.stack); // eslint-disable-line no-console
      res.status(500).end();
    });
});

app.get('/global/:component', (req, res) => {
  const glob = `consumables/html/global/${req.params.component}/**/*.html`;

  Promise.all([getContent(glob), allLinks])
    .then(results => {
      if (typeof results[0] === 'undefined') {
        res.status(404).end();
      } else {
        res.render('demo-all', {
          html: results[0],
          links: results[1]
        });
      }
    })
    .catch((error) => {
      console.error(error.stack); // eslint-disable-line no-console
      res.status(500).end();
    });
});

app.listen(8080);

{'fraction_non_alphanumeric': 0.2057935200439319, 'fraction_numerical': 0.010021965952773201, 'mean_word_length': 2.638861138861139, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4344640', 'n_tokens_mistral': 2411, 'n_tokens_neox': 2370, 'n_words': 395}