<commit_before>import express from 'express'
import helmet from 'helmet'
import expressStaticGzip from 'express-static-gzip'
import path from 'path'
import morgan from 'morgan'
import chokidar from 'chokidar'
import jsonfile from 'jsonfile'
import { ServerRenderer } from './utils/serverRender'
import { Scanner } from './utils/scanner'
import { DataGetter } from './utils/dataGetter'
import { DataHolder } from './utils/dataHolder'

const port = process.env.PORT || 3000
const app = express()
app.set('trust proxy', true)

const config = jsonfile.readFileSync(path.join(process.cwd(), 'config/config.json'))
if (config == null) {
  throw new Error('Config file not found!')
}

const dataHolder = new DataHolder()
const scanner = new Scanner(config, dataHolder)

const watcher = chokidar.watch(path.join(process.cwd(), 'content'), {
  ignored: /(^|[/\\])\../, // ignore dotfiles
  persistent: true
})

watcher
  .on('add', filepath => {
    console.log(`[Watcher] File ${filepath} has been added`)
    scanner.addFile(filepath)
  })
  .on('change', filepath => {
    console.log(`[Watcher] File ${filepath} has been changed`)
    scanner.updateFile(filepath)
  })
  .on('unlink', filepath => {
    console.log(`[Watcher] File ${filepath} has been removed`)
    scanner.deleteFile(filepath)
  })

app.use(morgan('common'))

app.use(helmet.contentSecurityPolicy({
  directives: {
    defaultSrc: ["'self'", `*.${config.baseUrl}`],
    scriptSrc: ["'self'", "'unsafe-inline'", "'unsafe-eval'", `*.${config.baseUrl}`],
    styleSrc: ["'self'", 'fonts.googleapis.com', 'fonts.gstatic.com', 'maxcdn.bootstrapcdn.com'],
    fontSrc: ["'self'", 'fonts.googleapis.com', 'fonts.gstatic.com', 'maxcdn.bootstrapcdn.com'],
    imgSrc: ['*'],
    workerSrc: false,
    blockAllMixedContent: true
  }
}))

app.use('/static', expressStaticGzip('public/static'))

app.get('/favicon.ico', (req, res) => {
  res.status(404).send('Not Found !!!')<commit_msg>Fix wrong return code for favicon<commit_after>import express from 'express'
import helmet from 'helmet'
import expressStaticGzip from 'express-static-gzip'
import path from 'path'
import morgan from 'morgan'
import chokidar from 'chokidar'
import jsonfile from 'jsonfile'
import { ServerRenderer } from './utils/serverRender'
import { Scanner } from './utils/scanner'
import { DataGetter } from './utils/dataGetter'
import { DataHolder } from './utils/dataHolder'

const port = process.env.PORT || 3000
const app = express()
app.set('trust proxy', true)

const config = jsonfile.readFileSync(path.join(process.cwd(), 'config/config.json'))
if (config == null) {
  throw new Error('Config file not found!')
}

const dataHolder = new DataHolder()
const scanner = new Scanner(config, dataHolder)

const watcher = chokidar.watch(path.join(process.cwd(), 'content'), {
  ignored: /(^|[/\\])\../, // ignore dotfiles
  persistent: true
})

watcher
  .on('add', filepath => {
    console.log(`[Watcher] File ${filepath} has been added`)
    scanner.addFile(filepath)
  })
  .on('change', filepath => {
    console.log(`[Watcher] File ${filepath} has been changed`)
    scanner.updateFile(filepath)
  })
  .on('unlink', filepath => {
    console.log(`[Watcher] File ${filepath} has been removed`)
    scanner.deleteFile(filepath)
  })

app.use(morgan('common'))

app.use(helmet.contentSecurityPolicy({
  directives: {
    defaultSrc: ["'self'", `*.${config.baseUrl}`],
    scriptSrc: ["'self'", "'unsafe-inline'", "'unsafe-eval'", `*.${config.baseUrl}`],
    styleSrc: ["'self'", 'fonts.googleapis.com', 'fonts.gstatic.com', 'maxcdn.bootstrapcdn.com'],
    fontSrc: ["'self'", 'fonts.googleapis.com', 'fonts.gstatic.com', 'maxcdn.bootstrapcdn.com'],
    imgSrc: ['*'],
    workerSrc: false,
    blockAllMixedContent: true
  }
}))

app.use('/static', expressStaticGzip('public/static'))

app.get('/favicon.ico', (req, res) => {
  res.status(204).send('Not Found !!!')
{'fraction_non_alphanumeric': 0.18172484599589322, 'fraction_numerical': 0.003593429158110883, 'mean_word_length': 4.664244186046512, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2793752', 'n_tokens_mistral': 1325, 'n_tokens_neox': 1304, 'n_words': 297}