<commit_before>const path = require('path')
const debug = require('debug')
const log = debug('app:provider:file')
const { promisify } = require('util')
const fs = require('fs')
const stat = promisify(fs.stat)
const mp3duration = promisify(require('mp3-duration'))
const getFiles = require('./lib/getFiles')
const Scanner = require('../Scanner')
const Media = require('../../Media')
const parseMeta = require('../../lib/parseMeta')
const parseMetaCfg = require('./lib/parseMetaCfg') // look for .js, .json

const mediaExts = ['.cdg', '.mp4', '.m4v']
const mediaExtPerms = mediaExts.reduce((perms, ext) => perms.concat(getPerms(ext)), [])

const audioExts = ['.m4a', '.mp3']
const audioExtPerms = audioExts.reduce((perms, ext) => perms.concat(getPerms(ext)), [])

class FileScanner extends Scanner {
  constructor (prefs) {
    super()
    this.paths = prefs.paths || []
  }

  async scan () {
    const offlinePaths = []
    const validIds = [] // mediaIds for cleanup
    let files = []

    // emit start
    this.emitStatus('Gathering file list', 0)

    // count files to scan from all paths
    for (const p of this.paths) {
      try {
        log('Searching path: %s', p)
        let list = await getFiles(p)
        list = list.filter(file => mediaExtPerms.includes(path.extname(file)))

        log('  => found %s files with valid extensions (%s)', list.length, mediaExts.join(', '))
        files = files.concat(list)
      } catch (err) {
        log(`  => ${err.message} (path offline)`)
        offlinePaths.push(p)
      }

      if (this.isCanceling) {
        return
      }
    }

    log('Processing %s total files', files.length)

    // process files
    for (let i = 0; i < files.length; i++) {
      const file = files[i]
      const basePath = this.paths.find(p => file.indexOf(p) === 0)

      // emit progress
      log('[%s/%s] %s', i + 1, files.length, file)
      this.emitStatus(`Scanning media files (${i + 1} of ${files.length})`, ((i + 1) / files.length) * 100)

      try {
        const mediaId = await this.processFile({ file, basePath })

        // successfuly processed
        validIds.push(mediaId)
      } catch (err) {
        log(err)
        // just try the next file...
      }

      if (this.isCanceling) {
        return
      }
    } // end for

    // cleanup
    log('Looking for orphaned media entries')

    try {
      const invalidIds = []
      const res = await Media.searchMedia({
        provider: 'file',
      })

      res.result.forEach(id => {
        // was this media item just verified?
        if (validIds.includes(id)) {
          return
        }

        // is media item in an offline path?
        if (offlinePaths.includes(res.entities[id].providerData.basePath)) {
          return
        }

        // looks like we need to remove it
        invalidIds.push(id)
        log(`  => ${res.entities[id].providerData.basePath}${res.entities[id].providerData.relPath}`)
      })

      log(`Removing ${invalidIds.length} orphaned media entries`)
<commit_msg>Minor log tweak<commit_after>const path = require('path')
const debug = require('debug')
const log = debug('app:provider:file')
const { promisify } = require('util')
const fs = require('fs')
const stat = promisify(fs.stat)
const mp3duration = promisify(require('mp3-duration'))
const getFiles = require('./lib/getFiles')
const Scanner = require('../Scanner')
const Media = require('../../Media')
const parseMeta = require('../../lib/parseMeta')
const parseMetaCfg = require('./lib/parseMetaCfg') // look for .js, .json

const mediaExts = ['.cdg', '.mp4', '.m4v']
const mediaExtPerms = mediaExts.reduce((perms, ext) => perms.concat(getPerms(ext)), [])

const audioExts = ['.m4a', '.mp3']
const audioExtPerms = audioExts.reduce((perms, ext) => perms.concat(getPerms(ext)), [])

class FileScanner extends Scanner {
  constructor (prefs) {
    super()
    this.paths = prefs.paths || []
  }

  async scan () {
    const offlinePaths = []
    const validIds = [] // mediaIds for cleanup
    let files = []

    // emit start
    this.emitStatus('Gathering file list', 0)

    // count files to scan from all paths
    for (const p of this.paths) {
      try {
        log('Searching path: %s', p)
        let list = await getFiles(p)
        list = list.filter(file => mediaExtPerms.includes(path.extname(file)))

        log('  => found %s files with valid extensions (%s)', list.length, mediaExts.join(', '))
        files = files.concat(list)
      } catch (err) {
        log(`  => ${err.message} (path offline)`)
        offlinePaths.push(p)
      }

      if (this.isCanceling) {
        return
      }
    }

    log('Processing %s total files', files.length)

    // process files
    for (let i = 0; i < files.length; i++) {
      const file = files[i]
      const basePath = this.paths.find(p => file.indexOf(p) === 0)

      // emit progress
      log('[%s/%s] %s', i + 1, files.length, file)
      this.emitStatus(`Scanning media files (${i + 1} of ${files.length})`, ((i + 1) / files.length) * 100)

      try {
        const mediaId = await this.processFile({ file, basePath })

        // successfuly processed
        validIds.push(mediaId)
      } catch (err) {
        log(err)
        // just try the next file...
      }

      if (this.isCanceling) {
        return
      }
    } // end for

    // cleanup
    log('Looking for orphaned media entries')

    try {
      const invalidIds = []
      const res = await Media.searchMedia({
        provider: 'file',
      })

      res.result.forEach(id => {
        // was this media item just verified?
        if (validIds.includes(id)) {
          return
        }

        // is media item in an offline path?
        if (offlinePaths.includes(res.entities[id].providerData.basePath)) {
          return
        }

        // looks like we need to remove it
        invalidIds.push(id)
        log(`  => ${res.entities[id].providerData.basePath}${res.entities[id].providerData.relPath}`)
      })

      log(`Found ${invalidIds.length} orphaned media entries`)

{'fraction_non_alphanumeric': 0.15354135710747896, 'fraction_numerical': 0.004952947003467063, 'mean_word_length': 2.4168076706147774, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1671092', 'n_tokens_mistral': 1930, 'n_tokens_neox': 1872, 'n_words': 492}