<commit_before>_ = require('lodash')
fs = require('fs')
path = require('path')
child_process = require('child_process')

ipythonDataDir = ->
    child_process.spawnSync('ipython',['locate']).stdout.toString().replace '\n',''

jupyterResolve = _.memoize (category) ->
    match = child_process.spawnSync('jupyter', ['--paths']).stdout.toString()
    _.filter (_.map (
        ///#{category}:\n(\s+.*\n)+///gim.exec(match)[0].split('\n')
    ), (e) -> e.replace(///#{category}:///,'').trim()), _.negate(_.isEmpty)


jupyterPath = _.memoize (subdirs...) ->
    paths = _.map process.env['JUPYTER_PATH']?.split(path.delimiter), (x) ->
        x.replace ///#{path.sep}+$///g, ""
    paths.push ipythonDataDir()
    paths.push p for p in jupyterResolve 'data'

    paths = (path.join(p, subdirs...) for p in paths)
    _.filter paths, (x) -> try fs.statSync(x).isDirectory() catch e

module.exports =
  jupyterResolve: jupyterResolve
  jupyterPath: jupyterPath
<commit_msg>Available kernels func<commit_after>_ = require('lodash')
fs = require('fs')
path = require('path')
child_process = require('child_process')

ipythonDataDir = ->
    child_process.spawnSync('ipython',['locate']).stdout.toString().replace '\n',''

jupyterResolve = _.memoize (category) ->
    match = child_process.spawnSync('jupyter', ['--paths']).stdout.toString()
    _.map ///#{category}:\n(\s+.*\n)+///gim.exec(match)[0].split('\n')[1...-1], (x) -> x.trim()


jupyterPath = _.memoize (subdirs...) ->
    paths = _.map process.env['JUPYTER_PATH']?.split(path.delimiter), (x) ->
        x.replace ///#{path.sep}+$///g, ""
    paths.push ipythonDataDir()
    paths.push p for p in jupyterResolve 'data'

    paths = (path.join(p, subdirs...) for p in paths)
    _.filter paths, (x) -> try fs.statSync(x).isDirectory() catch e

kernels = _.memoize ->
    stdout =  child_process.spawnSync('ipython',['kernelspec','list']).stdout.toString()
    _.map /Available kernels:(\s+.+\s+.+\n)+/.exec(stdout)[0].split('\n')[1...-1], (x) ->
        /\/.*/.exec(x)[0]

module.exports =
  jupyterResolve: jupyterResolve
  jupyterPath: jupyterPath
  availableKernels: kernels

{'fraction_non_alphanumeric': 0.2207609206200094, 'fraction_numerical': 0.0037576326914044154, 'mean_word_length': 3.600431965442765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1497872', 'n_tokens_mistral': 798, 'n_tokens_neox': 797, 'n_words': 131}