<commit_before>"use strict";
var express = require('express');
var app = express();


var server = app.listen( 5938, function(){
    console.log( 'Server started on port 5938' );
});

app.use( express.static('resources'));

app.get( '/', ( req, res ) =>{
    let options = {
        root : __dirname + '/resources'
    };
    res.sendFile( 'index.html', options);
});

app.get( '/files', ( req, res ) => {
    res.send( '<div>Found 4 files</div>' );
});

app.post( '/file/:filename/:sid/:tablename', (req, res ) => {

});<commit_msg>can list files from the data directory. no filtering yet<commit_after>"use strict";
var express = require('express');
var fs = require('fs');
var app = express();


var server = app.listen( 5938, function(){
    console.log( 'Server started on port 5938' );
});

app.use( express.static('resources'));

app.get( '/', ( req, res ) =>{
    let options = {
        root : __dirname + '/resources'
    };
    res.sendFile( 'index.html', options);
});

app.get( '/files', ( req, res ) => {
    let basePath = process.env.HOME_DIR + '/data';
    let getFilesInDirectory = function( path ){
        return fs.readdirSync( path ).map( (filename) => {
            if( fs.statSync( `${path}/${filename}`).isDirectory()){
                return getFilesInDirectory( `${path}/${filename}` );
            }else {
                return filename;
            }
        }).reduce( (a, b) => a.concat( b ), []);
    }

    res.json( getFilesInDirectory( basePath) );
    
});

app.post( '/file/:filename/:sid/:tablename', (req, res ) => {

});
{'fraction_non_alphanumeric': 0.20256410256410257, 'fraction_numerical': 0.010897435897435897, 'mean_word_length': 2.825980392156863, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6017985', 'n_tokens_mistral': 514, 'n_tokens_neox': 504, 'n_words': 126}