<commit_before>/*
 * fileutil.js
 *
 * file-related utility functions
 */

"use strict";

const fs = require('fs');
const path = require('path');

import * as config from './config';

export class FileUtil {

    public static expandPath(filepath: string): string {
        let idx: number = filepath.indexOf('~');
        return idx === 0 ? process.env.HOME + filepath.substring(1) : filepath;
    }

    public static getExtension(filepath: string): string {
        let f: string = path.basename(filepath);
        let idx: number = f.lastIndexOf('.');
        if (idx > 0 && idx < f.length - 1) {
            return f.substring(idx + 1);
        }
        return '';
    }

    public static getFileContents(filepath: string): string {
        return fs.readFileSync(filepath).toString();
    }

    public static getFileContentsAsync(filepath: string, cb): void {
        cb(fs.readFileSync(filepath).toString());
    }

    public static getFileLines(filepath: string): string[] {
        return FileUtil.getFileContents(filepath).split(/\r?\n/);
    }

    public static getFileLinesAsync(filepath: string, cb): void {
        cb(FileUtil.getFileContents(filepath).split(/\r?\n/));
    }

    public static getRelativePath(filepath: string, startpath: string): string {
        if (startpath === '.' && filepath.startsWith(process.env.HOME)) {
            return '.' + filepath.substring(process.env.HOME.length);
        }
    }

    public static isDotDir(filepath: string): boolean {
        return ['.', '..'].indexOf(filepath) > -1;<commit_msg>Add encoding param to file content retrieval methods, get HOME from config<commit_after>/*
 * fileutil.js
 *
 * file-related utility functions
 */

"use strict";

const fs = require('fs');
const path = require('path');

import * as config from './config';

export class FileUtil {

    public static expandPath(filepath: string): string {
        if (filepath[0] === "~") {
            return config.HOME + filepath.substring(1);
        }
        return filepath;
    }

    public static getExtension(filepath: string): string {
        let f: string = path.basename(filepath);
        let idx: number = f.lastIndexOf('.');
        if (idx > 0 && idx < f.length - 1) {
            return f.substring(idx + 1);
        }
        return '';
    }

    public static getFileContents(filepath: string, encoding: string): string {
        return fs.readFileSync(filepath, encoding).toString();
    }

    public static getFileContentsAsync(filepath: string, cb: (contents: string) => void): void {
        cb(fs.readFileSync(filepath).toString());
    }

    public static getFileLines(filepath: string, encoding: string): string[] {
        return FileUtil.getFileContents(filepath, encoding).split(/\r?\n/);
    }

    public static getFileLinesAsync(filepath: string, encoding: string, cb: (lines: string[]) => void): void {
        cb(FileUtil.getFileContents(filepath, encoding).split(/\r?\n/));
    }

    public static getRelativePath(filepath: string, startpath: string): string {
        if (startpath === '.' && filepath.startsWith(config.HOME)) {
            return '.' + filepath.substring(config.HOME.length);
        }
        return filepath;
    }

    public static isDotDir(filepath: string): boolean {
        return ['.', '..'].indexOf(filepath) > -1;
{'fraction_non_alphanumeric': 0.14290012033694344, 'fraction_numerical': 0.0036101083032490976, 'mean_word_length': 2.8043478260869565, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2870294', 'n_tokens_mistral': 985, 'n_tokens_neox': 961, 'n_words': 238}