<commit_before>import * as Path from 'path';
import { exec } from './sh';
import * as Config from './config';
import * as Shell from 'shelljs';
import * as fs from 'fs';

export const repositoryDir = Path.resolve(Path.join(__dirname, '..', 'repositories'));

const githubUrlRegexp = /^(git@github\.com:|https:\/\/github\.com\/).+$/i

export function addRepository(url: string): boolean {
    if (!url || !githubUrlRegexp.test(url)) {
        return false;
    }
    const httpsUrl = url.replace('<EMAIL>:', 'https://github.com/')
    const relPath = httpsUrl.replace(/\.git$/, '').split('/').slice(3);
    const dir = Path.join(repositoryDir, ...relPath);
    const path = relPath.join('/');

    addToGit(dir, httpsUrl);
    Config.addRepository(path, httpsUrl);

    return true;
}

export function updateRepository(path: string | undefined = '', useRemote: boolean = false): boolean {
    path = path.replace(/^repositories/, '');

    if (!path || !fs.existsSync(Path.join(repositoryDir, path))) {
        if (path) {
            console.log(`Repository: '${path}' not found.`)
        }
        return false;
    }
    const remote = useRemote ? '--remote' : ''
    Shell.pushd(repositoryDir)
    exec(`git submodule update --depth 1 ${remote} -- ${JSON.stringify(path)}`, { echo: true, bail: true });
    Shell.popd()

    return true;
}

export function listRepositories(pattern: string = '') {
    const config = Config.readConfig();
    config.repositories
    .filter(rep => rep.path.includes(pattern))
    .forEach(rep => {
        console.log(rep.path);
    });
}

function addToGit(dir: string, url: string) {
    const p = Path.dirname(dir);
    Shell.mkdir('-p', p);
    Shell.pushd(p);
    exec(`git submodule add --depth 1 ${JSON.stringify(url)}`, { echo: true, bail: true });
    Shell.popd();
}
<commit_msg>Make sure the submodule is correctly initialized. (#434)<commit_after>import * as Path from 'path';
import { exec } from './sh';
import * as Config from './config';
import * as Shell from 'shelljs';
import * as fs from 'fs';

export const repositoryDir = Path.resolve(Path.join(__dirname, '..', 'repositories'));

const githubUrlRegexp = /^(git@github\.com:|https:\/\/github\.com\/).+$/i

export function addRepository(url: string): boolean {
    if (!url || !githubUrlRegexp.test(url)) {
        return false;
    }
    const httpsUrl = url.replace('<EMAIL>:', 'https://github.com/')
    const relPath = httpsUrl.replace(/\.git$/, '').split('/').slice(3);
    const dir = Path.join(repositoryDir, ...relPath);
    const path = relPath.join('/');

    addToGit(dir, httpsUrl);
    Config.addRepository(path, httpsUrl);

    return true;
}

export function updateRepository(path: string | undefined = '', useRemote: boolean = false): boolean {
    path = path.replace(/^repositories/, '');

    if (!path || !fs.existsSync(Path.join(repositoryDir, path))) {
        if (path) {
            console.log(`Repository: '${path}' not found.`)
        }
        return false;
    }
    const remote = useRemote ? '--remote' : ''
    const init = useRemote ? '' : '--init';
    Shell.pushd(repositoryDir)
    exec(`git submodule update --depth 1 ${remote} ${init} -- ${JSON.stringify(path)}`, { echo: true, bail: true });
    Shell.popd()

    return true;
}

export function listRepositories(pattern: string = '') {
    const config = Config.readConfig();
    config.repositories
    .filter(rep => rep.path.includes(pattern))
    .forEach(rep => {
        console.log(rep.path);
    });
}

function addToGit(dir: string, url: string) {
    const p = Path.dirname(dir);
    Shell.mkdir('-p', p);
    Shell.pushd(p);
    exec(`git submodule add --depth 1 ${JSON.stringify(url)}`, { echo: true, bail: true });
    Shell.popd();
}

{'fraction_non_alphanumeric': 0.1806778756338404, 'fraction_numerical': 0.0024019215372297837, 'mean_word_length': 3.192393736017897, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4115234', 'n_tokens_mistral': 1196, 'n_tokens_neox': 1157, 'n_words': 290}