<commit_before>/*++

Module Name:

    GenomeIndex.cpp

Abstract:

    Index (hash table) builder for the SNAP sequencer

Authors:

    <NAME>, August, 2011

Environment:

    User mode service.

Revision History:

    Adapted from Matei Zaharia's Scala implementation.

--*/

#include "stdafx.h"
#include "ApproximateCounter.h"
#include "BigAlloc.h"
#include "Compat.h"
#include "FASTA.h"
#include "FixedSizeSet.h"
#include "FixedSizeVector.h"
#include "GenericFile.h"
#include "GenericFile_stdio.h"
#include "Genome.h"
#include "GenomeIndex.h"
#include "HashTable.h"
#include "Seed.h"
#include "exit.h"
#include "Error.h"
#include "directions.h"

using namespace std;

static const int DEFAULT_SEED_SIZE = 27;
static const double DEFAULT_SLACK = 0.3;
static const unsigned DEFAULT_PADDING = 500;

const char *GenomeIndexFileName = "GenomeIndex";
const char *OverflowTableFileName = "OverflowTable";
const char *GenomeIndexHashFileName = "GenomeIndexHash";
const char *GenomeFileName = "Genome";

static void usage()
{
    WriteErrorMessage(
        "Usage: %s index <input.fa> <output-dir> [<options>]\n"
        "Options:\n"
        "  -s               Seed size (default: %d)\n"
        "  -h               Hash table slack (default: %.1f)\n"
        " -t                Specify the maximum number of threads to use. Default is the number of cores. Do not leave a space after the -t, e.g., -t16\n"
        " -B<chars>         Specify characters to use as chromosome name terminators in the FASTA header line; these characters and anything after are\n"
        "                   not part of the chromosome name.  You must specify all characters on a single -B switch.  So, for example, with -B_|,\n"
        "                   the FASTA header line '>chr1|Chromosome 1' would generate a chromosome named 'chr1'.  There's a separate flag for\n"
        "                   indicating that a space is a terminator.\n"
        " -bSpace           Indicates that the space and tab characters are terminators for chromosome names (see -B above).  This may be used in addition\n"
        "                   to other terminators specified by -B.  -B and -bSpace are case sensitive.\n"
        " -p                Specify the number of Ns to put as padding between chromosomes.  This must be as large as the largest\n"
        "                   edit distance you'll ever use, and there's a performance advantage to have it be bigger than any\n"
        "                   read you'll process.  Default is %d.  Specify the amount of padding directly after -p without a space.\n"
        " -H                Build a histogram of seed popularity.  This is just for information, it's not used by SNAP.\n"
        "                   Specify the histogram file name directly after -H without leaving a space.\n"
        " -exact            Compute hash table sizes exactly.  This will slow down index build, but usually will result in smaller indices.\n"
        " -keysize          The number of bytes to use for the hash table key.  Larger values increase SNAP's memory footprint, but allow larger seeds.\n"
        "                   By default it's autoselected based on the seed size.\n"
        " -large            Build a larger index that's a little faster, particualrly for runs with quick/inaccurate parameters.  Increases index size by\n"<commit_msg>Format index help message<commit_after>/*++

Module Name:

    GenomeIndex.cpp

Abstract:

    Index (hash table) builder for the SNAP sequencer

Authors:

    <NAME>, August, 2011

Environment:

    User mode service.

Revision History:

    Adapted from <NAME>'s Scala implementation.

--*/

#include "stdafx.h"
#include "ApproximateCounter.h"
#include "BigAlloc.h"
#include "Compat.h"
#include "FASTA.h"
#include "FixedSizeSet.h"
#include "FixedSizeVector.h"
#include "GenericFile.h"
#include "GenericFile_stdio.h"
#include "Genome.h"
#include "GenomeIndex.h"
#include "HashTable.h"
#include "Seed.h"
#include "exit.h"
#include "Error.h"
#include "directions.h"

using namespace std;

static const int DEFAULT_SEED_SIZE = 27;
static const double DEFAULT_SLACK = 0.3;
static const unsigned DEFAULT_PADDING = 500;

const char *GenomeIndexFileName = "GenomeIndex";
const char *OverflowTableFileName = "OverflowTable";
const char *GenomeIndexHashFileName = "GenomeIndexHash";
const char *GenomeFileName = "Genome";

static void usage()
{
    WriteErrorMessage(
        "Usage: %s index <input.fa> <output-dir> [<options>]\n"
        "Options:\n"
        " -s                Seed size (default: %d)\n"
        " -h                Hash table slack (default: %.1f)\n"
        " -t                Specify the maximum number of threads to use. Default is the number of cores. Do not leave a space after the -t, e.g., -t16\n"
        " -B<chars>         Specify characters to use as chromosome name terminators in the FASTA header line; these characters and anything after are\n"
        "                   not part of the chromosome name.  You must specify all characters on a single -B switch.  So, for example, with -B_|,\n"
        "                   the FASTA header line '>chr1|Chromosome 1' would generate a chromosome named 'chr1'.  There's a separate flag for\n"
        "                   indicating that a space is a terminator.\n"
        " -bSpace           Indicates that the space and tab characters are terminators for chromosome names (see -B above).  This may be used in addition\n"
        "                   to other terminators specified by -B.  -B and -bSpace are case sensitive.\n"
        " -p                Specify the number of Ns to put as padding between chromosomes.  This must be as large as the largest\n"
        "                   edit distance you'll ever use, and there's a performance advantage to have it be bigger than any\n"
        "                   read you'll process.  Default is %d.  Specify the amount of padding directly after -p without a space.\n"
        " -H                Build a histogram of seed popularity.  This is just for information, it's not used by SNAP.\n"
        "                   Specify the histogram file name directly after -H without leaving a space.\n"
        " -exact            Compute hash table sizes exactly.  This will slow down index build, but usually will result in smaller indices.\n"
        " -keysize          The number of bytes to use for the hash table key.  Larger values increase SNAP's memory footprint, but allow larger seeds.\n"
        "                   By default it's autoselected based on the seed size.\n"
        " -large            Build a larger index that's a little faster, particualrly for runs with quick/inaccurate parameters.  Increases index size by\n"
{'fraction_non_alphanumeric': 0.083593633794437, 'fraction_numerical': 0.0050572661014428085, 'mean_word_length': 2.387405541561713, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4190749', 'n_tokens_mistral': 1964, 'n_tokens_neox': 1837, 'n_words': 787}