<commit_before>#!/usr/bin/env perl
use Getopt::Long;
use strict;
use warnings;
use Data::Dumper;
use List::Util 'shuffle';
$|=1;
$Data::Dumper::Sortkeys =1;

###############
# Description:
###############
# Given distinct "subject" (S) and a "target" (T) distributions, this script attempts to mimic T's density (i.e., its shape) by pseudo-randomly sampling from S's population.
# Warning: The script attempts to match T's density only, not its population size.

###################
# Arguments/Input:
###################
#    arg1: path to file containing T's values (1 column, 1 value per line)
#    arg2: number of bins to split the distributions into
#    arg3: path to tab-separated file containing S's identifiers and values (column 1: unique identifier; column 2: corresponding value)

###########
# Options:
###########
#    transform: bin `transform`-transformed values in both distributions. Output values will be the original, non-transformed ones, though.
#     Possible values: 'log10' only.
#     binning into log10-transformed is highly recommended e.g. for matching FPKM/RPKM distributions.

###############################
# Output (to standard output):
###############################
# Pseudo-random subset of subject file (i.e., arg3)

#########
# Notes:
#########

# Pseudo-randomness: items from S's population are randomly selected **within bins**, not within the entire population, hence the "pseudo" prefix

# Log-transform: binning into log10-transformed is highly recommended e.g. for matching FPKM/RPKM distributions (see `transform` option).

# Re-iteration: it might be necessary to call the script several times sequentially (i.e. input -> output1; output1 -> output2; output2 -> output3, etc.) until reaching an optimum. Here's an example of a wrapper bash script that does just this (100 iterations, 500 bins):

# export passes=100
# ln -s subject.txt output.0.txt
# for i in `seq 1 $passes`; do
# let j=$i-1
# matchDistribution.pl target 500 subject.txt output.$j.txt > output.$i.txt
# rm -f output.$j.txt
# done

# The final output will be in the file named output.$passes.txt

my $transform;
GetOptions (
        'transform=s' => \$transform

        );<commit_msg>Minor change in comments<commit_after>#!/usr/bin/env perl
use Getopt::Long;
use strict;
use warnings;
use Data::Dumper;
use List::Util 'shuffle';
$|=1;
$Data::Dumper::Sortkeys =1;

###############
# Description:
###############
# Given distinct "subject" (S) and a "target" (T) distributions, this script attempts to mimic T's density (i.e., its shape) by pseudo-randomly sampling from S's population.
# Warning: The script attempts to match T's density only, not its population size.

###################
# Arguments/Input:
###################
#    arg1: path to file containing T's values (1 column, 1 value per line)
#    arg2: number of bins to split the distributions into
#    arg3: path to tab-separated file containing S's identifiers and values (column 1: unique identifier; column 2: corresponding value)

###########
# Options:
###########
#    transform: bin `transform`-transformed values in both distributions. Output values will be the original, non-transformed ones, though.
#     Possible values: 'log10' only.
#     binning into log10-transformed is highly recommended e.g. for matching FPKM/RPKM distributions.

###############################
# Output (to standard output):
###############################
# Pseudo-random subset of subject file (i.e., arg3), such that its distribution matches T's density as closely as possible.

#########
# Notes:
#########

# Pseudo-randomness: items from S's population are randomly selected **within bins**, not within the entire population, hence the "pseudo" prefix

# Log-transform: binning into log10-transformed is highly recommended e.g. for matching FPKM/RPKM distributions (see `transform` option).

# Re-iteration: it might be necessary to call the script several times sequentially (i.e. input -> output1; output1 -> output2; output2 -> output3, etc.) until reaching an optimum. Here's an example of a wrapper bash script that does just this (100 iterations, 500 bins):

# export passes=100
# ln -s subject.txt output.0.txt
# for i in `seq 1 $passes`; do
# let j=$i-1
# matchDistribution.pl target.txt 500 output.$j.txt > output.$i.txt
# rm -f output.$j.txt
# done

# The final output will be in the file named output.$passes.txt

my $transform;
GetOptions (
        'transform=s' => \$transform

        );
{'fraction_non_alphanumeric': 0.16997989725262452, 'fraction_numerical': 0.016082197900379718, 'mean_word_length': 4.52157829839704, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4496305', 'n_tokens_mistral': 1340, 'n_tokens_neox': 1244, 'n_words': 521}