<commit_before>#!/usr/bin/perl -w

# $HeadURL$
# $LastChangedDate$
# $LastChangedBy$
# $LastChangedRevision$

$| = 1;

use strict;
use Carp;
use Cwd;
use Digest::MD5  2.20;
use File::Copy   2.03;
use File::Find;
use File::Path   1.0404;
use File::Temp   0.12   qw(tempdir tempfile);
use Getopt::Long 2.25;
use Text::Wrap;
use URI          1.17;
use English;

$Text::Wrap::columns = 72;

# Specify the location of the svn command.
my $svn = '/usr/bin/svn';

# Process the command line options.

# The base URL for the portion of the repository to work in.  Note
# that this does not have to be the root of the subversion repository,
# it can point to a subdirectory in the repository.
my $repos_base_url;

# The relative path from the repository base URL to work in to the
# directory to load the input directories into.
my $repos_load_rel_path;

# To specify where tags, which are simply copies of the imported
# directory, should be placed relative to the repository base URL, use
# the -t command line option.  This value must contain regular
# expressions that match portions of the input directory names to
# create an unique tag for each input directory.  The regular
# expressions are surrounded by a specified character to distinguish<commit_msg>Changed location of the svn command in svn_load_dirs.pl script from '/usr/bin/svn' to 'svn'. That makes the script more universal and it should work on Linux, Windows and Mac OS X if the svn is in the PATH.<commit_after>#!/usr/bin/perl -w

# $HeadURL$
# $LastChangedDate$
# $LastChangedBy$
# $LastChangedRevision$

$| = 1;

use strict;
use Carp;
use Cwd;
use Digest::MD5  2.20;
use File::Copy   2.03;
use File::Find;
use File::Path   1.0404;
use File::Temp   0.12   qw(tempdir tempfile);
use Getopt::Long 2.25;
use Text::Wrap;
use URI          1.17;
use English;

$Text::Wrap::columns = 72;

# Specify the location of the svn command.
my $svn = 'svn';

# Process the command line options.

# The base URL for the portion of the repository to work in.  Note
# that this does not have to be the root of the subversion repository,
# it can point to a subdirectory in the repository.
my $repos_base_url;

# The relative path from the repository base URL to work in to the
# directory to load the input directories into.
my $repos_load_rel_path;

# To specify where tags, which are simply copies of the imported
# directory, should be placed relative to the repository base URL, use
# the -t command line option.  This value must contain regular
# expressions that match portions of the input directory names to
# create an unique tag for each input directory.  The regular
# expressions are surrounded by a specified character to distinguish
{'fraction_non_alphanumeric': 0.0795539033457249, 'fraction_numerical': 0.017843866171003718, 'mean_word_length': 4.03932584269663, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4531218', 'n_tokens_mistral': 839, 'n_tokens_neox': 801, 'n_words': 387}