<commit_before>#!/usr/bin/perl

##########LICENCE##########
# Copyright (c) 2014-2019 Genome Research Ltd.
#
# Author: <NAME> <<EMAIL>>
#
# This file is part of cgpBattenberg.
#
# cgpBattenberg is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
##########LICENCE##########

use FindBin;
use lib "$FindBin::Bin/../lib";

use strict;
use warnings FATAL => 'all';
use autodie qw(:all);

use File::Path qw( remove_tree make_path );
use File::Spec;
use Getopt::Long;
use File::Copy qw(move);
use Pod::Usage qw(pod2usage);
use Const::Fast qw(const);
use File::Fetch;
use File::Basename;
use Archive::Extract;
use Carp;

use Data::Dumper;

use Sanger::CGP::Battenberg;

const my $DEFAULT_URL => q{https://mathgen.stats.ox.ac.uk/impute/};
const my $IMPUTE_TGZ_PATTERN => q{ALL_1000G_phase1integrated_%s_impute.tgz};
const my $IMPUTE_LEGEND_TGZ_PATTERN => q{ALL_1000G_phase1integrated_%s_annotated_legends.tgz};
const my $IMPUTE_UNPACK_PATTERN => q{ALL_1000G_phase1integrated_%s_impute};
const my $IMPUTE_HAP_PATTERN => q{ALL_1000G_phase1integrated_%s_chr%s_impute.hap.gz};
const my $IMPUTE_LEGEND_PATTERN => q{ALL_1000G_phase1integrated_%s_chr%s_impute.legend.gz};
const my $GENETIC_MAP_PATTERN => q{genetic_map_chr%s_combined_b37.txt};
const my $IMPUTE_INFO_FILENAME => q{impute_info.txt};
const my $ONEKGEN_ALLELE_FILE_PATTERN => q{1000genomesAlleles2012_chr%s.txt};
const my $ONEKGEN_LOCI_FILE_PATTERN => q{1000genomesloci2012_chr%s.txt};


const my @THOUSAND_GENOMES_FILE_ORDER => qw(  1 2 3 4 5 6 7 8 9 10 11
                                                12 13 14 15 16 17 18 19
                                                20 21 22 X );

const my @THOUSAND_GENOMES_START => qw(  );

const my @THOUSAND_GENOMES_END => qw(  );

const my @THOUSAND_GENOMES_CHRS => qw( 1 2 3 4 5 6 7 8 9 10 11
                                        12 13 14 15 16 17 18 19
                                        20 21 22 X );

const my @IMPUTE_INFO_FILE_ORDER => qw( 1 2 3 4 5 6 7 8 9 10 11
                                        12 13 14 15 16 17 18 19
                                        20 21 22 X_PAR1 X_nonPAR X_PAR2);

const my @IMPUTE_START => qw{ 0 0 0 0 0 0 0 0 0 0 0
                                0 0 0 0 0 0 0 0 20000000 0
                                0 0 2600000 154000000};

const my @IMPUTE_END => qw{ 250000000 245000000 200000000 195000000 185000000
                                175000000 160000000 150000000 145000000 140000000
                                135000000 135000000 115000000 110000000 105000000
                                90000000 80000000 80000000 65000000 65000000 50000000
                                50000000 2700000 155000000 156000000 };

const my @IMPUTE_MALE => qw{ 1 1 1 1 1 1 1 1 1 1 1 1
                             1 1 1 1 1 1 1 1 1 1 1 0 1};

const my @IMPUTE_CHRS => qw{ 1 2 3 4 5 6 7 8 9 10 11 12 13
                             14 15 16 17 18 19 20 21 22 X X X};

const my %BASE_LOOKUP => ("A" => 1,
                            "C" => 2,
                            "G" => 3,
                            "T" => 4);

const my $IMPUTE_INFO_LINE => "%s\t%s\t%s\t%s\t%d\t%d\t%d\n";
const my $ALLELE_LINE => "%d\t%d\t%d\n";
const my $LOCI_LINE => "%s\t%d\n";

my $DOWNLOAD_VERSION = "v3";

{
    my $opts = setup();
    #Download Both sets of files to directory and unpack.
    print "Download and unpack files\n";
    download_unpack_files($opts);
    #Get the date/V3 from data download.
    ##copy hap.gz without replace
    my $impute_unp = sprintf($IMPUTE_UNPACK_PATTERN,$DOWNLOAD_VERSION);
    print "Move hap.gz files\n";
    my $hap_files = get_filenames_for_match(File::Spec->catdir($opts->{'tmp'},$impute_unp),$IMPUTE_HAP_PATTERN,[$DOWNLOAD_VERSION]);
    my $haps = move_files(File::Spec->catdir($opts->{'tmp'},$impute_unp),$opts->{'impdir'},$hap_files);
    ##copy genetic_map_chr2_combined_b37.txt without replace
    print "Move genetic map files\n";
    my $gen_map_files = get_filenames_for_match(File::Spec->catdir($opts->{'tmp'},$impute_unp),$GENETIC_MAP_PATTERN,[]);
    my $gens = move_files(File::Spec->catdir($opts->{'tmp'},$impute_unp),$opts->{'impdir'},$gen_map_files);
    #Replace chr in legend files as we copy.
    print "Unpack and edit legend files\n";
    my $leg_files = get_filenames_for_match(File::Spec->catdir($opts->{'tmp'},$impute_unp),$IMPUTE_LEGEND_PATTERN,[$DOWNLOAD_VERSION]);
    my $legs = copy_files_with_convert(File::Spec->catdir($opts->{'tmp'},$impute_unp),$opts->{'impdir'},$leg_files);
    print "Create impute info file\n";
    create_impute_info_file($opts,$legs,$gens,$haps);
    print "Create 1000genomes files\n";
    create_one_k_genomes_files($opts,$legs);
    print "Cleaning up\n";
    cleanup($opts);
    print "Done\n";
}

sub cleanup{
  my ($opts) = @_;
  remove_tree($opts->{'tmp'}) or croak("Error removing temporary directory $opts->{tmp}");
  return;
}

sub create_one_k_genomes_files{
    my ($opts,$legs) = @_;
    #create the 1k genomes data files.
    for(my $i=0; $i<scalar(@THOUSAND_GENOMES_FILE_ORDER); $i++){
        my $oneKGenAllFile = File::Spec->catfile($opts->{'onekdir'},
                                        sprintf($ONEKGEN_ALLELE_FILE_PATTERN,$THOUSAND_GENOMES_FILE_ORDER[$i]));
        my $oneKGenLociFile =  File::Spec->catfile($opts->{'onekdir'},
                                        sprintf($ONEKGEN_LOCI_FILE_PATTERN,$THOUSAND_GENOMES_FILE_ORDER[$i]));

        my ($ALLELE,$LOCI);
        open($ALLELE,'>',$oneKGenAllFile);
            open($LOCI,'>',$oneKGenLociFile);
                print $ALLELE "position\ta0\ta1\n" or croak ("Error writing header to allele file '$oneKGenAllFile'\n");
                #Open relevant legend file
                #Account for X having multiple sections in the legend files.
                #NB this will break if not human...
        if($THOUSAND_GENOMES_FILE_ORDER[$i] == "X"){ #If we're on chromosome X
          my $j=$i;<commit_msg>fixed but in chrX section<commit_after>#!/usr/bin/perl

##########LICENCE##########
# Copyright (c) 2014-2019 Genome Research Ltd.
#
# Author: <NAME> <<EMAIL>>
#
# This file is part of cgpBattenberg.
#
# cgpBattenberg is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation; either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
##########LICENCE##########

use FindBin;
use lib "$FindBin::Bin/../lib";

use strict;
use warnings FATAL => 'all';
use autodie qw(:all);

use File::Path qw( remove_tree make_path );
use File::Spec;
use Getopt::Long;
use File::Copy qw(move);
use Pod::Usage qw(pod2usage);
use Const::Fast qw(const);
use File::Fetch;
use File::Basename;
use Archive::Extract;
use Carp;

use Data::Dumper;

use Sanger::CGP::Battenberg;

const my $DEFAULT_URL => q{https://mathgen.stats.ox.ac.uk/impute/};
const my $IMPUTE_TGZ_PATTERN => q{ALL_1000G_phase1integrated_%s_impute.tgz};
const my $IMPUTE_LEGEND_TGZ_PATTERN => q{ALL_1000G_phase1integrated_%s_annotated_legends.tgz};
const my $IMPUTE_UNPACK_PATTERN => q{ALL_1000G_phase1integrated_%s_impute};
const my $IMPUTE_HAP_PATTERN => q{ALL_1000G_phase1integrated_%s_chr%s_impute.hap.gz};
const my $IMPUTE_LEGEND_PATTERN => q{ALL_1000G_phase1integrated_%s_chr%s_impute.legend.gz};
const my $GENETIC_MAP_PATTERN => q{genetic_map_chr%s_combined_b37.txt};
const my $IMPUTE_INFO_FILENAME => q{impute_info.txt};
const my $ONEKGEN_ALLELE_FILE_PATTERN => q{1000genomesAlleles2012_chr%s.txt};
const my $ONEKGEN_LOCI_FILE_PATTERN => q{1000genomesloci2012_chr%s.txt};


const my @THOUSAND_GENOMES_FILE_ORDER => qw(  1 2 3 4 5 6 7 8 9 10 11
                                                12 13 14 15 16 17 18 19
                                                20 21 22 X );

const my @THOUSAND_GENOMES_START => qw(  );

const my @THOUSAND_GENOMES_END => qw(  );

const my @THOUSAND_GENOMES_CHRS => qw( 1 2 3 4 5 6 7 8 9 10 11
                                        12 13 14 15 16 17 18 19
                                        20 21 22 X );

const my @IMPUTE_INFO_FILE_ORDER => qw( 1 2 3 4 5 6 7 8 9 10 11
                                        12 13 14 15 16 17 18 19
                                        20 21 22 X_PAR1 X_nonPAR X_PAR2);

const my @IMPUTE_START => qw{ 0 0 0 0 0 0 0 0 0 0 0
                                0 0 0 0 0 0 0 0 20000000 0
                                0 0 2600000 154000000};

const my @IMPUTE_END => qw{ 250000000 245000000 200000000 195000000 185000000
                                175000000 160000000 150000000 145000000 140000000
                                135000000 135000000 115000000 110000000 105000000
                                90000000 80000000 80000000 65000000 65000000 50000000
                                50000000 2700000 155000000 156000000 };

const my @IMPUTE_MALE => qw{ 1 1 1 1 1 1 1 1 1 1 1 1
                             1 1 1 1 1 1 1 1 1 1 1 0 1};

const my @IMPUTE_CHRS => qw{ 1 2 3 4 5 6 7 8 9 10 11 12 13
                             14 15 16 17 18 19 20 21 22 X X X};

const my %BASE_LOOKUP => ("A" => 1,
                            "C" => 2,
                            "G" => 3,
                            "T" => 4);

const my $IMPUTE_INFO_LINE => "%s\t%s\t%s\t%s\t%d\t%d\t%d\n";
const my $ALLELE_LINE => "%d\t%d\t%d\n";
const my $LOCI_LINE => "%s\t%d\n";

my $DOWNLOAD_VERSION = "v3";

{
    my $opts = setup();
    #Download Both sets of files to directory and unpack.
    print "Download and unpack files\n";
    download_unpack_files($opts);
    #Get the date/V3 from data download.
    ##copy hap.gz without replace
    my $impute_unp = sprintf($IMPUTE_UNPACK_PATTERN,$DOWNLOAD_VERSION);
    print "Move hap.gz files\n";
    my $hap_files = get_filenames_for_match(File::Spec->catdir($opts->{'tmp'},$impute_unp),$IMPUTE_HAP_PATTERN,[$DOWNLOAD_VERSION]);
    my $haps = move_files(File::Spec->catdir($opts->{'tmp'},$impute_unp),$opts->{'impdir'},$hap_files);
    ##copy genetic_map_chr2_combined_b37.txt without replace
    print "Move genetic map files\n";
    my $gen_map_files = get_filenames_for_match(File::Spec->catdir($opts->{'tmp'},$impute_unp),$GENETIC_MAP_PATTERN,[]);
    my $gens = move_files(File::Spec->catdir($opts->{'tmp'},$impute_unp),$opts->{'impdir'},$gen_map_files);
    #Replace chr in legend files as we copy.
    print "Unpack and edit legend files\n";
    my $leg_files = get_filenames_for_match(File::Spec->catdir($opts->{'tmp'},$impute_unp),$IMPUTE_LEGEND_PATTERN,[$DOWNLOAD_VERSION]);
    my $legs = copy_files_with_convert(File::Spec->catdir($opts->{'tmp'},$impute_unp),$opts->{'impdir'},$leg_files);
    print "Create impute info file\n";
    create_impute_info_file($opts,$legs,$gens,$haps);
    print "Create 1000genomes files\n";
    create_one_k_genomes_files($opts,$legs);
    print "Cleaning up\n";
    cleanup($opts);
    print "Done\n";
}

sub cleanup{
  my ($opts) = @_;
  remove_tree($opts->{'tmp'}) or croak("Error removing temporary directory $opts->{tmp}");
  return;
}

sub create_one_k_genomes_files{
    my ($opts,$legs) = @_;
    #create the 1k genomes data files.
    for(my $i=0; $i<scalar(@THOUSAND_GENOMES_FILE_ORDER); $i++){
        my $oneKGenAllFile = File::Spec->catfile($opts->{'onekdir'},
                                        sprintf($ONEKGEN_ALLELE_FILE_PATTERN,$THOUSAND_GENOMES_FILE_ORDER[$i]));
        my $oneKGenLociFile =  File::Spec->catfile($opts->{'onekdir'},
                                        sprintf($ONEKGEN_LOCI_FILE_PATTERN,$THOUSAND_GENOMES_FILE_ORDER[$i]));

        my ($ALLELE,$LOCI);
        open($ALLELE,'>',$oneKGenAllFile);
            open($LOCI,'>',$oneKGenLociFile);
                print $ALLELE "position\ta0\ta1\n" or croak ("Error writing header to allele file '$oneKGenAllFile'\n");
                #Open relevant legend file
                #Account for X having multiple sections in the legend files.
                #NB this will break if not human...
        if($THOUSAND_GENOMES_FILE_ORDER[$i] eq "X"){ #If we're on chromosome X
          my $j=$i;
{'fraction_non_alphanumeric': 0.12304147465437788, 'fraction_numerical': 0.07665130568356375, 'mean_word_length': 2.7840743969776227, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 113, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 24, 'word_list_counts': {'cursed_substrings.json': 58, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '244306', 'n_tokens_mistral': 5302, 'n_tokens_neox': 4336, 'n_words': 1222}