<commit_before># Convert an arelequin .arp file to eigenstrat format
# Generate .arp file using fastsimcoal2 -s0 option 
# usage: python arelequin2eigenstrat.py -a arp_file.arp(.gz) -o out_root [other options]
# will generate out_root.[snp,ind,geno].
# By default will convert to genotype data, so use the -p option if you
# want to keep it unphased. 

from __future__ import division
import gzip, sys, getopt, pdb
import numpy as np

################################################################################

def parse_options():
    """
    Options are described by the help() function
    """
    options ={ "arp":None, "out":"out", "phased":False  }
	
    try:
        opts, args = getopt.getopt(sys.argv[1:], "a:o:p", ["arp", "out", "phased"])
        print opts, args
    except Exception as err:
        print str(err)
        sys.exit()

    for o, a in opts:
        print o,a
        if o in ["-a","--arp"]:         options["arp"] = a
        if o in ["-o","--out"]:         options["out"] = a
        elif o in ["-p","--phased"]:    options["phased"] = True

    print "found options:"
    print options
    return options

################################################################################

def open2(file, mode="r"):
	"""
	Open a file, or a gzipped file if it ends in .gz
	"""
	if file[-3:]==".gz":
		return gzip.open(file, mode)
	else:
		return open(file, mode)

################################################################################

def load_from_arp(arp):
    """
    Parse an arp file into an internal format, returning site data, which is a 
    list of lists, with the sites on each chromosome, and gt data which is a 
    dict with infomation on the samples and genotypes. 
    I'm really just guessing what the structure of the file is.
    """
    arp_file=open2(arp)

    ec={} 
    N_chr=-1                                # data for error checking. 
    N_sites=-1
    site_data=[]
    gt_data={}
    for line in arp_file: 
        if line.startswith("#Number of independent chromosomes"):
            N_chr=int(line.split()[-1])
        if line.startswith("#Total number of polymorphic sites"):
            N_sites=int(line.split()[-1])
        if "polymorphic positions on chromosome" in line:
            npos=int(line.split()[1])
            chrom=int(line.split()[-1])
            line=next(arp_file)
            site_data.append(None)
            site_data[chrom-1]=[int(x.replace(",", "")) for x in line[1:].split()]
            ec[chrom]=npos
        if "SampleName=" in line:
            sname=line.split("\"")[-2]
            line=arp_file.next()
            ssize=int(line.split("=")[-1])
            gt=np.zeros((N_sites, ssize), dtype='int')
            arp_file.next()               # "SampleData= {" line
            for i in range(ssize):
                line=arp_file.next()
                gt[:,i]=[int(int(x)>0) for x in line.split()[2]]   # converting {1,2,3}->1
            gt_data[sname]={"size":ssize, "gt":gt}

    if not N_chr==len(site_data):
        raise Exception( "Number of chromosomes does not match site data" )
    if not N_sites==sum([len(x) for x in site_data]):
        raise Exception( "Total number of sites does not match" )
<commit_msg>Deal with ascertained data correctly<commit_after># Convert an arelequin .arp file to eigenstrat format
# Generate .arp file using fastsimcoal2 -s0 option
# Only DNA sequence suppoerted (with -s0), but
# multiple populations/chromosomes are allowed.
# Ascertained data automatically detected. 
# usage: python arelequin2eigenstrat.py -a arp_file.arp(.gz) -o out_root [other options]
# will generate out_root.[snp,ind,geno].
# By default will convert to genotype data, so use the -p option if you
# want to keep it unphased. 

from __future__ import division
import gzip, sys, getopt, pdb
import numpy as np

################################################################################

def parse_options():
    """
    Options are described by the help() function
    """
    options ={ "arp":None, "out":"out", "phased":False  }
	
    try:
        opts, args = getopt.getopt(sys.argv[1:], "a:o:p", ["arp", "out", "phased"])
        print opts, args
    except Exception as err:
        print str(err)
        sys.exit()

    for o, a in opts:
        print o,a
        if o in ["-a","--arp"]:         options["arp"] = a
        if o in ["-o","--out"]:         options["out"] = a
        elif o in ["-p","--phased"]:    options["phased"] = True

    print "found options:"
    print options
    return options

################################################################################

def open2(file, mode="r"):
	"""
	Open a file, or a gzipped file if it ends in .gz
	"""
	if file[-3:]==".gz":
		return gzip.open(file, mode)
	else:
		return open(file, mode)

################################################################################

def load_from_arp(arp):
    """
    Parse an arp file into an internal format, returning site data, which is a 
    list of lists, with the sites on each chromosome, and gt data which is a 
    dict with infomation on the samples and genotypes. 
    I'm really just guessing what the structure of the file is.
    """
    arp_file=open2(arp)

    N_chr=-1                                
    N_sites=-1
    site_data=[]
    gt_data={}
    ascertained=False                   
    for line in arp_file: 
        if line.startswith("#Number of independent chromosomes"):
            N_chr=int(line.split()[-1])
        if line.startswith("#Total number of polymorphic sites") and not ascertained:
            N_sites=int(line.split()[-1])
        if line.startswith("#ASCERTAINED DATA"):
            ascertained=True
            N_sites=-1
        if line.startswith("#Number of polym. sites meeting ascertainment criterion:"):
            N_sites=int(line.split()[-1])
        if "polymorphic positions on chromosome" in line and not ascertained:
            npos=int(line.split()[1])
            chrom=int(line.split()[-1])
            line=next(arp_file)
            site_data.append(None)
            site_data[chrom-1]=[int(x.replace(",", "")) for x in line[1:].split()]
        if "polymorphic positions on chromosome" in line and ascertained:
            if not line.startswith("#Ascertained"):
                raise Exception("Line not ascertained")
            chrom=int(line.split()[-1])
            line=next(arp_file)
            npos=len(line.split())
            site_data.append(None)
            site_data[chrom-1]=[int(x.replace(",", "")) for x in line[1:].split()]
        if "SampleName=" in line:
            sname=line.split("\"")[-2]
            line=arp_file.next()
            ssize=int(line.split("=")[-1])
            gt=np.zeros((N_sites, ssize), dtype='int')
            arp_file.next()               # "SampleData= {" line
            for i in range(ssize):
                line=arp_file.next()
                if len(line.split)!=N_sites:
                    raise Exception("Wrong number of sites")
                gt[:,i]=[int(int(x)>0) for x in line.split()[2]]   # converting {1,2,3}->1
            gt_data[sname]={"size":ssize, "gt":gt}

    if not N_chr==len(site_data):
        raise Exception( "Number of chromosomes does not match site data" )
    if not N_sites==sum([len(x) for x in site_data]):
        raise Exception( "Total number of sites does not match" )

{'fraction_non_alphanumeric': 0.190740490050088, 'fraction_numerical': 0.007039393529172871, 'mean_word_length': 2.548511047070125, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1034303', 'n_tokens_mistral': 2309, 'n_tokens_neox': 2177, 'n_words': 660}