<jupyter_start><jupyter_text>Read the dadi sfs<jupyter_code>def dadi_to_momi(infile, outfile=None, verbose=False):
    if outfile == None:
        outfile = infile.split(".sfs")[0] + "_momi.sfs"
    dat = open(infile).readlines()
    ## Get rid of comment lines
    dat = [x.strip() for x in dat if not x.startswith("#")]
    if not len(dat) == 3:
        raise Exception("Malformed dadi sfs {}.\n  Must have 3 lines, yours has {}".format(infile, len(dat)))

    ## Parse the info line into nsamps per pop (list of ints), folding flag, and pop names list (if they are given)
    info = dat[0].split()
    nsamps = []
    ## Keep carving off values as long as they cast successfully as int
    for i in info:
        try:
            nsamps.append(int(i))
        except:
            pass
    nsamps = np.array(nsamps)
    pops = [x.replace('"', '') for x in info[len(nsamps)+1:]]
    folded = info[len(nsamps)]
    folded = False if "un" in folded else True
    if not len(pops) == len(nsamps):
        print("Number of populations doesn't agree with number of pop names, using generic names.")
        pops = ["pop"+x for x in range(len(nsamps))]
    if verbose: print("Info nsamps={} folded={} pops={}".format(nsamps, folded, pops))

    ## Get mask
    mask = list(map(int, dat[2].split()))
    if verbose: print(mask)

    ## Get sfs, and reshape based on sample sizes
    sfs = list(map(float, dat[1].split()))
    if verbose: print(sfs)
    length = np.ma.array(sfs, mask=mask).sum()
    sfs = np.array(sfs).reshape(nsamps)
    if verbose: print("length {}".format(length))
    if verbose: print(sfs)

    ## Get counts per sfs bin
    counts = Counter()
    for sfsbin in product(*[range(y) for y in [x for x in nsamps]]):
        ## Ignore monomorphic snps
        ## nsamps minus 1 here because of the off by one diff between number
        ## of bins in the sfs and actual number of samples
        if sfsbin == tuple(nsamps-1) or sfsbin == tuple([0] * len(nsamps)):
            continue
        ## ignore zero bin counts
        if sfs[sfsbin] == 0:
            continue
        if verbose: print(sfsbin, sfs[sfsbin]),
        counts.update({sfsbin:sfs[sfsbin]})
    if verbose: print("nbins {}".format(len(counts)))

    ## Convert SFS bin style into momi config style
    configs = pd.DataFrame(index=range(len(counts)), columns=pops)
    
    locus_info = []
    for i, c in enumerate(counts):
        ## (n-1) here because nbins in dadi sfs is n+1
        cfg = np.array([[(n-1)-x, x] for x,n in zip(c, nsamps)])
        configs.iloc[i] = [list(map(int, list(x))) for x in cfg]
        locus_info.append([0, i, counts[c]])
    if verbose: print("n_snps {}".format(np.sum([x[2] for x in locus_info])))

    ## Finally build the momi style sfs dictionary and write it out
    momi_sfs = {"sampled_pops":pops,
        "folded":folded,
        "length":int(length),
        "configs":configs.values.tolist(),
        "(locus,config_id,count)":locus_info}

    with open(outfile, 'w') as out:
        out.write("{}".format(json.dumps(momi_sfs)))
    ## make it pretty
    sfs = momi.Sfs.load(outfile)
    ## Fold if unfolded
    if folded: sfs = sfs.fold()
    sfs.dump(outfile)
#dadi_to_momi(infile, verbose=True)
dadi_to_momi("/home/isaac/easySFS/jupyter-notebooks/1000_folded/dadi/pop1-pop2.sfs", verbose=False)
dadi_to_momi("/home/isaac/easySFS/jupyter-notebooks/1000_unfolded/dadi/pop1-pop2.sfs", verbose=False)
d = OrderedDict([('[[7 1]\n [5 3]]', 2), ('[[5 3]\n [8 0]]', 50), ('[[8 0]\n [0 8]]', 74), ('[[8 0]\n [1 7]]', 18), ('[[6 2]\n [2 6]]', 2), ('[[7 1]\n [2 6]]', 1), ('[[5 3]\n [1 7]]', 1), ('[[6 2]\n [3 5]]', 2), ('[[8 0]\n [5 3]]', 48), ('[[4 4]\n [8 0]]', 32), ('[[5 3]\n [5 3]]', 1), ('[[8 0]\n [2 6]]', 23), ('[[7 1]\n [3 5]]', 4), ('[[5 3]\n [0 8]]', 38), ('[[6 2]\n [6 2]]', 1), ('[[5 3]\n [3 5]]', 2), ('[[4 4]\n [7 1]]', 2), ('[[7 1]\n [7 1]]', 2), ('[[8 0]\n [4 4]]', 32), ('[[6 2]\n [0 8]]', 23), ('[[7 1]\n [4 4]]', 3), ('[[8 0]\n [3 5]]', 28), ('[[6 2]\n [7 1]]', 1), ('[[4 4]\n [6 2]]', 1), ('[[7 1]\n [8 0]]', 185), ('[[8 0]\n [7 1]]', 185), ('[[7 1]\n [0 8]]', 17), ('[[5 3]\n [2 6]]', 2), ('[[6 2]\n [1 7]]', 4), ('[[8 0]\n [6 2]]', 66), ('[[6 2]\n [8 0]]', 87), ('[[5 3]\n [4 4]]', 2), ('[[6 2]\n [4 4]]', 2)])
for i, row in configs.iterrows():
    print(i, row.values)
dat = open(infile).readlines()
## Get rid of comment lines
dat = [x.strip() for x in dat if not x.startswith("#")]
if not len(dat) == 3:
    print("Malformed dadi sfs. Must have 3 lines, yours has {}".format(len(dat)))
dat

## Get nsamps
info = dat[0].split()
nsamps = []
for i in info:
    try:
        nsamps.append(int(i))
    except:
        pass
nsamps = np.array(nsamps)
pops = info[len(nsamps)+1:]
folded = info[len(nsamps)]
if not len(pops) == len(nsamps):
    print("Number of populations doesn't agree with number of pop names, using generic names.")
    pops = ["pop"+str(x) for x in range(len(nsamps))]
print(nsamps, folded, pops)

## Get mask
mask = map(int, dat[2].split())
print(mask)
if np.sum(mask) > 2:
    print("momi sfs conversion requires unfolded sfs. Folded sfs will 'work' but with unknown results.")

## Get sfs
sfs = map(int, map(np.round, map(float, dat[1].split())))
print(sfs)
length = np.ma.array(sfs, mask=mask).sum()
sfs = np.array(sfs).reshape(nsamps)
print("length {}".format(length))
print(sfs)

from collections import Counter
counts = Counter()
for sfsbin in product(*[range(y) for y in [x for x in nsamps]]):
    ## Ignore monomorphic snps
    ## nsamps minus 1 here because of the off by one diff between number
    ## of bins in the sfs and actual number of samples
    if sfsbin == tuple(nsamps-1) or sfsbin == tuple([0] * len(nsamps)):
        continue
    ## ignore zero bin counts
    if sfs[sfsbin] == 0:
        continue
    print(sfsbin, sfs[sfsbin])
    counts.update({sfsbin:sfs[sfsbin]})
counts
configs = pd.DataFrame(index=range(len(counts)), columns=pops)

#print(configs)

locus_count = 0
locus_info = []
for i, c in enumerate(counts):
    ## (n-1) here because nbins in dadi sfs is n+1
    configs.iloc[i] = [[(n-1)-x, x] for x,n in zip(c, nsamps)]
    locus_info.extend([x, i, 1] for x in range(locus_count, locus_count+counts[c]))
    locus_count += counts[c]
configs
locus_info
momi_sfs = {"sampled_pops":pops,
        "folded":folded,
        "length":length,
        "configs":configs.values.tolist(),
        "(locus,config_id,count)":locus_info}

with open("/tmp/watdo.sfs", 'w') as outfile:
    outfile.write("{}".format(momi_sfs))<jupyter_output><empty_output><jupyter_text>Crap below here<jupyter_code>!pwd
!../easySFS.py -i 25loci.vcf -p simout/simpops.txt --proj 8,8 -o 25_folded -f
!../easySFS.py -i 25loci.vcf -p simout/simpops.txt --proj 8,8 -o 25_unfolded --unfolded -f
dadi_to_momi("./25_folded/dadi/pop1-pop2.sfs")
dadi_to_momi("./25_unfolded/dadi/pop1-pop2.sfs")

!pwd
!../easySFS.py -i 100loci.vcf -p simout/simpops.txt --proj 8,8 -o 100_folded -f
!../easySFS.py -i 100loci.vcf -p simout/simpops.txt --proj 8,8 -o 100_unfolded --unfolded -f
dadi_to_momi("./100_folded/dadi/pop1-pop2.sfs")
dadi_to_momi("./100_unfolded/dadi/pop1-pop2.sfs")

!pwd
!../easySFS.py -i 1000loci.vcf -p simout/simpops.txt --proj 8,8 -o 1000_folded -f
!../easySFS.py -i 1000loci.vcf -p simout/simpops.txt --proj 8,8 -o 1000_unfolded --unfolded -f
dadi_to_momi("./1000_folded/dadi/pop1-pop2.sfs")
dadi_to_momi("./1000_unfolded/dadi/pop1-pop2.sfs")<jupyter_output>/home/isaac/easySFS/jupyter-notebooks
Processing 2 populations - ['pop1', 'pop2']
  Sampling one snp per locus
Doing 1D sfs - pop1
Doing 1D sfs - pop2
Doing 2D sfs - ('pop1', 'pop2')
Doing multiSFS for all pops
Processing 2 populations - ['pop1', 'pop2']
  Sampling one snp per locus
Doing 1D sfs - pop1
Doing 1D sfs - pop2
Doing 2D sfs - ('pop1', 'pop2')
Doing multiSFS for all pops
momi sfs conversion requires unfolded sfs. Folded sfs will 'work' but with unknown results.
{'fraction_non_alphanumeric': 0.1889873417721519, 'fraction_numerical': 0.0410126582278481, 'mean_word_length': 3.3340647284695555, 'pattern_counts': {'":': 10, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '167282', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 3372, 'n_tokens_neox': 3112, 'n_words': 885, 'path': 'jupyter-notebooks/easySFS-dadi_sfs_conversion.ipynb', 'repo_name': 'itots/easySFS', 'size': '7900'}