<commit_before>import os
import cPickle as pickle

import params
import refsequtils


# ==============================
# = First-time initializations =
# ==============================

# does the pickle directory exist?
if not os.path.exists( os.path.join(params.vdj_dir,params.pickle_dir) ):
    os.mkdir( os.path.join(params.vdj_dir,params.pickle_dir) )

# test for each gene type pickle file
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGHV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGHV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHD_pickle)): refsequtils.process_IMGT_references( refsequtils.ReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGHD_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGHD_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGHJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGHJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGKV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGKV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGKV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGKJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGKJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGKJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGLV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGLV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGLV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGLJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGLJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGLJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRBV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRBV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBD_pickle)): refsequtils.process_IMGT_references( refsequtils.ReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRBD_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRBD_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRBJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRBJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRAV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRAV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRAV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRAJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRAJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRAJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRDV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRDV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDD_pickle)): refsequtils.process_IMGT_references( refsequtils.ReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRDD_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRDD_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRDJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRDJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRGV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRGV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRGV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRGJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRGJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRGJ_pickle),verbose=True)

# at this point, there should be pickle files with fully processed reference genes
# (including the LIGM-dependent parts)


# =======================
# = Load reference data =
# =======================

IGHV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHV_pickle)))
IGHD = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHD_pickle)))
IGHJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHJ_pickle)))
IGKV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGKV_pickle)))
IGKJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGKJ_pickle)))
IGLV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGLV_pickle)))
IGLJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGLJ_pickle)))
TRBV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBV_pickle)))
TRBD = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBD_pickle)))
TRBJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBJ_pickle)))
TRAV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRAV_pickle)))
TRAJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRAJ_pickle)))
TRDV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDV_pickle)))
TRDD = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDD_pickle)))
TRDJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDJ_pickle)))
TRGV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRGV_pickle)))
TRGJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRGJ_pickle)))


# =========================================================
# = Define data structures for compatibility with aligner =
# =========================================================

# IGHn_list -- list of the refseq identifiers (e.g., IGHJ1, IGHV6-1, etc.)
# IGHn_seqs -- dict where the keys are refseq IDs and the values are the actual sequences
#                 from refseq.
# IGHn_idx -- dict where the keys are refseq IDs and the values are the indices into the
#                 IGHn_list list.
# IGHV_offset

def legacy_data(ref_data):
    """Generate data structure used by aln tools etc."""
    locus_list = [elt.allele for elt in ref_data]
    locus_seqs = dict([(elt.allele,elt.seq) for elt in ref_data])
    locus_idx  = dict([(g,i) for i,g in enumerate(locus_list)])
    try:
        locus_offset = dict([(elt.allele,elt.cdr3_boundary-elt.accession_coords[0]) for elt in ref_data])
        return (locus_list,locus_seqs,locus_idx,locus_offset)
    except AttributeError:
        return (locus_list,locus_seqs,locus_idx)

(IGHV_list,IGHV_seqs,IGHV_idx,IGHV_offset) = legacy_data(IGHV)
(IGHD_list,IGHD_seqs,IGHD_idx)             = legacy_data(IGHD)
(IGHJ_list,IGHJ_seqs,IGHJ_idx,IGHJ_offset) = legacy_data(IGHJ)
(IGKV_list,IGKV_seqs,IGKV_idx,IGKV_offset) = legacy_data(IGKV)
(IGKJ_list,IGKJ_seqs,IGKJ_idx,IGKJ_offset) = legacy_data(IGKJ)
(IGLV_list,IGLV_seqs,IGLV_idx,IGLV_offset) = legacy_data(IGLV)
(IGLJ_list,IGLJ_seqs,IGLJ_idx,IGLJ_offset) = legacy_data(IGLJ)
(TRBV_list,TRBV_seqs,TRBV_idx,TRBV_offset) = legacy_data(TRBV)
(TRBD_list,TRBD_seqs,TRBD_idx)             = legacy_data(TRBD)
(TRBJ_list,TRBJ_seqs,TRBJ_idx,TRBJ_offset) = legacy_data(TRBJ)
(TRAV_list,TRAV_seqs,TRAV_idx,TRAV_offset) = legacy_data(TRAV)
(TRAJ_list,TRAJ_seqs,TRAJ_idx,TRAJ_offset) = legacy_data(TRAJ)
(TRDV_list,TRDV_seqs,TRDV_idx,TRDV_offset) = legacy_data(TRDV)
(TRDD_list,TRDD_seqs,TRDD_idx)             = legacy_data(TRDD)
(TRDJ_list,TRDJ_seqs,TRDJ_idx,TRDJ_offset) = legacy_data(TRDJ)
(TRGV_list,TRGV_seqs,TRGV_idx,TRGV_offset) = legacy_data(TRGV)
(TRGJ_list,TRGJ_seqs,TRGJ_idx,TRGJ_offset) = legacy_data(TRGJ)


# # refseq.py
# 
# """Contains data and functions for dealing with reference IMGT data.
# 
# refdatadir must be set to the directory where the IMGT flat file release
# is present.  This must include imgt.dat, imgt.fasta, and imgtrefseq.fasta.
# 
# IGHn_list -- list of the refseq identifiers (e.g., IGHJ1, IGHV6-1, etc.)
# IGHn_acc -- dict where keys are refseq IDs and values are the IMGT accession numbers
# IGHn_coords -- dict where keys are refseq IDs and values are pairs of coords that are
#                 extracted from the refseq database.  These numbers are unmodified seq
#                 coords of the reference element within the LIGM entry
# IGHn_seqs -- dict where the keys are refseq IDs and the values are the actual sequences
#                 from refseq.
# IGHn_idx -- dict where the keys are refseq IDs and the values are the indices into the
#                 IGHn_list list.
# IGHV_FR3_IMGT_end_coord -- dict where the keys are refseq IDs and the values are the coord of
#                             the end of FR3-IMGT (incl the 2nd-CYS) unmodified
# IGHJ_J_TRP_start_coord -- dict where the keys are refseq IDs and the values are are the
#                             coord of the start of the J-TRP site unmodified<commit_msg>working on lazy loading of refseq<commit_after>#DEBUG
import time

import os
import cPickle as pickle

import params
import refsequtils

#DEBUG
t0 = time.time()

# ==============================
# = First-time initializations =
# ==============================

# does the pickle directory exist?
if not os.path.exists( os.path.join(params.vdj_dir,params.pickle_dir) ):
    os.mkdir( os.path.join(params.vdj_dir,params.pickle_dir) )

# test for each gene type pickle file
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGHV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGHV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHD_pickle)): refsequtils.process_IMGT_references( refsequtils.ReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGHD_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGHD_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGHJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGHJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGKV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGKV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGKV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGKJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGKJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGKJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGLV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGLV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGLV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.IGLJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.IGLJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.IGLJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRBV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRBV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBD_pickle)): refsequtils.process_IMGT_references( refsequtils.ReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRBD_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRBD_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRBJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRBJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRAV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRAV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRAV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRAJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRAJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRAJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRDV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRDV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDD_pickle)): refsequtils.process_IMGT_references( refsequtils.ReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRDD_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRDD_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRDJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRDJ_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRGV_pickle)): refsequtils.process_IMGT_references(refsequtils.VReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRGV_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRGV_pickle),verbose=True)
if not os.path.exists(os.path.join(params.vdj_dir,params.pickle_dir,params.TRGJ_pickle)): refsequtils.process_IMGT_references(refsequtils.JReferenceEntry,os.path.join(params.vdj_dir,params.data_dir,params.TRGJ_fasta),os.path.join(params.vdj_dir,params.pickle_dir,params.TRGJ_pickle),verbose=True)

# at this point, there should be pickle files with fully processed reference genes
# (including the LIGM-dependent parts)


#DEBUG
t1 = time.time()
print "Check existance of pickle files: %s" % (t1-t0)

# =======================
# = Load reference data =
# =======================

def __getattr__():

IGHV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHV_pickle)))
IGHD = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHD_pickle)))
IGHJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGHJ_pickle)))
IGKV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGKV_pickle)))
IGKJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGKJ_pickle)))
IGLV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGLV_pickle)))
IGLJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.IGLJ_pickle)))
TRBV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBV_pickle)))
TRBD = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBD_pickle)))
TRBJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRBJ_pickle)))
TRAV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRAV_pickle)))
TRAJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRAJ_pickle)))
TRDV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDV_pickle)))
TRDD = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDD_pickle)))
TRDJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRDJ_pickle)))
TRGV = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRGV_pickle)))
TRGJ = pickle.load(open(os.path.join(params.vdj_dir,params.pickle_dir,params.TRGJ_pickle)))

#DEBUG
t2 = time.time()
print "Loading pickled files: %s" % (t2-t1)

# =========================================================
# = Define data structures for compatibility with aligner =
# =========================================================

# IGHn_list -- list of the refseq identifiers (e.g., IGHJ1, IGHV6-1, etc.)
# IGHn_seqs -- dict where the keys are refseq IDs and the values are the actual sequences
#                 from refseq.
# IGHn_idx -- dict where the keys are refseq IDs and the values are the indices into the
#                 IGHn_list list.
# IGHV_offset

def legacy_data(ref_data):
    """Generate data structure used by aln tools etc."""
    locus_list = [elt.allele for elt in ref_data]
    locus_seqs = dict([(elt.allele,elt.seq) for elt in ref_data])
    locus_idx  = dict([(g,i) for i,g in enumerate(locus_list)])
    try:
        locus_offset = dict([(elt.allele,elt.cdr3_boundary-elt.accession_coords[0]) for elt in ref_data])
        return (locus_list,locus_seqs,locus_idx,locus_offset)
    except AttributeError:
        return (locus_list,locus_seqs,locus_idx)

(IGHV_list,IGHV_seqs,IGHV_idx,IGHV_offset) = legacy_data(IGHV)
(IGHD_list,IGHD_seqs,IGHD_idx)             = legacy_data(IGHD)
(IGHJ_list,IGHJ_seqs,IGHJ_idx,IGHJ_offset) = legacy_data(IGHJ)
(IGKV_list,IGKV_seqs,IGKV_idx,IGKV_offset) = legacy_data(IGKV)
(IGKJ_list,IGKJ_seqs,IGKJ_idx,IGKJ_offset) = legacy_data(IGKJ)
(IGLV_list,IGLV_seqs,IGLV_idx,IGLV_offset) = legacy_data(IGLV)
(IGLJ_list,IGLJ_seqs,IGLJ_idx,IGLJ_offset) = legacy_data(IGLJ)
(TRBV_list,TRBV_seqs,TRBV_idx,TRBV_offset) = legacy_data(TRBV)
(TRBD_list,TRBD_seqs,TRBD_idx)             = legacy_data(TRBD)
(TRBJ_list,TRBJ_seqs,TRBJ_idx,TRBJ_offset) = legacy_data(TRBJ)
(TRAV_list,TRAV_seqs,TRAV_idx,TRAV_offset) = legacy_data(TRAV)
(TRAJ_list,TRAJ_seqs,TRAJ_idx,TRAJ_offset) = legacy_data(TRAJ)
(TRDV_list,TRDV_seqs,TRDV_idx,TRDV_offset) = legacy_data(TRDV)
(TRDD_list,TRDD_seqs,TRDD_idx)             = legacy_data(TRDD)
(TRDJ_list,TRDJ_seqs,TRDJ_idx,TRDJ_offset) = legacy_data(TRDJ)
(TRGV_list,TRGV_seqs,TRGV_idx,TRGV_offset) = legacy_data(TRGV)
(TRGJ_list,TRGJ_seqs,TRGJ_idx,TRGJ_offset) = legacy_data(TRGJ)


#DEBUG
t3 = time.time()
print "Converting to legacy reference objects: %s" % (t3-t2)

# # refseq.py
# 
# """Contains data and functions for dealing with reference IMGT data.
# 
# refdatadir must be set to the directory where the IMGT flat file release
# is present.  This must include imgt.dat, imgt.fasta, and imgtrefseq.fasta.
# 
# IGHn_list -- list of the refseq identifiers (e.g., IGHJ1, IGHV6-1, etc.)
# IGHn_acc -- dict where keys are refseq IDs and values are the IMGT accession numbers
# IGHn_coords -- dict where keys are refseq IDs and values are pairs of coords that are
#                 extracted from the refseq database.  These numbers are unmodified seq
#                 coords of the reference element within the LIGM entry
# IGHn_seqs -- dict where the keys are refseq IDs and the values are the actual sequences
#                 from refseq.
# IGHn_idx -- dict where the keys are refseq IDs and the values are the indices into the
#                 IGHn_list list.
# IGHV_FR3_IMGT_end_coord -- dict where the keys are refseq IDs and the values are the coord of
#                             the end of FR3-IMGT (incl the 2nd-CYS) unmodified
# IGHJ_J_TRP_start_coord -- dict where the keys are refseq IDs and the values are are the
#                             coord of the start of the J-TRP site unmodified
{'fraction_non_alphanumeric': 0.140574782244166, 'fraction_numerical': 0.0014905212166379431, 'mean_word_length': 7.020171834142697, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2693840', 'n_tokens_mistral': 8431, 'n_tokens_neox': 8233, 'n_words': 953}