<commit_before># Built-in modules #
import os, sh

# First party modules #
from seqsearch.databases import Database
from plumbing.autopaths import DirectoryPath, AutoPaths
from plumbing.cache import property_cached
from fasta import FASTA

# Constants #
home = os.environ['HOME'] + '/'

###############################################################################
class Pfam(Database):
    """The Pfam database is a large collection of protein families,
    each represented by multiple sequence alignments and HMMs.

    http://pfam.xfam.org

    Pfam-A is the manually curated portion of the database that
    contains over 16,000 entries. An automatically generated supplement
    is provided called Pfam-B. Pfam-B was discontinued.

    To install:
        from seqsearch.databases.pfam import pfam
        pfam.download()
        pfam.unzip()
    """

    all_paths = """
    /raw/
    /unzipped/Pfam-A.hmm
    /unzipped/Pfam-A.fasta
    /specific/
    """

    short_name = "pfam"
    ftp_url    = "ftp.ebi.ac.uk"
    ftp_dir    = "/pub/databases/Pfam/current_release/"
    files      = ("Pfam-A.hmm.gz", "Pfam-A.fasta.gz")

    @property_cached
    def hmm_db(self):
        hmm_db = self.p.hmm
        hmm_db.seqtype = 'hmm_prot'
        return hmm_db

    @property_cached
    def fasta(self):
        fasta = FASTA(self.p.fasta)
        return fasta

###############################################################################
pfam = Pfam("hmm")

###############################################################################
class SpecificFamily(object):
    """When you are interested in having an HMM 'database' with only
    one specific Pfam in it."""

    all_paths = """
    /model.hmm
    /proteins.fasta
    """

    def __init__(self, fam_name):
        self.fam_name = fam_name
        self.base_dir = DirectoryPath(pfam.p.specific_dir + self.fam_name)
        self.p        = AutoPaths(self.base_dir, self.all_paths)

    @property_cached
    def hmm_db(self):
        hmm_db = self.p.model
        hmm_db.seqtype = 'hmm_prot'
        if not hmm_db.exists:
            print sh.hmmfetch('-o', hmm_db, pfam.hmm_db, self.fam_name)
            assert hmm_db
        return hmm_db

    @property_cached
    def fasta(self):
        fasta = self.p.proteins
        if not fasta.exists:
            fasta.create()
            for seq in pfam.fasta:
                if self.fam_name in seq.description: fasta.add_seq(seq)
            fasta.close()
            assert fasta
        return fasta<commit_msg>Fasta<commit_after># Built-in modules #
import os, sh

# First party modules #
from seqsearch.databases import Database
from plumbing.autopaths import DirectoryPath, AutoPaths
from plumbing.cache import property_cached
from fasta import FASTA

# Constants #
home = os.environ['HOME'] + '/'

###############################################################################
class Pfam(Database):
    """The Pfam database is a large collection of protein families,
    each represented by multiple sequence alignments and HMMs.

    http://pfam.xfam.org

    Pfam-A is the manually curated portion of the database that
    contains over 16,000 entries. An automatically generated supplement
    is provided called Pfam-B. Pfam-B was discontinued.

    To install:
        from seqsearch.databases.pfam import pfam
        pfam.download()
        pfam.unzip()
    """

    all_paths = """
    /raw/
    /unzipped/Pfam-A.hmm
    /unzipped/Pfam-A.fasta
    /specific/
    """

    short_name = "pfam"
    ftp_url    = "ftp.ebi.ac.uk"
    ftp_dir    = "/pub/databases/Pfam/current_release/"
    files      = ("Pfam-A.hmm.gz", "Pfam-A.fasta.gz")

    @property_cached
    def hmm_db(self):
        hmm_db = self.p.hmm
        hmm_db.seqtype = 'hmm_prot'
        return hmm_db

    @property_cached
    def fasta(self):
        fasta = FASTA(self.p.fasta)
        return fasta

###############################################################################
pfam = Pfam("hmm")

###############################################################################
class SpecificFamily(object):
    """When you are interested in having an HMM 'database' with only
    one specific Pfam in it."""

    all_paths = """
    /model.hmm
    /proteins.fasta
    """

    def __init__(self, fam_name):
        self.fam_name = fam_name
        self.base_dir = DirectoryPath(pfam.p.specific_dir + self.fam_name)
        self.p        = AutoPaths(self.base_dir, self.all_paths)

    @property_cached
    def hmm_db(self):
        hmm_db = self.p.model
        hmm_db.seqtype = 'hmm_prot'
        if not hmm_db.exists:
            print sh.hmmfetch('-o', hmm_db, pfam.hmm_db, self.fam_name)
            assert hmm_db
        return hmm_db

    @property_cached
    def fasta(self):
        fasta = FASTA(self.p.proteins)
        if not fasta.exists:
            fasta.create()
            for seq in pfam.fasta:
                if self.fam_name in seq.description: fasta.add_seq(seq)
            fasta.close()
            assert fasta
        return fasta
{'fraction_non_alphanumeric': 0.18239366963402573, 'fraction_numerical': 0.0019782393669634025, 'mean_word_length': 2.6037063435495367, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5295785', 'n_tokens_mistral': 1643, 'n_tokens_neox': 1521, 'n_words': 377}