# Path: inmembrane/plugins/signalp4.py
# def parse_signalp(signalp4_lines, proteins, id_mapping=None):
#     if id_mapping is None:
#         id_mapping = []
# 
#     past_preamble = False
#     for line in signalp4_lines:
#         if line.startswith("#"):
#             past_preamble = True
#             continue
#         if not past_preamble and line.strip() == '':  # skip empty lines
#             continue
#         if past_preamble:
#             if line.strip() == '':
#                 # in the case of web output of concatenated signalp output
#                 # files, an empty line after preamble means we have finished all
#                 # 'result' lines for that section
#                 past_preamble = False
#                 continue
#             words = line.split()
#             seqid = parse_fasta_header(words[0])[0]
#             if id_mapping:
#                 seqid = id_mapping[seqid]
#             proteins[seqid]['signalp_cleave_position'] = int(words[4])
#             proteins[seqid]['is_signalp'] = (words[9] == 'Y')
# 
#     return proteins
# 
# Path: inmembrane/helpers.py
# def log_stderr(s, width=76, comment=True):
#     """
#     Wrapper for all stderr out. Allows future customization.
#     """
#     if LOG_SILENT:
#         return
#     if s and s[-1] != "\n":
#         s += "\n"
#     if not s.startswith("#"):
#         s = "# " + s
#     sys.stderr.write(s)
# 
# Path: inmembrane/helpers.py
# def generate_safe_seqids(proteins):
#     """
#     Takes a 'proteins' dictionary, keyed by sequence id, and
#     adds a 'safe' sequence id ('safe_seqid') that is less likely
#     to be munged or break various external programs.
# 
#     Returns a tuple of the updated proteins dictionary and a dictionary
#     mapping the 'safe' sequence ids to the original ids.
#     """
#     id_mapping = {}
#     count = 0
#     for seqid in proteins:
#         safe_id = re.sub(r'[^\w]', "", seqid) + '_' + `count`
#         id_mapping[safe_id] = seqid
#         proteins[seqid]['safe_seqid'] = safe_id
#         count += 1
# 
#     return (proteins, id_mapping)
# 
# def proteins_to_fasta(proteins, seqids=[], use_safe_seqid=False, width=50):
#     """
#     Takes a proteins dictionary and returns a string containing
#     all the sequences in FASTA format. Option parameters are
#     a list of seqids to output (seqids) and the line width (width).
#     """
#     if seqids:
#         idlist = seqids
#     else:
#         idlist = proteins
# 
#     fasta_out = ""
#     for seqid in idlist:
#         seq_wrap = textwrap.fill(proteins[seqid]['seq'], width)
#         if use_safe_seqid:
#             header = proteins[seqid]['safe_seqid']
#         else:
#             header = proteins[seqid]['name']
#         fasta_out += ">%s\n%s\n" % (header, seq_wrap)
#     return fasta_out
# 
# def html2text(page, aggressive=False):
#     soup = BeautifulSoup(page, 'html.parser')
# 
#     # kill all script and style elements
#     for script in soup(["script", "style"]):
#         script.extract()
# 
#     # get text
#     text = soup.get_text()
# 
#     # break into lines and remove leading and trailing space on each
#     lines = (line.strip() for line in text.splitlines())
#     if aggressive:
#         # break multi-headlines into a line each
#         chunks = (phrase.strip() for line in lines for phrase in
#                   line.split("  "))
#         # drop blank lines
#         text = '\n'.join(chunk for chunk in chunks if chunk)
#     else:
#         text = '\n'.join(lines)
# 
#     return text


import sys, os, time
import requests
import inmembrane
from StringIO import StringIO
from BeautifulSoup import BeautifulSoup
    from collections import OrderedDict
    from ordereddict import OrderedDict
from inmembrane.plugins.signalp4 import parse_signalp
from inmembrane.helpers import log_stderr
from inmembrane.helpers import (generate_safe_seqids,
                                proteins_to_fasta,
                                html2text)
            waittime += 1;
            retries += 1
            waittime = min(waittime, 20)

        sys.stderr.write(" .. done !\n")

        if __DEBUG__:
            log_stderr(resultpage)
            # Example:
            #
            #   <pre>
            # # lcl_AE004092.1_cdsid_AAK33146.1 CYT score=-0.200913
            # # Cut-off=-3
            # lcl_AE004092.1_cdsid_AAK33146.1	LipoP1.0:Best	CYT	1	1	-0.200913
            # <P>
            # <hr>
            # # lcl_AE004092.1_cdsid_AAK33147.1 CYT score=-0.200913
            # # Cut-off=-3
            # lcl_AE004092.1_cdsid_AAK33147.1	LipoP1.0:Best	CYT	1	1	-0.200913
            # <P>
            # <hr>

        allresultpages += html2text(
            resultpage)  # += clean_result_page(resultpage)

    # we store the cleaned up result pages concatenated together
    fh = open(outfile, 'a+')
    fh.write(allresultpages)
    fh.close()


# Path: inmembrane/plugins/signalp4.py
# def parse_signalp(signalp4_lines, proteins, id_mapping=None):
#     if id_mapping is None:
#         id_mapping = []
# 
#     past_preamble = False
#     for line in signalp4_lines:
#         if line.startswith("#"):
#             past_preamble = True
#             continue
#         if not past_preamble and line.strip() == '':  # skip empty lines
#             continue
#         if past_preamble:
#             if line.strip() == '':
#                 # in the case of web output of concatenated signalp output
#                 # files, an empty line after preamble means we have finished all
#                 # 'result' lines for that section
#                 past_preamble = False
#                 continue
#             words = line.split()
#             seqid = parse_fasta_header(words[0])[0]
#             if id_mapping:
#                 seqid = id_mapping[seqid]
#             proteins[seqid]['signalp_cleave_position'] = int(words[4])
#             proteins[seqid]['is_signalp'] = (words[9] == 'Y')
# 
#     return proteins
# 
# Path: inmembrane/helpers.py
# def log_stderr(s, width=76, comment=True):
#     """
#     Wrapper for all stderr out. Allows future customization.
#     """
#     if LOG_SILENT:
#         return
#     if s and s[-1] != "\n":
#         s += "\n"
#     if not s.startswith("#"):
#         s = "# " + s
#     sys.stderr.write(s)
# 
# Path: inmembrane/helpers.py
# def generate_safe_seqids(proteins):
#     """
#     Takes a 'proteins' dictionary, keyed by sequence id, and
#     adds a 'safe' sequence id ('safe_seqid') that is less likely
#     to be munged or break various external programs.
# 
#     Returns a tuple of the updated proteins dictionary and a dictionary
#     mapping the 'safe' sequence ids to the original ids.
#     """
#     id_mapping = {}
#     count = 0
#     for seqid in proteins:
#         safe_id = re.sub(r'[^\w]', "", seqid) + '_' + `count`
#         id_mapping[safe_id] = seqid
#         proteins[seqid]['safe_seqid'] = safe_id
#         count += 1
# 
#     return (proteins, id_mapping)
# 
# def proteins_to_fasta(proteins, seqids=[], use_safe_seqid=False, width=50):
#     """
#     Takes a proteins dictionary and returns a string containing
#     all the sequences in FASTA format. Option parameters are
#     a list of seqids to output (seqids) and the line width (width).
#     """
#     if seqids:
#         idlist = seqids
#     else:
#         idlist = proteins
# 
#     fasta_out = ""
#     for seqid in idlist:
#         seq_wrap = textwrap.fill(proteins[seqid]['seq'], width)
#         if use_safe_seqid:
#             header = proteins[seqid]['safe_seqid']
#         else:
#             header = proteins[seqid]['name']
#         fasta_out += ">%s\n%s\n" % (header, seq_wrap)
#     return fasta_out
# 
# def html2text(page, aggressive=False):
#     soup = BeautifulSoup(page, 'html.parser')
# 
#     # kill all script and style elements
#     for script in soup(["script", "style"]):
#         script.extract()
# 
#     # get text
#     text = soup.get_text()
# 
#     # break into lines and remove leading and trailing space on each
#     lines = (line.strip() for line in text.splitlines())
#     if aggressive:
#         # break multi-headlines into a line each
#         chunks = (phrase.strip() for line in lines for phrase in
#                   line.split("  "))
#         # drop blank lines
#         text = '\n'.join(chunk for chunk in chunks if chunk)
#     else:
#         text = '\n'.join(lines)
# 
#     return text


# Path: inmembrane/plugins/signalp_scrape_web.py
import sys, os, time
import requests
import inmembrane
from StringIO import StringIO
from BeautifulSoup import BeautifulSoup
    from collections import OrderedDict
    from ordereddict import OrderedDict
from inmembrane.plugins.signalp4 import parse_signalp
from inmembrane.helpers import log_stderr
from inmembrane.helpers import (generate_safe_seqids,
                                proteins_to_fasta,
                                html2text)

            waittime += 1;
            retries += 1
            waittime = min(waittime, 20)

        sys.stderr.write(" .. done !\n")

        if __DEBUG__:
            log_stderr(resultpage)
            # Example:
            #
            #   <pre>
            # # lcl_AE004092.1_cdsid_AAK33146.1 CYT score=-0.200913
            # # Cut-off=-3
            # lcl_AE004092.1_cdsid_AAK33146.1	LipoP1.0:Best	CYT	1	1	-0.200913
            # <P>
            # <hr>
            # # lcl_AE004092.1_cdsid_AAK33147.1 CYT score=-0.200913
            # # Cut-off=-3
            # lcl_AE004092.1_cdsid_AAK33147.1	LipoP1.0:Best	CYT	1	1	-0.200913
            # <P>
            # <hr>

        allresultpages += html2text(
            resultpage)  # += clean_result_page(resultpage)

    # we store the cleaned up result pages concatenated together
    fh = open(outfile, 'a+')
    fh.write(allresultpages)
    fh.close()


    proteins = parse_signalp(allresultpages.splitlines(), proteins,