<commit_before>"""
PipelineUtilities.py - helper functions for CGAT pipelines
=============================================================

Purpose
-------

To make excuting database commands and files IO as trivial and (TODO:
robust) as possible possible. Before changing the default behaviour of
any of these functions please discuss!. Better to add new functions
and then deprecate.

.. note::

   This module needs refactoring as it implements
   some functionality found elsewhere such as the
   Database commands and text IO commands.

Reference
---------


"""
import CGAT.IOTools as IOTools
import pickle


def write(outfile, lines, header=False):
    ''' expects [[[line1-field1],[line1-field2 ] ],... ]'''
    handle = IOTools.openFile(outfile, "w")

    if header:
        handle.write("\t".join([str(title) for title in header]) + "\n")

    for line in lines:
        handle.write("\t".join([str(field) for field in line]) + "\n")

    handle.close()

# ########################## Biomart Access


def biomart_iterator(attributes,
                     host,
                     biomart,
                     dataset,
                     filters=None,
                     values=None
                     ):
    ''' Modified from pipeline biomart... '''

    # Use PipelineBiomant.biomart_iterator
    raise NotImplementedError(
        "deprecated, use Biomart.biomart_iterator")

# ########################## File Utitilies


def txtToDict(filename, key=None, sep="\t"):
    '''make a dictionary from a text file keyed
    on the specified column '''

    # Please see function in IOTools: readDict()
    count = 0
    result = {}
    valueidx, keyidx = False, False
    field_names = []

    with open(filename, "r") as fh:
        for line in fh:
            if line.startswith("#"):
                continue
            if count == 0:
                fieldn = 0
                for rawfield in line.split(sep):
                    field = rawfield.strip()
                    if field == key:
                        keyidx = fieldn
                    field_names.append(field)
                    fieldn += 1
<commit_msg>remove biomart_iterator<commit_after>"""
PipelineUtilities.py - helper functions for CGAT pipelines
=============================================================

Purpose
-------

To make excuting database commands and files IO as trivial and (TODO:
robust) as possible possible. Before changing the default behaviour of
any of these functions please discuss!. Better to add new functions
and then deprecate.

.. note::

   This module needs refactoring as it implements
   some functionality found elsewhere such as the
   Database commands and text IO commands.

Reference
---------


"""
import CGAT.IOTools as IOTools
import pickle


def write(outfile, lines, header=False):
    ''' expects [[[line1-field1],[line1-field2 ] ],... ]'''
    handle = IOTools.openFile(outfile, "w")

    if header:
        handle.write("\t".join([str(title) for title in header]) + "\n")

    for line in lines:
        handle.write("\t".join([str(field) for field in line]) + "\n")

    handle.close()


def txtToDict(filename, key=None, sep="\t"):
    '''make a dictionary from a text file keyed
    on the specified column '''

    # Please see function in IOTools: readDict()
    count = 0
    result = {}
    valueidx, keyidx = False, False
    field_names = []

    with open(filename, "r") as fh:
        for line in fh:
            if line.startswith("#"):
                continue
            if count == 0:
                fieldn = 0
                for rawfield in line.split(sep):
                    field = rawfield.strip()
                    if field == key:
                        keyidx = fieldn
                    field_names.append(field)
                    fieldn += 1

{'fraction_non_alphanumeric': 0.14629678094739598, 'fraction_numerical': 0.004187385501177702, 'mean_word_length': 2.1639072847682117, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5375689', 'n_tokens_mistral': 1061, 'n_tokens_neox': 985, 'n_words': 345}