<commit_before>from collections import defaultdict
import gzip
import re

old_illumina_header = [("instrument", str),
                       ("flowcell_lane", str),
                       ("flowcell_number", int),
                       None,  # x-tile
                       None,  # y-tile
                       ("index", str),
                       ("pair", int)]

illumina_header = [("instrument", str),
                   ("run_id", int),
                   ("flowcell_id", str),
                   ("flowcell_lane", int),
                   None,  # tile number
                   None,  # x-tile
                   None,  # y-tile
                   ("pair", int),
                   ("filtered", str),
                   ("control_bits", int),
                   None]  # index sequence

def set_type(var, val):
    if var == str:
        return str(val)
    elif var == int:
        return int(val)

class fastq:
    # Simple class for reading fastq files.
    def __init__(self, filename):
        self.filename = filename
        # Get fastq information
        header_lines = [x["info"] for x in self.read(100)]
        header = re.split(r'(\:|#| )',header_lines[0])[::2]
        if len(header) == 11:
            self.instrument = header[0]
            for attr, val in zip(illumina_header, header):
                if attr is not None:
                    val = set_type(attr[1], val) # Set variable type
                    setattr(self, attr[0], val)


        elif len(header) == 21:
            pass
        else:
            raise Exception("Unknown header")

    def read(self, n=-1):
        """
            Iterate through gzipped fastq file and put
            yield sequence+info in dictionary.
        """
        with gzip.open(self.filename, 'rb') as f:
            dna = {}
            for linenum, line in enumerate(f):
                dna["info"] = line.strip()
                dna["seq"] = f.next().strip()
                f.next()
                dna["qual"] = f.next().strip()
                if linenum*4 <= n or n == -1:
                    yield dna
                else:
                    break

    def count_non_overlapping(self, needle, n=-1):
        """
            Count the number of non-overlapping repeats
            and return a tuple of the distribution.
        """
        count = defaultdict(int)
        for i in self.read(n):
            count[i["seq"].count(needle)] += 1
        return sorted(count.items())

    def longest_sequential_repeat(self, needle, n=-1):
        """
            Count the longest sequential repeat,
            and return a tuple of the distribution.
        """
        count = defaultdict(int)
        for i in self.read(n):
            needle_rep = needle<commit_msg>added barcodes to fastq<commit_after>from collections import defaultdict
import gzip
import re
from itertools import groupby as g

old_illumina_header = [("instrument", str),
                       ("flowcell_lane", int),
                       ("flowcell_number", int),
                       None,  # x-tile
                       None,  # y-tile
                       ("barcode",str),  # barcode - fetched later
                       ("pair", int)]

illumina_header = [("instrument", str),
                   ("run_id", int),
                   ("flowcell_id", str),
                   ("flowcell_lane", int),
                   None,  # tile number
                   None,  # x-tile
                   None,  # y-tile
                   ("pair", int),
                   ("filtered", str),
                   ("control_bits", int),
                   ("barcode",str)]  # barcode/index sequence; fetched later.

def most_common(L):
    # Fetch most common item from a list.
  try:
    return max(g(sorted(L)), key=lambda(x, v):(len(list(v)),-L.index(x)))[0]
  except:
    return ""

def set_type(var, val):
    # Set variable types.
    if var == str:
        return str(val)
    elif var == int:
        return int(val)

class fastq:
    # Simple class for reading fastq files.
    def __init__(self, filename):
        self.filename = filename
        # Get fastq information
        header_lines = [x["info"] for x in self.read(1)]
        header = re.split(r'(\:|#|/| )',header_lines[0])[::2]
        if len(header) == 11:
            # Use new header format.
            use_header = illumina_header
        elif len(header) == 6:
            # Use old header format.
            use_header = old_illumina_header
        elif len(header) == 7:
            # Use old header and add pair if available.
            use_header = old_illumina_header + ["pair"]
        else:
            raise Exception("Unknown header")
        # Fetch index
        index_loc = use_header.index(("barcode",str))
        fetch_index = [re.split(r'(\:|#|/| )',x["info"])[::2][index_loc] for x in self.read(1000)]
        self.barcode = most_common(fetch_index)

        # Set remaining attributes.
        for attr, val in zip(use_header, header):
            if attr is not None:
                val = set_type(attr[1], val) # Set variable type
                setattr(self, attr[0], val)

        # Fetch index
        #line["index"] = most_common(index_set)


    def read(self, n=-1):
        """
            Iterate through gzipped fastq file and put
            yield sequence+info in dictionary.
        """
        with gzip.open(self.filename, 'rb') as f:
            dna = {}
            for linenum, line in enumerate(f):
                dna["info"] = line.strip()
                dna["seq"] = f.next().strip()
                f.next()
                dna["qual"] = f.next().strip()
                if linenum < n or n == -1:
                    yield dna
                else:
                    break

    def count_non_overlapping(self, needle, n=-1):
        """
            Count the number of non-overlapping repeats
            and return a tuple of the distribution.
        """
        count = defaultdict(int)
        for i in self.read(n):
            count[i["seq"].count(needle)] += 1
        return sorted(count.items())

    def longest_sequential_repeat(self, needle, n=-1):
        """
            Count the longest sequential repeat,
            and return a tuple of the distribution.
        """
        count = defaultdict(int)
        for i in self.read(n):
            needle_rep = needle
{'fraction_non_alphanumeric': 0.11824748743718592, 'fraction_numerical': 0.005967336683417085, 'mean_word_length': 1.4638297872340424, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1821075', 'n_tokens_mistral': 1832, 'n_tokens_neox': 1732, 'n_words': 498}