<commit_before># FOR GRID SEARCH CANDIDATES

import itertools
import numpy
import os
import pandas
import scipy.stats

from grocsvs import step
from grocsvs import structuralvariants

from grocsvs.stages import sv_candidates


class CombineRefinedBreakpointsStep(step.StepChunk):
    @staticmethod
    def get_steps(options):
        yield CombineRefinedBreakpointsStep(options)
        
    def __init__(self, options):
        self.options = options


    def __str__(self):
        return ".".join([self.__class__.__name__])
        
    def outpaths(self, final):
        directory = self.results_dir if final \
                    else self.working_dir

        paths = {
            "refined_pairs": os.path.join(directory, "refined_pairs.tsv")
        }

        return paths


    def run(self):
        inputs = []
        
        chroms = self.options.reference.chroms

        for chromx, chromy in itertools.product(chroms, chroms):
            if self.options.reference.compare_chroms(chromx, chromy) < 0: continue
            input_step = RefineGridSearchBreakpointsStep(self.options, chromx, chromy)
            inpath = input_step.outpaths(final=True)["refined_pairs"]

            try:
                inputs.append(pandas.read_table(inpath))
            except pandas.io.common.EmptyDataError:
                pass
        if len(inputs) == 0:
            raise Exception("No candidate SVs discovered.")

        combined = pandas.concat(inputs)
        combined["chromx"] = combined["chromx"].astype("string")
        combined["chromy"] = combined["chromy"].astype("string")

        combined.to_csv(self.outpaths(final=False)["refined_pairs"], sep="\t", index=False)
        

class RefineGridSearchBreakpointsStep(step.StepChunk):
    """
    Takes the rough grid search (aka barcode overlaps) candidates, then performs
    breakpoint refinement to find the best potential breakpoint in the expected
    orientation
    """

    # TODO: refactor so that this and the final refine breakpoints steps share 
    # most of the refinement code
    @staticmethod
    def get_steps(options):
        chroms = options.reference.chroms

        for chromx, chromy in itertools.product(chroms, chroms):
            if options.reference.compare_chroms(chromx, chromy) < 0: continue
            yield RefineGridSearchBreakpointsStep(options, chromx, chromy)
        
    def __init__(self, options, chromx, chromy):
        self.options = options
        self.chromx = chromx
        self.chromy = chromy


    def __str__(self):
        return ".".join([self.__class__.__name__,
                     self.chromx,
                     self.chromy])

        
    def outpaths(self, final):
        directory = self.results_dir if final \
                    else self.working_dir

        paths = {
            "refined_pairs": os.path.join(directory, "refined_pairs.{}.{}.tsv".format(self.chromx, self.chromy))
        }

        return paths


    def run(self):
        outpath = self.outpaths(final=False)["refined_pairs"]

        events = self.load_events()
        if len(events) > 0:
            refined = refine_events(events, self.options, self.logger)

            refined.to_csv(outpath, sep="\t", index=False)
        else:
            open(outpath, "w")

    def load_events(self):
        significant_events = []
        cur_events = []
        
        for sample, dataset in self.options.iter_10xdatasets():
            if self.options.reference.compare_chroms(self.chromx, self.chromy) < 0: continue

            input_step = sv_candidates.SVCandidatesStep(self.options, sample, dataset, self.chromx, self.chromy)
            inpath = input_step.outpaths(final=True)["svs"]
            try:
                cur_events.append(pandas.read_table(inpath))
            except pandas.io.common.EmptyDataError:<commit_msg>testing fix for error combining empty sv candidates step file to non-empty sv candidates file<commit_after># FOR GRID SEARCH CANDIDATES

import itertools
import numpy
import os
import pandas
import scipy.stats

from grocsvs import step
from grocsvs import structuralvariants

from grocsvs.stages import sv_candidates


class CombineRefinedBreakpointsStep(step.StepChunk):
    @staticmethod
    def get_steps(options):
        yield CombineRefinedBreakpointsStep(options)
        
    def __init__(self, options):
        self.options = options


    def __str__(self):
        return ".".join([self.__class__.__name__])
        
    def outpaths(self, final):
        directory = self.results_dir if final \
                    else self.working_dir

        paths = {
            "refined_pairs": os.path.join(directory, "refined_pairs.tsv")
        }

        return paths


    def run(self):
        inputs = []
        
        chroms = self.options.reference.chroms

        for chromx, chromy in itertools.product(chroms, chroms):
            if self.options.reference.compare_chroms(chromx, chromy) < 0: continue
            input_step = RefineGridSearchBreakpointsStep(self.options, chromx, chromy)
            inpath = input_step.outpaths(final=True)["refined_pairs"]

            try:
                inputs.append(pandas.read_table(inpath))
            except pandas.io.common.EmptyDataError:
                pass
        if len(inputs) == 0:
            raise Exception("No candidate SVs discovered.")

        combined = pandas.concat(inputs)
        combined["chromx"] = combined["chromx"].astype("string")
        combined["chromy"] = combined["chromy"].astype("string")

        combined.to_csv(self.outpaths(final=False)["refined_pairs"], sep="\t", index=False)
        

class RefineGridSearchBreakpointsStep(step.StepChunk):
    """
    Takes the rough grid search (aka barcode overlaps) candidates, then performs
    breakpoint refinement to find the best potential breakpoint in the expected
    orientation
    """

    # TODO: refactor so that this and the final refine breakpoints steps share 
    # most of the refinement code
    @staticmethod
    def get_steps(options):
        chroms = options.reference.chroms

        for chromx, chromy in itertools.product(chroms, chroms):
            if options.reference.compare_chroms(chromx, chromy) < 0: continue
            yield RefineGridSearchBreakpointsStep(options, chromx, chromy)
        
    def __init__(self, options, chromx, chromy):
        self.options = options
        self.chromx = chromx
        self.chromy = chromy


    def __str__(self):
        return ".".join([self.__class__.__name__,
                     self.chromx,
                     self.chromy])

        
    def outpaths(self, final):
        directory = self.results_dir if final \
                    else self.working_dir

        paths = {
            "refined_pairs": os.path.join(directory, "refined_pairs.{}.{}.tsv".format(self.chromx, self.chromy))
        }

        return paths


    def run(self):
        outpath = self.outpaths(final=False)["refined_pairs"]

        events = self.load_events()
        if len(events) > 0:
            refined = refine_events(events, self.options, self.logger)

            refined.to_csv(outpath, sep="\t", index=False)
        else:
            open(outpath, "w")

    def load_events(self):
        significant_events = []
        cur_events = []
        
        for sample, dataset in self.options.iter_10xdatasets():
            if self.options.reference.compare_chroms(self.chromx, self.chromy) < 0: continue

            input_step = sv_candidates.SVCandidatesStep(self.options, sample, dataset, self.chromx, self.chromy)
            inpath = input_step.outpaths(final=True)["svs"]
            try:
                sample_events = pandas.read_table(inpath)
                if len(sample_events) > 0:
                    cur_events.append(sample_events)
            except pandas.io.common.EmptyDataError:
{'fraction_non_alphanumeric': 0.09466853289222547, 'fraction_numerical': 0.001908639776052933, 'mean_word_length': 2.35753951302862, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7655029', 'n_tokens_mistral': 2357, 'n_tokens_neox': 2206, 'n_words': 502}