<commit_before>import pandas as pd
from natsort import natsorted
import numpy as np
from collections import defaultdict, OrderedDict

import logging


try:
    # ray.init(logging_level=logging.CRITICAL) # logging_level=logging.CRITICAL # local_mode=True
    import ray
    ray.init(local_mode=True, logging_level=logging.CRITICAL) # logging_level=logging.CRITICAL # local_mode=True
except:
    import pyranges.raymock as ray

from tabulate import tabulate


# from pyranges.settings import pyranges_settings
from pyranges.genomicfeatures import GenomicFeaturesMethods
from pyranges.subset import get_string, get_slice, get_tuple
# from pyranges.methods import _cluster, _subtraction, _set_union, _set_intersection, _intersection, _nearest, _coverage, _overlap_write_both, _overlap, _tss, _tes, _jaccard, _lengths, _slack
from pyranges.multithreaded import (_cluster, pyrange_apply_single, _write_both, _jaccard, _coverage,
                                    _intersection, pyrange_apply, _nearest, _overlap, _first_df, _subtraction, _tss, _tes, _slack, _sort, merge_dfs, _concat)

def fill_kwargs(kwargs):

    if not "strandedness" in kwargs:
        kwargs["strandedness"] = "same"
    if not "suffix" in kwargs:
        kwargs["suffix"] = "_b"
    if not "overlap" in kwargs:
        kwargs["overlap"] = True
    if not "how" in kwargs:
        kwargs["how"] = None
    if not "invert" in kwargs:
        kwargs["invert"] = None
    if not "new_pos" in kwargs:
        kwargs["new_pos"] = None
    if not "suffixes" in kwargs:
        kwargs["suffixes"] = ["_a", "_b"]
    if not "suffix" in kwargs:
        kwargs["suffixes"] = ["_b"]

    return kwargs

def return_copy_if_view(df, df2):
    # https://stackoverflow.com/questions/26879073/checking-whether-data-frame-is-copy-or-view-in-pandas

    if df.values.base is df2.values.base:
        return df.copy(deep=True)
    else:
        return df


def create_df_dict(df):

    if "Strand" in df:
        grpby_key = "Chromosome Strand".split()
    else:
        grpby_key = "Chromosome"

    return {k: v for k, v in df.groupby(grpby_key)}


def create_pyranges_df(seqnames, starts, ends, strands=None):

    if isinstance(seqnames, str):
        seqnames = pd.Series([seqnames] * len(starts), dtype="category")

    if strands != None and strands != False:

        if isinstance(strands, str):
            strands = pd.Series([strands] * len(starts), dtype="category")

        columns = [seqnames, starts, ends, strands]
        lengths = list(str(len(s)) for s in columns)
        assert len(set(lengths)) == 1, "seqnames, starts, ends and strands must be of equal length. But are {}".format(", ".join(lengths))
        colnames = "Chromosome Start End Strand".split()
    else:
        columns = [seqnames, starts, ends]
        lengths = list(str(len(s)) for s in columns)
        assert len(set(lengths)) == 1, "seqnames, starts and ends must be of equal length. But are {}".format(", ".join(lengths))
        colnames = "Chromosome Start End".split()

    idx = range(len(starts))
    series_to_concat = []
    for s in columns:
        # print(s.head())
        # print(idx)
        if isinstance(s, pd.Series):
            s = pd.Series(s.values, index=idx)
        else:<commit_msg>Fix ValueError series comp<commit_after>import pandas as pd
from natsort import natsorted
import numpy as np
from collections import defaultdict, OrderedDict

import logging


try:
    # ray.init(logging_level=logging.CRITICAL) # logging_level=logging.CRITICAL # local_mode=True
    import ray
    ray.init(local_mode=True, logging_level=logging.CRITICAL) # logging_level=logging.CRITICAL # local_mode=True
except:
    import pyranges.raymock as ray

from tabulate import tabulate


# from pyranges.settings import pyranges_settings
from pyranges.genomicfeatures import GenomicFeaturesMethods
from pyranges.subset import get_string, get_slice, get_tuple
# from pyranges.methods import _cluster, _subtraction, _set_union, _set_intersection, _intersection, _nearest, _coverage, _overlap_write_both, _overlap, _tss, _tes, _jaccard, _lengths, _slack
from pyranges.multithreaded import (_cluster, pyrange_apply_single, _write_both, _jaccard, _coverage,
                                    _intersection, pyrange_apply, _nearest, _overlap, _first_df, _subtraction, _tss, _tes, _slack, _sort, merge_dfs, _concat)

def fill_kwargs(kwargs):

    if not "strandedness" in kwargs:
        kwargs["strandedness"] = "same"
    if not "suffix" in kwargs:
        kwargs["suffix"] = "_b"
    if not "overlap" in kwargs:
        kwargs["overlap"] = True
    if not "how" in kwargs:
        kwargs["how"] = None
    if not "invert" in kwargs:
        kwargs["invert"] = None
    if not "new_pos" in kwargs:
        kwargs["new_pos"] = None
    if not "suffixes" in kwargs:
        kwargs["suffixes"] = ["_a", "_b"]
    if not "suffix" in kwargs:
        kwargs["suffixes"] = ["_b"]

    return kwargs

def return_copy_if_view(df, df2):
    # https://stackoverflow.com/questions/26879073/checking-whether-data-frame-is-copy-or-view-in-pandas

    if df.values.base is df2.values.base:
        return df.copy(deep=True)
    else:
        return df


def create_df_dict(df):

    if "Strand" in df:
        grpby_key = "Chromosome Strand".split()
    else:
        grpby_key = "Chromosome"

    return {k: v for k, v in df.groupby(grpby_key)}


def create_pyranges_df(seqnames, starts, ends, strands=None):

    if isinstance(seqnames, str):
        seqnames = pd.Series([seqnames] * len(starts), dtype="category")

    if strands is not None:

        if isinstance(strands, str):
            strands = pd.Series([strands] * len(starts), dtype="category")

        columns = [seqnames, starts, ends, strands]
        lengths = list(str(len(s)) for s in columns)
        assert len(set(lengths)) == 1, "seqnames, starts, ends and strands must be of equal length. But are {}".format(", ".join(lengths))
        colnames = "Chromosome Start End Strand".split()
    else:
        columns = [seqnames, starts, ends]
        lengths = list(str(len(s)) for s in columns)
        assert len(set(lengths)) == 1, "seqnames, starts and ends must be of equal length. But are {}".format(", ".join(lengths))
        colnames = "Chromosome Start End".split()

    idx = range(len(starts))
    series_to_concat = []
    for s in columns:
        # print(s.head())
        # print(idx)
        if isinstance(s, pd.Series):
            s = pd.Series(s.values, index=idx)
        else:
{'fraction_non_alphanumeric': 0.11203447214527547, 'fraction_numerical': 0.003693444136657433, 'mean_word_length': 3.2560576293385726, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4399571', 'n_tokens_mistral': 2166, 'n_tokens_neox': 2093, 'n_words': 577}