<commit_before>import camoco as co
import pandas as pd
from camoco.Tools import DummyRefGen,log
from camoco.Locus import Gene

def build_cob(args):
    # Build the refgen
    refgen = co.RefGen(args.refgen)
    # Check that the sep is likely right.
    if len(pd.read_table(args.filename,sep=args.sep).columns) == 1:
        print(
            ("Detected only 1 column in {}, are you sure "
            "colunms are separated by '{}'?").format(args.filename,args.sep)
        )
        return None
    if args.allow_non_membership:
        refgen = refgen.copy(
            '{}_tmp'.format(refgen.name), 
            'temp refgen'.format(refgen.name)
        )
        # Add non membership genes
        for gid in pd.read_table(args.filename,sep=args.sep).index:
            refgen.add_gene(Gene(None,None,id=gid))

    quality_control = False if args.skip_quality_control else True
    normalize = False if args.skip_normalization else True
        
    # Basically just pass all the CLI arguments to the COB class method  
    cob = co.COB.from_table(
        args.filename,
        args.name,
        args.description,
        refgen,
        # Optional arguments
        sep=args.sep,
        rawtype=args.rawtype,
        # Data Processing
        quality_control=quality_control,
        normalization=normalize,
        quantile=args.quantile,
        # Data processing parameters
        max_gene_missing_data=args.max_gene_missing_data,
        max_accession_missing_data=args.max_accession_missing_data,
        min_single_sample_expr=args.min_single_sample_expr,
        min_expr=args.min_expr,
        max_val=args.max_val,
        dry_run=args.dry_run,
        index_col=args.index_col
    )
    print("Build successful!")
    print(cob.summary())

def build_refgen(args):<commit_msg>remove data structures when using the build command if they already exist<commit_after>import camoco as co
import pandas as pd
from camoco.Tools import DummyRefGen,log
from camoco.Locus import Gene

def build_cob(args):
    # Build the refgen
    refgen = co.RefGen(args.refgen)
    # Check that the sep is likely right.
    if len(pd.read_table(args.filename,sep=args.sep).columns) == 1:
        print(
            ("Detected only 1 column in {}, are you sure "
            "colunms are separated by '{}'?").format(args.filename,args.sep)
        )
        return None
    if args.allow_non_membership:
        refgen = refgen.copy(
            '{}_tmp'.format(refgen.name), 
            'temp refgen'.format(refgen.name)
        )
        # Add non membership genes
        for gid in pd.read_table(args.filename,sep=args.sep).index:
            refgen.add_gene(Gene(None,None,id=gid))

    quality_control = False if args.skip_quality_control else True
    normalize = False if args.skip_normalization else True

    # Check to see if this dataset is already built
    if co.available_datasets('Expr',args.name):
        print('Warning! This dataset has already been built.')
        co.del_dataset('Expr',args.name,safe=args.force)
        
    # Basically just pass all the CLI arguments to the COB class method  
    cob = co.COB.from_table(
        args.filename,
        args.name,
        args.description,
        refgen,
        # Optional arguments
        sep=args.sep,
        rawtype=args.rawtype,
        # Data Processing
        quality_control=quality_control,
        normalization=normalize,
        quantile=args.quantile,
        # Data processing parameters
        max_gene_missing_data=args.max_gene_missing_data,
        max_accession_missing_data=args.max_accession_missing_data,
        min_single_sample_expr=args.min_single_sample_expr,
        min_expr=args.min_expr,
        max_val=args.max_val,
        dry_run=args.dry_run,
        index_col=args.index_col
    )
    print("Build successful!")
    print(cob.summary())

def build_refgen(args):
{'fraction_non_alphanumeric': 0.08743874129481558, 'fraction_numerical': 0.0010317255610007739, 'mean_word_length': 2.5940685820203893, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1526066', 'n_tokens_mistral': 1201, 'n_tokens_neox': 1171, 'n_words': 267}