<commit_before>import numpy as np
from optparse import OptionParser
import scipy.linalg as la
import scipy.stats as stats
import sys
import scipy.linalg.blas as blas
import pandas as pd
import time
from pysnptools.snpreader.bed import Bed
import pysnptools.util as pstutil
import pysnptools.util.pheno as phenoUtils
np.set_printoptions(precision=3, linewidth=200)



def loadData(bfile, extractSim, phenoFile, missingPhenotype='-9', loadSNPs=False, keep=None, standardize=False, fileNum=None):
	bed = Bed(bfile, count_A1=True)
	
	if (extractSim is not None):
		try:
			extractSim.union(extractSim)
			extractSnpsSet = extractSim
		except:
			df = pd.read_csv(extractSim, header=None)
			assert len(df.columns)==1, 'extract file must have exactly one column'<commit_msg>added support for older PySnpTools<commit_after>import numpy as np
from optparse import OptionParser
import scipy.linalg as la
import scipy.stats as stats
import sys
import scipy.linalg.blas as blas
import pandas as pd
import time
from pysnptools.snpreader.bed import Bed
import pysnptools.util as pstutil
import pysnptools.util.pheno as phenoUtils
np.set_printoptions(precision=3, linewidth=200)



def loadData(bfile, extractSim, phenoFile, missingPhenotype='-9', loadSNPs=False, keep=None, standardize=False, fileNum=None):
	try: bed = Bed(bfile, count_A1=True)
	except: bed = Bed(bfile)
	
	if (extractSim is not None):
		try:
			extractSim.union(extractSim)
			extractSnpsSet = extractSim
		except:
			df = pd.read_csv(extractSim, header=None)
			assert len(df.columns)==1, 'extract file must have exactly one column'
{'fraction_non_alphanumeric': 0.08364779874213836, 'fraction_numerical': 0.00880503144654088, 'mean_word_length': 5.601659751037344, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4461491', 'n_tokens_mistral': 581, 'n_tokens_neox': 542, 'n_words': 159}