<jupyter_start><jupyter_text>Steps towards defining the potential subtypes of ASD on the basis of frequent patient profiles  1. Connecting to Claims database and select the ASD and non-ASD participants  A. Connecting to the databaseWe identify the ASD and non-ASD patients from Claims database.<jupyter_code>library(SqlServerJtds)
library(SqlTools)
library(FactToCube)

my.username="****"
my.password="*********"

##connecting to the database
cn = connect.sql.server(
  database="********", 
  domain="***", 
  server.address="*************", 
  user=my.username,
  password=my.password)<jupyter_output><empty_output><jupyter_text>B. Selecting ASD patientsThe criteria for selecting the ASD patients is:1. to have at least 3 times diagnosis of ASD2. having age between 0-183. Being in the system for at least 12 monthsThen we statirfied the population into different age groups of 0-2, 3-5, 6-11,and 12-18<jupyter_code>## Selecting all the ASD patients having ICD9-CM diagnosis records

dbSendUpdate(cn,"select  
             distinct  F.MemberId 
             INTO #tmpAsd
             FROM 
             FactIcd F 
             WHERE 
             F.Icd IN ('299.0','299.00','299.01','299.8','299.9', '299.80','299.81','299.90', '299.91')")

##Selecting the ones having age between 0-18

dbSendUpdate(cn,"select T.MemberId,YEAR(F.DateServiceStarted)-M.BirthYear as Age, M.Gender, F.DateServiceStarted, F.Icd
into #tmpAsdAge
from #tmpAsd T
inner join Members M
on M.MemberId= T.MemberId
inner join FactIcd F
on F.MemberId=T.MemberId
where  YEAR(F.DateServiceStarted)-M.BirthYear<=18")


##having at least 3 times ASD diagnosis based on ICD9 code

dbSendUpdate(cn,"select distinct AC.MemberId into #asd3T from (select distinct AC.MemberId, AC.Icd, AC.DateServiceStarted 
  from #tmpAsdcode AC AC
  where AC.Icd IN ('299.0','299.00','299.01','299.8','299.9', '299.80','299.81','299.90', '299.91') group by AC.MemberId
  having count(*)>=3")


###Removing the ones with unknown gender or being in the system for less than 12 months

dbSendUpdatey(cn, "select E.MemberId,M.BirthYear 
INTO #atE
FROM Enrollment E 
INNER JOIN Members M ON
E.MemberId = M.MemberId 
INNER JOIN #asd3T A ON
M.MemberId = A.MemberId
WHERE
Year(E.EffectiveDate) - M.BirthYear <= 18 and 
E.MedicalIndicator='Y' and
M.Gender in ('M','F')
GROUP BY E.MemberId,M.BirthYear
HAVING  count (distinct E.EffectiveDate)>12")

dbSendUpdate(cn,"select AC.Age,AC.DateServiceStarted,AC.Icd, AC.Gender, aE.MemberId,aE.BirthYear into #ASDF 
from  #tmpAsdcode AC 
inner join #atE aE 
on AC.MemberId=aE.MemberId")

##saving the ASD table in new database
dbSendUpdate (cn,"SELECT * INTO nj.dbo.ASDFinal FROM #ASDF") 


###Adding the phecode table to the database

phecode=read.csv(file="phecode_icd9_rolled.csv",sep=",")
names(phecode)[2]="ICD9String"
dbWriteTable(cn,"nj.dbo.phecode",phecode[,c(1,2,3,4)])

dbSendUpdate(cn,"Select * into #tmp from nj.dbo.noASDFinal 
                          F inner join nj.dbo.phecode P on F.Icd=P.ICD9" )

dbSendUpdate (cn,"select * into nj.dbo.noASDFinal from #tmp")

#####Creating the different groups of Age

dbSendUpdate (cn,"select * into #ASDG1 from nj.dbo.ASDFinal where Age>=0 and Age<=2")
dbSendUpdate (cn,"select * INTO nj.dbo.ASDG1 FROM #ASDG1")


dbSendUpdate (cn,"select * into #ASDG2 from nj.dbo.ASDFinal where Age>=3 and Age<=5")
dbSendUpdate (cn,"select * INTO nj.dbo.ASDG2 FROM #ASDG2")


dbSendUpdate (cn,"select * into #ASDG3 from nj.dbo.ASDFinal where Age>=6 and Age<=11")
dbSendUpdate (cn,"select * INTO nj.dbo.ASDG3 FROM #ASDG3")


dbSendUpdate (cn,"select * into #ASDG4 from nj.dbo.ASDFinal where Age>=12 and Age<=18")
dbSendUpdate (cn,"select * INTO nj.dbo.ASDG4 FROM #ASDG4")<jupyter_output><empty_output><jupyter_text>C. Selecting non ASD patientsSimilarly for selecting the Non-ASD patients, we identify individuals:1. having no diagnosis record of ASD 2. having age 0-18, 3. being at least 12 months in the systemSimilar to ASD, we stratified patients into 4 different age groups and select the sample (10 times population of ASD) as the control<jupyter_code>## selecting all the patients who are having the age between 0-18, being in the system for at least 12 months
dbSendUpdate (cn,"Select E.MemberId, M.BirthYear into #Efix 
from Enrollment E
inner join Members M
on E.MemberId=M.MemberId
WHERE
Year(E.EffectiveDate) - M.BirthYear >=0 and 
Year(E.EffectiveDate) - M.BirthYear <= 18 and 
E.MedicalIndicator='Y' and
M.Gender in ('M','F') 
GROUP BY E.MemberId,M.BirthYear
HAVING  count (distinct E.EffectiveDate)>12")

##Remove the ones had atleast one diagnosis of ASD
dbSendUpdate (cn,"Select Ef.MemberId into #noASDId from #Efix Ef 
WHERE MemberId not in (select MemberId from #tmpAsdMembers)")


#Save the results in the database
dbSendUpdate (cn,"SELECT * INTO nj.dbo.noASDId FROM #noASDId")

## adding age, gender and birthyear of patients from Member Id table

dbSendUpdate (cn,"select M.MemberId, M.BirthYear, M.Gender into #NB from Members M
where MemberId in (select MemberId from nj.dbo.noASDId)")


##Creating the different age group
dbSendUpdate (cn,"Select B.MemberId,B.BirthYear,B.Gender, 
Year(F.DateServiceStarted)-B.BirthYear as Age, F.Icd, F.DateServiceStarted
into #NoasdGroup1
from #NB B inner join FactIcd F
on B.MemberId=F.MemberId
Where Year(F.DateServiceStarted)-B.BirthYear>=0 and
Year(F.DateServiceStarted)-B.BirthYear<=2")

dbSendUpdate (cn,"SELECT * INTO nj.dbo.noasdF1 FROM #NoasdGroup1")



dbSendUpdate (cn,"Select B.MemberId,B.BirthYear,B.Gender, 
Year(F.DateServiceStarted)-B.BirthYear as Age, F.Icd, F.DateServiceStarted
into #NoasdGroup2
from #NB B inner join FactIcd F
on B.MemberId=F.MemberId
Where Year(F.DateServiceStarted)-B.BirthYear>=3 and
Year(F.DateServiceStarted)-B.BirthYear<=5")

dbSendUpdate (cn,"SELECT * INTO nj.dbo.noasdF2 FROM #NoasdGroup2")


dbSendUpdate (cn,"Select B.MemberId,B.BirthYear,B.Gender, 
Year(F.DateServiceStarted)-B.BirthYear as Age, F.Icd, F.DateServiceStarted
into #NoasdGroup3
from #NB B inner join FactIcd F
on B.MemberId=F.MemberId
Where Year(F.DateServiceStarted)-B.BirthYear>=6 and
Year(F.DateServiceStarted)-B.BirthYear<=11")

dbSendUpdate (cn,"SELECT * INTO nj.dbo.noasdF3 FROM #NoasdGroup3")



dbSendUpdate (cn,"Select B.MemberId,B.BirthYear,B.Gender, 
Year(F.DateServiceStarted)-B.BirthYear as Age, F.Icd, F.DateServiceStarted
into #NoasdGroup4
from #NB B inner join FactIcd F
on B.MemberId=F.MemberId
Where Year(F.DateServiceStarted)-B.BirthYear>=12 and
Year(F.DateServiceStarted)-B.BirthYear<=18")

dbSendUpdate (cn,"SELECT * INTO nj.dbo.noasdF4 FROM #NoasdGroup4")


##Since the total number of patients in this group is much more than the ASD patients, 
#we select the 10 times population of ASD, as the number of non-ASD patients


dbSendUpdate (cn,"select distinct top 91560 MemberId  into #pt1 from noASDG1")
dbSendUpdate (cn,"select * into #temp1 from noASDG1 where MemberId in (select MemberId from #pt1")
dbSendUpdate(cn,"Select * into #tmp1 from nj.dbo.noasdF1 
                          F inner join nj.dbo.phecode P on F.Icd=P.ICD9")
  
dbSendUpdate (cn,"select * into nj.dbo.noasdT1 from #tmp1")



dbSendUpdate (cn,"select distinct top 208300 MemberId  into #pt2 from noASDG2")
dbSendUpdate (cn,"select * into #temp2 from noASDG2 where MemberId in (select MemberId from #pt2")
dbSendUpdate(cn,"Select * into #tmp2 from nj.dbo.noasdF2 
                          F inner join nj.dbo.phecode P on F.Icd=P.ICD9")
  
dbSendUpdate (cn,"select * into nj.dbo.noasdT2 from #tmp2")



dbSendUpdate (cn,"select distinct top 378040 MemberId  into #pt3 from noASDG3")
dbSendUpdate (cn,"select * into #temp3 from noASDG3 where MemberId in (select MemberId from #pt3")
dbSendUpdate(cn,"Select * into #tmp3 from nj.dbo.noasdF3 
                          F inner join nj.dbo.phecode P on F.Icd=P.ICD9")
  
dbSendUpdate (cn,"select * into nj.dbo.noasdT3 from #tmp3")



dbSendUpdate (cn,"select distinct top 291350 MemberId  into #pt4 from noASDG4")
dbSendUpdate (cn,"select * into #temp4 from noASDG4 where MemberId in (select MemberId from #pt4")
dbSendUpdate(cn,"Select * into #tmp4 from nj.dbo.noasdF4 
                          F inner join nj.dbo.phecode P on F.Icd=P.ICD9")
  
dbSendUpdate (cn,"select * into nj.dbo.noasdT4 from #tmp4")<jupyter_output><empty_output><jupyter_text>D. Creating the count matrix for different age groupFor each group of ASD and non-ASD, we create the matrix defining the number of occurrence of different comorbidities for each patients.<jupyter_code>##SELECTING THE COUNT MATrix ( showing the number of doagnosis of different comorbidities for ASD patients)
for( t in 1:4){
  dbSendUpdate(cn,sprintf("select distinct MemberId, PheCode, DateServiceStarted,Gender
                          into #asdm from  nj.dbo.ASDG%d G1", t))

  dbSendUpdate(cn,"select MemberId, PheCode, Gender, count(PheCode)as n into #temp from #asdm
               group by MemberId, Phecode, Gender")

  dbSendUpdate(cn,sprintf("SELECT * INTO nj.dbo.ASDcountG%d FROM #temp", t))
  dbSendUpdate(cn,"drop table #asdm")
  dbSendUpdate(cn,"drop table #temp")
}

##SELECTING THE COUNT MATrix ( showing the number of doagnosis of different comorbidities for non-ASD patients)

for( t in 1:4){
  dbSendUpdate(cn,sprintf("select distinct MemberId, PheCode, DateServiceStarted,Gender
                          into #asdm from  nj.dbo.noASDlG%d G1", t))
  
  dbSendUpdate(cn,"select MemberId, PheCode, Gender, count(PheCode)as n into #temp from #asdm
               group by MemberId, Phecode, Gender")
  
  dbSendUpdate(cn,sprintf("SELECT * INTO nj.dbo.noASDlcountG%d FROM #temp", t))
  dbSendUpdate(cn,"drop table #asdm")
  dbSendUpdate(cn,"drop table #temp")
}<jupyter_output><empty_output><jupyter_text>2. Phenome Wide Association Study with Respect to different Age groupsTo Identify the significant comorbidity associated to ASD, we perform the Phenome Wide Assocation study, by taking into account, statistical comparison between ASD and non-ASD. To this aim, for each comorbidity, we perform the logistic regression with respect to the sex. To be considered as positive diagnosis, at least 3 times of diagnsosi in different dates is required.  comorbidities that meet the bonferronin adjusted  p-value of 0.01 and OR>2 are defined as the significant results.<jupyter_code>## We use the parallel computing approach to perfomr the PheWAS. (n,k) are the arguman of the paraller process
n = as.numeric(commandArgs(trailingOnly=TRUE)[2])
k = as.numeric(commandArgs(trailingOnly=TRUE)[1])

library(tidyr)
library(data.table)
library(SqlServerJtds)
library(SqlTools)
library(FactToCube)
library(dplyr)



my.username="****"
my.password="********"

cn = connect.sql.server(
  database="****", 
  domain="***", 
  server.address="***************", 
  user=my.username,
  password=<PASSWORD>)

## for each group of age we perfomr the phewas: t={1,2,3,4}
t=1

asdMat  = dbGetQuery(cn,sprintf("Select  * from nj.dbo.ASDcountG%d",t))
noasdMat= dbGetQuery(cn, sprintf("Select * from nj.dbo.noasdcountG%d",t))

## Selecting the common comorbidity between ASD and non-ASD groups
pheno=intersect(unique(asdMat$PheCode),unique(noasdMat$PheCode))
asdMat=asdMat[asdMat$PheCode %in% pheno,]
noasdMat=noasdMat[noasdMat$PheCode %in% pheno,]

#catln = function(...)cat(...,"\n")
# ns=20

Phewas=function(input,condition){
  D3=data.table::dcast(data.table::as.data.table(input),input$MemberId~input$PheCode,value.var="n")
  D4=as.data.frame(D3)
  D4[is.na(D4)] <- 0
  D4=dplyr::rename(D4,MemberId=input)
  inputG=unique(input[,c("MemberId","Gender")])
  inputG$Condition=rep(condition,nrow(inputG))
  DataOut=merge(inputG,D4,by="MemberId")
  return(DataOut)
}

noasd_data=Phewas(noasdMat,"A-")
asd_data=Phewas(asdMat,"A+")


rm(asdMat)
rm(noasdMat)

DataMatrix=rbind(asd_data,noasd_data)


## delete unneeded variables and call the garbage collector
rm(noasd_data)
rm(asd_data)
gc()


################## END OF DATA PREPARATION ##################
  
  require (data.table)
  input=DataMatrix[,1:3]
  input=as.data.frame(input)
  output=DataMatrix[,4:ncol(DataMatrix)]
  Pheno=colnames(output)
  ## Pheno=Pheno[-1]
  
  Reg=function(x){
    newTable=data.frame(cbind(output[,x],input))
    names(newTable)[1]="x"
    newTable=newTable[!(newTable[,1]>0 & newTable[,1]<3),]
    newTable[,1]=ifelse(newTable[,1] >=3,1,0)
    newTable=as.data.frame(apply(newTable,2, as.factor))
    fit <- glm(x~Gender+Condition , data=newTable,family="binomial")
    pval=data.frame(summary(fit)$coefficients[,4])  #p value
    or=data.frame(round(exp(coef(fit)),2))          # odd ratio
    cI=round(exp(confint.default(fit,level = 0.95)),2)
    a= length(unique(newTable[newTable$Condition=="A+" & newTable[,1]==1,"MemberId"]))      # total ASD patients that have phenotype
    b= length(unique(newTable[newTable$Condition=="A-" & newTable[,1]==1,"MemberId"]))      # total non-ASD patients that have phenotype
    c= length(unique(newTable[newTable$Condition=="A+" & newTable[,1]==0,"MemberId"]))      # total ASD patients that doesnt have phenotype
    d= length(unique(newTable[newTable$Condition=="A-" & newTable[,1]==0,"MemberId"])) 
    # n
    row=list(
      "Phenotype" = paste(x),
      "A+OR" = or["ConditionA+",],
      "A+OR(95% CI)" = paste0("(",cI["ConditionA+",1],"-",cI["ConditionA+",2],")"), #OR for Condition
      "GMOR" = or["GenderM",],
      "GM(95% CI)" = paste0("(",cI["GenderM",1],"-",cI["GenderM",2],")"),
      "A+Pval" = pval["ConditionA+",], #Pval for Condition
      "GMPval" = pval["GenderM",],
      "Pheno+" = a+b,
      "tPheno+" = paste0("(",a,"/",b,")"),
      "Pheno-" = c+d,
      "tPheno-" = paste0("(",c,"/",d,")")
    )
    
    # print(paste(which(Pheno==x),":", paste(row, collapse = ',')))
    return(row)
  }
  
  
  B=length(Pheno)
  size=ceiling(B/as.numeric(n))
  k=as.numeric(k)
  require(data.table)
  
  PhenoArray = na.omit(Pheno[ ((k-1)*size+1) : (k*size) ])
  
  V=lapply(PhenoArray, function(x){
    row = Reg(x)
    gc()
    return(row)
  })
  


WW=rbindlist(V)


####We change the name of the PheWAS file with respect to different age group

write.table(WW, file = sprintf("phewasG1_%02d.csv",k),
            col.names= k == 1,
            row.names=F,
            sep = ',',
            dec = '.')

print(paste(Sys.time(), "end of",k))<jupyter_output><empty_output><jupyter_text>3. Creating the patient profiles with respect to different age groupsOur selection criteria for identifying the comorbdities associated to ASD, is to have OR>1.5 and being diagnosed at least more than 10 patients.We remove some comorbidities that are not meaningful or vague. The diagnosis profile of each patient is identified by different co-occurrences of those comorbidities.We perform the similar approach for both ASD and non-ASD group for different ages.<jupyter_code>library(tidyr)
library(data.table)
library(SqlServerJtds)
library(SqlTools)
library(FactToCube)
my.username="****"
my.password="********"

cn = connect.sql.server(
  database="****", 
  domain="***", 
  server.address="***************", 
  user=my.username,
  password=<PASSWORD>)


## removing the non-meaningful comorbidities from the PheWAS results
vouge=c(
  "Random mental disorder. Ignored for now",
  "Abnormal findings on study of brain, nervous system",
  "Foreign body injury",
  "Complications of gastrostomy, colostomy and enterostomy",
  "Symptoms involving nervous and musculoskeletal systems",
  "Symptoms concerning nutrition, metabolism, and development",
  "Symptoms involving digestive system",
  "Symptoms of the muscles",
  "Other tests",
  "Effects of other external causes",
  "Other endocrine disorders",
  "Other symptoms",
  "Other conditions of brain",
  "Other diseases of lung",
  "Other disorders of metabolic, endocrine, immunity disorders",
  "Persistent mental disorders due to other conditions",
  "Other persistent mental disorders due to conditions classified elsewhere",
  "Other persistent mental disorders due to conditions classified elsewhere",
  "Other disorders of the nervous system",
  "Other nutritional deficiency",
  "Other ill-defined and unknown causes of morbidity and mortality",
  "Complications of surgical and medical procedures",
  "Nonspecific abnormal findings on radiological and other examination of skull and head",
  "Insect bite","Nonspecific abnormal results of other endocrine function study",
  "Burns","Calculus of ureter","Chemotherapy","Insulin pump user",
   "Nonspecific abnormal results of function study of brain and central nervous system",
   "Abnormal findings on study of brain and/or nervous system",
  "Other disorders of metabolism",
    "Other and unspecified disorders of the nervous system",
  "Other conditions of brain, NOS",
  "Late effects of cerebrovascular disease",
  "Other and unspecified disorders of the nervous system",
  "Other and unspecified congenital anomalies", "Ingrowing nail",
 "Transient mental disorders due to conditions classified elsewhere"
)

####by changing the t from 1 to 4 we can identify the different patient profiles for all groups of ages respectively
t=1

ASD=dbGetQuery(cn, 	sprintf("
               SELECT
               *
               FROM
               nj.dbo.ASDG%d",t)
)



##selectig the associated comorbidities having OR>1.5 and appearing in more than 10 patients
table=read.csv(file=sprintf("phewasG%s.csv",t),sep=",")
table=table[,-c(1)]
table=table[table$A.OR.95..CI.!="(0-Inf)",]
table$asdCount=gsub("[][()]", "",table$tPheno. )
table$asdCount=sapply(table$asdCount,function(x) unlist(strsplit(x, "/"))[1])
table=table[,c(12,2,3,6,9,13,4,7,5,8,10,11,1)]
tableAss=table[table$A.OR>1.5& as.numeric(table$asdCount)>10,]
tableAss=tableAss[!tableAss$Phenotype %in% vouge,]

PPA=ASD[ASD$Phenotype %in% tableAss$Phenotype, ]
PPA=unique(PPA[,c("MemberId","Phenotype","DateServiceStarted")])
At1=PPA[,c("MemberId","Phenotype")] 
At1= data.frame(table(At1))
At1$Freq = ifelse(At1$Freq >= 3, 1, 0)
At1=At1[At1$Freq==1,]
At1=At1[,c(1,2)]

dbWriteTable(cn,sprintf("#countG%d",t),At1)
At2=dbGetQuery(cn,
               sprintf("select MemberId, string_agg(Phenotype,' * ') as phenotype from #countG%d  group by MemberId",t))

At2$x=sapply(At2$phenotype,function(x) paste(collapse=' * ', sort(strsplit(as.character(x), ' * ', fixed=T)[[1]])))
dd=plyr::count(At2$phenotype)
save(dd,file=sprintf("ASDpprofileGr%d.RData",t))<jupyter_output><empty_output><jupyter_text>4. Defining the Potential Subtypes of ASD for different groups of agesWe identify the potential subtypes of ASD, on the basis of frequent patient profiles. to do so:1. we define the mutual patient profiles among ASD and non-ASD patients (for each age group)2. we define the profiles that are associated to ASD (fisher test, Pval1.5)3. we select the group of patient profiles that are only appeared in ASD or the ones that appeares in ASD and non-ASD but significantly associated to ASD.4. we examine different values of cutoff between (3-40), for the  minimum  number of patients in associated patient profiles.5. we perform the similarity measurement between:        a. associated patient profiles meet the cutoff criteria,         b. the rest of the patients profiles (not associated OR associated BUT not meeting the cut off criteria)6. Similarity measurement process:        I. each profile of group b is compared with all the profiles in group a        II. the score of similarity is defined by  (number of similar comorbidity)/(total number of comorbidity in the profile of group a)        III.the profile that has the maximum score (at least more than 0.6) can represent the profile in group b.        IV. patients who are defined by the profile b are now defined by similar profile in a7. for each cut off value, we see how many profiles in group a is determined and what percent of the population are characterized at the end of similarity calculation.8. Gain of cutoff (3-40): (-1)* (number of profile) +(percent of the characterized population)9. the optimum cut off is the one that has the maximum gain  (1-4). Defining the  mutual patient profiles  and selecting the ones associated to ASD<jupyter_code>## changing the value of k from 1:4 will identify the potential ASD subgroups for different age groups.

k=1
## the datafiles contain the list of profile and count of the patients
load(sprintf("~/ASDpprofileGr%d.RData",k)) 
aspp=dd
load(sprintf("~/noASDpprofileGr%d.RData",k))
noaspp=dd
library(dplyr)

aspp$x= sapply(aspp$x,function(x) paste(collapse=' * ', sort(strsplit(as.character(x), ' * ', fixed=T)[[1]])))
noaspp$x= sapply(noaspp$x,function(x) paste(collapse=' * ', sort(strsplit(as.character(x), ' * ', fixed=T)[[1]])))

aspp%>% group_by(x) %>% summarise_all(sum) %>% data.frame()->aspp
noaspp%>% group_by(x) %>% summarise_all(sum) %>% data.frame()->noaspp

## fisher test between Similar patient profiles in ASD ad non-ASD groups
common=aspp[aspp$x %in% noaspp$x, ]
names(common)[2]="AsD"
common=merge(common,noaspp,by="x")
names(common)[3]="noAsD"
common$ANpair=sum(aspp$freq)-common$AsD
common$NANpair=sum(noaspp$freq)-common$noAsD
common$Pval=apply(common[,2:5],1,function(x) fisher.test(matrix(c(x[1],x[2],x[3],x[4]),nrow=2))$p.value)
common$OR=apply(common[,2:5],1,function(x) fisher.test(matrix(c(x[1],x[2],x[3],x[4]),nrow=2))$estimate)
                
## we select the associated patient profiles by common1 and non-associated patient profiles by common2
common1=common[common$OR>1.5 & common$Pval<0.01/nrow(common),]
common2=common[!common$x %in% common1$x,]<jupyter_output><empty_output><jupyter_text>5. Similarity measurement<jupyter_code>threshold=function(t){

  commonref=aspp[aspp$freq>=t,]
## the reference are defined by associated patient profiles that meet the threshold (cut off) criteria
    
  commonref=commonref[!commonref$x %in% common2$x,]
  commontest=dplyr::setdiff(aspp,commonref)
  
  Ref=sapply(commonref$x, function(x) strsplit(as.character(x), " \\* "))
  Test=sapply(commontest$x, function(x) strsplit(as.character(x), " \\* "))
 
## Similarity measurements between group b (commontest) and group a (commonref)
              
  p=lapply(Test,function(x) sapply(Ref,function(y) length(intersect(x,y))))
  RefLengths = sapply(Ref, length) ## defining the length of each profiles in group a

# for every profile in group b:
      RefRow=sapply(p, function(x){
        if (max(x)<=1){
          # if the maximum # of identical comorbidities is less than 2, whatever
          0
        }else{
          # index=which(x==max(x))
          # good = sapply(index,function(y) ifelse((max(x)/length(Ref[[y]]))>0.6,y,0))
          percent = x / RefLengths
          if(max(percent) <= 0.6){
            return(0)
          }else{
            # goodindices is the indices of x that have the maximum percentage in common with the Ref
            goodindices = which(percent==max(percent))
            # if(length(goodindices) > 1 ) print(sprintf('%s: there is more than one good stuff here, returning only the first one', x))
            
            df = data.frame(id = goodindices,
                            length = RefLengths[goodindices])
            
            return(df[ df$length == max(df$length), 'id' ][1])
          }
          
        }
      }
      )
      
      for(i in 1:length(RefRow)){ 
        if(i>0){
          commonref[RefRow[i],2]=commontest[i,2]+commonref[RefRow[i],2]
        }
      }<jupyter_output><empty_output><jupyter_text>6. Examining the different value of cutoff<jupyter_code>d=sapply(commonref$x, function(x) strsplit(as.character(x), " \\* "))
D=data.table::rbindlist(lapply(d, function(x) data.frame(x)))
D=data.frame(unique(D$x))
names(D)[1]="x"
D$frq=sapply(D$x, function(x) sum(aspp[grep(x,aspp$x,fixed = T),"freq"]))
D$frq2=sapply(D$x, function(x) sum(commonref[grep(x,commonref$x,fixed = T),"freq"]))
result=data.frame("threshod"=sprintf("x>=%d",t),
"TotalComorbidity"=nrow(D),
"TotalPP"=nrow(commonref),
"Totalcoverage"=round(100*sum(commonref$freq)/sum(aspp$freq)))
return(list(commonref,result))
}

t=c(3:40)
L=lapply(t,threshold)

V1=lapply(L,function(x) list(x[[1]]))
CC=data.frame()
for (i in 1:length(L)){CC= rbind(CC, L[[i]][[2]])}<jupyter_output><empty_output><jupyter_text>(7-9). defining the optimum cutoff<jupyter_code>norm=function(x,mean,sd){return((x-mean)/sd)}
## normalizing the results of number of profiles

m1=mean(CC[,3])
sd1=sd(CC[,3])
NS=sapply(CC[,3],function(x) norm(x,m1,sd1))
## normalizing the results of percentage of characterized patients 
          
m2=mean(CC[,4])
sd2=sd(CC[,4])
PC=sapply(CC[,4],function(x) norm(x,m2,sd2))
## Defining the gain of each cut off  from 3-40 
## minimizing the total number of profiles and maximizing the percentage of classified patients
normres=(-NS+PC)
          
## the optimum value define the total number of patient profiles and well as the total number of classified patients          
optimumval=CC[which(normres==max(normres)),"threshod"]<jupyter_output><empty_output><jupyter_text>After defining the optimum cutoff we can assign their associated patient profiles as the potential subtypes of ASD.V1 is showing the list of different sets of patient profiles, as a value of different cut off<jupyter_code>sessionInfo()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13356626697371557, 'fraction_numerical': 0.020090526482966727, 'mean_word_length': 4.700995925758262, 'pattern_counts': {'":': 1, '<': 64, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '597589', 'license': 'Apache-2.0', 'n_tokens_mistral': 9040, 'n_tokens_neox': 8558, 'n_words': 2367, 'path': 'Identifying the potential Subtypes of ASD.ipynb', 'repo_name': 'hms-dbmi/Large-scale-subtyping-of-ASD-in-different-Age-groups', 'size': '25188'}