1 This dataset

#SP21
ids21 = read.csv("~/Research/001 NNA Data and Talks Cass/Spring 2021/Spring 2021 Student Data Names with Deidentifiers/deidentifiers_email.csv",header = TRUE)
ids21$email = tolower(gsub("@.*","",ids21$email))
ids21_es = ids21[which( grepl("ES",ids21$deidentifier,fixed = TRUE) ),]
ids21_ob = ids21[which( grepl("OB",ids21$deidentifier,fixed = TRUE) ),]
ids21_ns = ids21[which( grepl("NS",ids21$deidentifier,fixed = TRUE) ),]
ids21_ps = ids21[which( grepl("^S",ids21$deidentifier,fixed = FALSE) ),]
#F22
idf21 = read.csv("~/Research/001 NNA Data and Talks Cass/UMR Data F21/Ochem2/f21_deidentifiers.csv",header = TRUE)
idf21$email = tolower(gsub("@.*","",idf21$email))
idf21_es = idf21[which( grepl("ES",idf21$deidentifier,fixed = TRUE) ),]
idf21_ob = idf21[which( grepl("OB",idf21$deidentifier,fixed = TRUE) ),]
idf21_ns = idf21[which( grepl("NS",idf21$deidentifier,fixed = TRUE) ),]
idf21_ps = idf21[which( grepl("PS",idf21$deidentifier,fixed = TRUE) ),]
#F22
idf22 = read.csv("~/Research/001 NNA Data and Talks Cass/UMR Data F22/BIOC/f22_deidentifiers.csv",header = TRUE)
idf22$email = tolower(gsub("@.*","",idf22$email))

idf22_es = idf22[which( grepl("ES",idf22$deidentifier,fixed = TRUE) ),]
idf22_ob = idf22[which( grepl("OB",idf22$deidentifier,fixed = TRUE) ),]
idf22_ps = idf22[which( grepl("PS",idf22$deidentifier,fixed = TRUE) ),]
idf22_ns = idf22[which( grepl("NS",idf22$deidentifier,fixed = TRUE) ),]

connectIds = function(s21,f21,f22){
  output <- matrix(ncol=4, nrow=nrow(f22))
  for (i in 1:nrow(f22)){
    thisEmail = f22[i,]$email
    #the ID for F21
    thisF21 = f21[grepl(thisEmail,f21$email),]$deidentifier
    if ( identical(thisF21, character(0)) ){ thisF21 = NA}
    #the ID for S21
    thisS21 = s21[grepl(thisEmail,s21$email),]$deidentifier
    if ( identical(thisS21, character(0)) ){thisS21 = NA}
    output[i,1] = thisEmail
    output[i,2] = f22[i,]$deidentifier
    output[i,3] = thisF21
    output[i,4] = thisS21
  }
  output = as.data.frame(output)
  output = output[complete.cases(output),]
  colnames(output) = c("email","f22","f21","s21")
  return(output)
}
id_es = connectIds(ids21_es,idf21_es,idf22_es)
id_ob = connectIds(ids21_ob,idf21_ob,idf22_ob)
id_ns = connectIds(ids21_ns,idf21_ns,idf22_ns)
id_ps = connectIds(ids21_ps,idf21_ps,idf22_ps)

setwd("~/Research/02b Neural Network Research UMR/Data + Analysis/Clustering_Xavier")
all = rbind(id_es,id_ob)
all = rbind(all,id_ns)
all = rbind(all,id_ps)
#write.csv(all,file = "deids_f22_f21_s21.csv")
umr = read.csv("UMR_all_for_R_with_courses.csv", header = TRUE)

addDataAlong = function(deids,surveyName,umr){
  #filter course, term and survey, and grow from it
  alongUMR_1 = umr[which(umr$Course_collected == "Biochem 1" & umr$Term_collected == "Fall2022" & umr$Survey == surveyName),]
  alongUMR_1$PLC_s21 = NA
  alongUMR_1$PLC_f21 = NA
  alongUMR_1$NS_s21 = NA
  alongUMR_1$NS_f21 = NA
  alongUMR_1$Coher_s21 = NA
  alongUMR_1$Coher_f21 = NA
  for (i in 1:nrow(deids)){
    #get the previous deidentifiers
    f21 = deids$f21[i]
    s21 = deids$s21[i]
  
    #we need to check for the deidentifier and survey name because the ES deidentifiers have two surveys
    alongUMR_1[i,]$PLC_s21   = umr[which(umr$Deidentifier == s21 & umr$Survey == surveyName),]$PLC
    alongUMR_1[i,]$NS_s21    = umr[which(umr$Deidentifier == s21 & umr$Survey == surveyName),]$NS
    alongUMR_1[i,]$Coher_s21 = umr[which(umr$Deidentifier == s21 & umr$Survey == surveyName),]$Coherency
  
    alongUMR_1[i,]$PLC_f21 = umr[which(umr$Deidentifier == f21 & umr$Survey == surveyName),]$PLC
    alongUMR_1[i,]$NS_f21 = umr[which(umr$Deidentifier == f21 & umr$Survey == surveyName),]$NS
    alongUMR_1[i,]$Coher_f21 = umr[which(umr$Deidentifier == f21 & umr$Survey == surveyName),]$Coherency
    
  }
  alongUMR_1 = alongUMR_1[,c("Institution", "Course_collected", "Deidentifier","Sex_birth","Race_ethnicity","Coherency","NS","PLC","PLC_s21","PLC_f21","NS_s21","NS_f21","Coher_s21","Coher_f21")]
  
  alongUMR_1$Coherency = as.numeric(alongUMR_1$Coherency)
  alongUMR_1$NS = as.numeric(alongUMR_1$NS)
  alongUMR_1$PLC = as.numeric(alongUMR_1$PLC)
  
  alongUMR_1$PLC_f21 = as.numeric(alongUMR_1$PLC_f21)
  alongUMR_1$NS_f21 = as.numeric(alongUMR_1$NS_f21)
  alongUMR_1$Coher_f21 = as.numeric(alongUMR_1$Coher_f21)
  
  alongUMR_1$PLC_s21 = as.numeric(alongUMR_1$PLC_s21)
  alongUMR_1$NS_s21 = as.numeric(alongUMR_1$NS_s21)
  alongUMR_1$Coher_s21 = as.numeric(alongUMR_1$Coher_s21)
  
  alongUMR_1$race_binary <- ifelse(alongUMR_1$Race_ethnicity == "White/Caucasian" , 'White', "Non-white")
  alongUMR_1 = alongUMR_1[complete.cases(alongUMR_1),] 
  return(alongUMR_1)
}

alongUMR_1 = addDataAlong(id_es,"ES_Chemical_Reaction",umr)
alongUMR_2 = addDataAlong(id_es,"ES_Glucosidase",umr)
alongUMR_3 = addDataAlong(id_ns,"Nucleic_Acids",umr)
alongUMR_4 = addDataAlong(id_ob,"Oxygen_Binding",umr)
alongUMR_5 = addDataAlong(id_ps,"Protein_Structure",umr)

addDataAlongRows = function(deids,surveyName,umr){
  #adding a new column to track of students
  umr$student = NA
  #filter course, term and survey, and grow from it
  alongUMR_1 = umr[which(umr$Course_collected == "Biochem 1" & umr$Term_collected == "Fall2022" & umr$Survey == surveyName),]
  for (i in 1:nrow(deids)){
    #get the previous deidentifiers
    f21 = deids$f21[i]
    s21 = deids$s21[i]
    #we need to check for the deidentifier and survey name because the ES deidentifiers have two surveys
    alongUMR_1 = rbind(alongUMR_1, 
                       umr[which(umr$Deidentifier == f21 & umr$Survey == surveyName),]
                       )
    alongUMR_1 = rbind(alongUMR_1, 
                       umr[which(umr$Deidentifier == s21 & umr$Survey == surveyName),]
                       )
    #We added two rows of the same student "i" at the end of the df
    g = nrow(alongUMR_1)
    alongUMR_1$student[i] = as.character(i) 
    alongUMR_1$student[g-1] = as.character(i) 
    alongUMR_1$student[g] = as.character(i) 
  }
  
  alongUMR_1$Coherency = as.numeric(alongUMR_1$Coherency)
  alongUMR_1$NS = as.numeric(alongUMR_1$NS)
  alongUMR_1$PLC = as.numeric(alongUMR_1$PLC)
  
  alongUMR_1$race_binary <- ifelse(alongUMR_1$Race_ethnicity == "White/Caucasian" , 'White', "Non-white")
  alongUMR_1 = alongUMR_1[,c("Institution", "Course_collected", "Deidentifier","Sex_birth","Race_ethnicity","Coherency","NS","PLC","student")]
  #get rid of bioc students for whom we dont have data in previous courses
  alongUMR_1 = alongUMR_1[complete.cases(alongUMR_1),] 
  alongUMR_1$Course_collected = gsub('Biochem 1', 'Chem5', alongUMR_1$Course_collected)
  alongUMR_1$Course_collected = gsub('O Chem 1', 'Chem2', alongUMR_1$Course_collected)
  alongUMR_1$Course_collected = gsub('O Chem 2', 'Chem3', alongUMR_1$Course_collected)
  alongUMR_1 = alongUMR_1[order(alongUMR_1$Course_collected),]
  alongUMR_1$Course_collected = gsub( 'Chem5','Biochem 1', alongUMR_1$Course_collected)
  alongUMR_1$Course_collected = gsub('Chem2', 'O Chem 1', alongUMR_1$Course_collected)
  alongUMR_1$Course_collected = gsub('Chem3', 'O Chem 2', alongUMR_1$Course_collected)
  return(alongUMR_1)
}
#write.csv(alongUMR_1,file = "alongUMR_1.csv",row.names = FALSE)
#write.csv(alongUMR_2,file = "alongUMR_2.csv",row.names = FALSE)
#write.csv(alongUMR_3,file = "alongUMR_3.csv",row.names = FALSE)
#write.csv(alongUMR_4,file = "alongUMR_4.csv",row.names = FALSE)
#write.csv(alongUMR_5,file = "alongUMR_5.csv",row.names = FALSE)

2 Survey 1: ES Chemical Reaction

alongUMRrows_1 = addDataAlongRows(id_es,"ES_Chemical_Reaction",umr)
library(plotly)
p = ggplot(alongUMRrows_1, aes(NS,PLC)) + 
  geom_point(aes(color=Course_collected),size=5) +
  geom_path(aes(group = student),color="grey",arrow=arrow())

plotly::ggplotly(p)
import pandas as pd
#import plotly.express as px
#students = students[["PLC_s21","PLC_f21","PLC"]]
##sortedModelUse = modelUse.sort_values(by=['didash','diaste','newman','dimeso','enanto'], axis=0, ascending=False)
#fig = px.parallel_coordinates(students,
#                              color="PLC",
#                              dimensions = ['PLC_s21','PLC_f21','PLC'],
##                              #color_continuous_scale='Bluered_r',
#                              labels={ 
#                                "PLC_s21":"PLC OChem1",
#                                "PLC_f21":"PLC OChem2",
#                                "PLC":"PLC Bioc1"
#                              })
#fig.show()

import plotly.graph_objects as go
df = pd.read_csv("~/Research/02b Neural Network Research UMR/Data + Analysis/Clustering_Xavier/alongUMR_1.csv")
fig = go.Figure(data=
    go.Parcoords(
        line = dict(color = df['PLC']),
                 #  colorscale = [[0,'purple'],[0.5,'lightseagreen'],[1,'gold']]),
        dimensions = list([
            dict(range = [-0.2,0.6],
                #constraintrange = [4,8],
                label = 'PLC OChem1', values = df['PLC_s21']),
            dict(range = [-0.2,0.6],
                label = 'PLC OChem2', values = df['PLC_f21']),
            dict(range = [-0.2,0.6],
                label = 'PLC Bioc1', values = df['PLC'])
        ])
    )
)
fig.show()

3 Survey 2: ES Glucosidase

alongUMRrows_1 = addDataAlongRows(id_es,"ES_Glucosidase",umr)
p = ggplot(alongUMRrows_1, aes(NS,PLC)) + 
  geom_point(aes(color=Course_collected),size=5) +
  geom_path(aes(group = student),color="grey",arrow=arrow())

plotly::ggplotly(p)
import pandas as pd
import plotly.graph_objects as go
df = pd.read_csv("~/Research/02b Neural Network Research UMR/Data + Analysis/Clustering_Xavier/alongUMR_2.csv")
fig = go.Figure(data=
    go.Parcoords(
        line = dict(color = df['PLC']),
                 #  colorscale = [[0,'purple'],[0.5,'lightseagreen'],[1,'gold']]),
        dimensions = list([
            dict(range = [-0.2,0.6],
                #constraintrange = [4,8],
                label = 'PLC OChem1', values = df['PLC_s21']),
            dict(range = [-0.2,0.6],
                label = 'PLC OChem2', values = df['PLC_f21']),
            dict(range = [-0.2,0.6],
                label = 'PLC Bioc1', values = df['PLC'])
        ])
    )
)
fig.show()

4 Survey 3: Nucleic Acids

alongUMRrows_1 = addDataAlongRows(id_ns,"Nucleic_Acids",umr)
p = ggplot(alongUMRrows_1, aes(NS,PLC)) + 
  geom_point(aes(color=Course_collected),size=5) +
  geom_path(aes(group = student),color="grey",arrow=arrow())

plotly::ggplotly(p)
import pandas as pd
import plotly.graph_objects as go
df = pd.read_csv("~/Research/02b Neural Network Research UMR/Data + Analysis/Clustering_Xavier/alongUMR_3.csv")
fig = go.Figure(data=
    go.Parcoords(
        line = dict(color = df['PLC']),
                 #  colorscale = [[0,'purple'],[0.5,'lightseagreen'],[1,'gold']]),
        dimensions = list([
            dict(range = [-0.2,0.6],
                #constraintrange = [4,8],
                label = 'PLC OChem1', values = df['PLC_s21']),
            dict(range = [-0.2,0.6],
                label = 'PLC OChem2', values = df['PLC_f21']),
            dict(range = [-0.2,0.6],
                label = 'PLC Bioc1', values = df['PLC'])
        ])
    )
)
fig.show()

5 Survey 4: Oxygen Binding

alongUMRrows_1 = addDataAlongRows(id_ob,"Oxygen_Binding",umr)
p = ggplot(alongUMRrows_1, aes(NS,PLC)) + 
  geom_point(aes(color=Course_collected),size=5) +
  geom_path(aes(group = student),color="grey",arrow=arrow())

plotly::ggplotly(p)
import pandas as pd
import plotly.graph_objects as go
df = pd.read_csv("~/Research/02b Neural Network Research UMR/Data + Analysis/Clustering_Xavier/alongUMR_4.csv")
fig = go.Figure(data=
    go.Parcoords(
        line = dict(color = df['PLC']),
                 #  colorscale = [[0,'purple'],[0.5,'lightseagreen'],[1,'gold']]),
        dimensions = list([
            dict(range = [-0.2,0.6],
                #constraintrange = [4,8],
                label = 'PLC OChem1', values = df['PLC_s21']),
            dict(range = [-0.2,0.6],
                label = 'PLC OChem2', values = df['PLC_f21']),
            dict(range = [-0.2,0.6],
                label = 'PLC Bioc1', values = df['PLC'])
        ])
    )
)
fig.show()

6 Survey 5: Protein Structure

alongUMRrows_1 = addDataAlongRows(id_ps,"Protein_Structure",umr)
p = ggplot(alongUMRrows_1, aes(NS,PLC)) + 
  geom_point(aes(color=Course_collected),size=5) +
  geom_path(aes(group = student),color="grey",arrow=arrow())

plotly::ggplotly(p)
import pandas as pd
import plotly.graph_objects as go
df = pd.read_csv("~/Research/02b Neural Network Research UMR/Data + Analysis/Clustering_Xavier/alongUMR_5.csv")
fig = go.Figure(data=
    go.Parcoords(
        line = dict(color = df['PLC']),
                 #  colorscale = [[0,'purple'],[0.5,'lightseagreen'],[1,'gold']]),
        dimensions = list([
            dict(range = [-0.2,0.6],
                #constraintrange = [4,8],
                label = 'PLC OChem1', values = df['PLC_s21']),
            dict(range = [-0.2,0.6],
                label = 'PLC OChem2', values = df['PLC_f21']),
            dict(range = [-0.2,0.6],
                label = 'PLC Bioc1', values = df['PLC'])
        ])
    )
)
fig.show()