Expert Evaluations: Round 2

Author

T. Eerola and C. Kirts

This analysis details the Round 2 of the expert ratings.

Show the code 
rm(list=ls())  # clear memory
library(tidyverse) 
library(stringr)
library(psychometric)
library(ggplot2)
library(CVI)
#packageVersion("CVI") # Should be 0.1.4.0 at least

source('scr/read_episode_data_round2.R') # requires ToDo! 
source('scr/preprocess_qualtrics_round2.R') # Not Finished, Complete
source('scr/analyse_episode_data_round2.R') # Not tested
source('scr/add_constructs_round2.R') #Needs work

# write the data to file (as an option)
file_write <- FALSE

Read and preprocess the data

Show the code 
Item_CodeBook <- read.csv("data/Task2_Items_302.csv",header = TRUE)
FC2  <- read_episode_data_round2("data/DEEM2ndFC_March162025.csv")
LMA2 <- read_episode_data_round2("data/DEEM2ndLMA_March162025.csv")
MM2 <- read_episode_data_round2("data/DEEM2ndMM_March162025.csv")

# there is one ID_Code left empty, let's fix it first

FC2$ID_Code.[FC2$ID_Code.==""] <- "XXX"
LMA2$ID_Code.[LMA2$ID_Code.==""] <- c("XXX","SJYU")
MM2$ID_Code.[MM2$ID_Code.==""] <- c("XXX","YYY")

#table(FC2$ID_Code.) # One duplicate, EDU, remove the first take
dupl_index <- which(FC2$ID_Code.=="EDU")[1]
FC2 <- FC2[-dupl_index,] # delete the duplicated response

# diagnose who were the missing experts
#table(LMA2$ID_Code.)
#table(MM2$ID_Code.)
#table(FC2$ID_Code.)
#length(table(FC2$ID_Code.))

# check whether XXX and YYY are test responses (or partial responses)
#tmp1<-FC2[,FC2$ID_Code.=="AMU"]
#tmp2<-FC2[,FC2$ID_Code.=="XXX"]
#tmp3<-MM2[,MM2$ID_Code.=="YYY"]

# YYY seems to be a genuine but lone response, let's remove it for now
dupl_index <- which(MM2$ID_Code.=="YYY")[1]
MM2 <- MM2[-dupl_index,] # delete the duplicated response

# Now we have 14 experts across all three constructs

# this function does not bring the ranking data at all, let's fix it.
FC2B <- preprocess_qualtrics_round2(FC2) # FC2<-FC2$rating not populating spreadsheet properly
LMA2B <- preprocess_qualtrics_round2(LMA2)
MM2B <- preprocess_qualtrics_round2(MM2)

Define constructs and merge into a data frame

Show the code 
FC2_constructs <- add_constructs_round2(FC2B$rating,'FC2')
LMA2_constructs <- add_constructs_round2(LMA2B$rating,'LMA2')
MM2_constructs <- add_constructs_round2(MM2B$rating, 'MM2')

DF <- rbind(FC2_constructs,LMA2_constructs,MM2_constructs)
#table(DF$construct)                # just checking, all OK
#print(knitr::kable(table(DF$construct,DF$ID_Code.)))    # just checking, all OK

Summarise what we have (no of ratings across constructs and experts)

Show the code 
#knitr::kable(table(DF$construct,DF$IDCode))
#length(unique(DF$ID_Code.)) # number of experts

DF$ID_Code. <- factor(DF$ID_Code.,labels = paste0('Ex',sprintf('%0.2d',1:length(unique(DF$ID_Code.)))))
knitr::kable(table(DF$construct,DF$ID_Code.))
Ex01 Ex02 Ex03 Ex04 Ex05 Ex06 Ex07 Ex08 Ex09 Ex10 Ex11 Ex12 Ex13 Ex14
Aesthetics 10 10 10 10 10 10 10 10 10 10 10 10 10 10
Associative 12 12 12 12 12 12 12 12 12 12 12 12 12 12
Belonging 14 14 14 14 14 14 14 14 14 14 14 14 14 14
Bodily 13 13 13 13 13 13 13 13 13 13 13 13 13 13
Connection 18 18 18 18 18 18 18 18 18 18 18 18 18 18
Coping 12 12 12 12 12 12 12 12 12 12 12 12 12 12
Curiosity 13 13 13 13 13 13 13 13 13 13 13 13 13 13
Diffuse 13 13 13 13 13 13 13 13 13 13 13 13 13 13
Distraction 10 10 10 10 10 10 10 10 10 10 10 10 10 10
Emotional 19 19 19 19 19 19 19 19 19 19 19 19 19 19
Energy Control 10 10 10 10 10 10 10 10 10 10 10 10 10 10
Enjoyment 11 11 11 11 11 11 11 11 11 11 11 11 11 11
Feeling 15 15 15 15 15 15 15 15 15 15 15 15 15 15
Focus 15 15 15 15 15 15 15 15 15 15 15 15 15 15
Moved 12 12 12 12 12 12 12 12 12 12 12 12 12 12
Preference & Familiarity 17 17 17 17 17 17 17 17 17 17 17 17 17 17
Qualia 7 7 7 7 7 7 7 7 7 7 7 7 7 7
Reduced 10 10 10 10 10 10 10 10 10 10 10 10 10 10
Relaxation 15 15 15 15 15 15 15 15 15 15 15 15 15 15
Self 17 17 17 17 17 17 17 17 17 17 17 17 17 17
Source 16 16 16 16 16 16 16 16 16 16 16 16 16 16
Structural 10 10 10 10 10 10 10 10 10 10 10 10 10 10
Structure 13 13 13 13 13 13 13 13 13 13 13 13 13 13

Add actual items to the data frame from the codebook.

Show the code 
#### CHECK in progress
U_codebook <- unique(Item_CodeBook$Rele_ID)
U_data <- unique(FC2B$rating$name)
U_data2 <- unique(MM2B$rating$name)
U_data3 <- unique(LMA2B$rating$name)
U_comb <- unique(c(U_data,U_data2,U_data3))
length(U_comb) # 302

m <- U_codebook %in% U_comb
sum(m) # 302, correct!
FC2B$rating$item_text <- Item_CodeBook$Item[match(FC2B$rating$name,Item_CodeBook$Rele_ID)]

Calculate Content Validity

Here we use the CVI package. In the analysis of the items in stage one, we rely on CVI (adjusted) and the same for the figures.

Show the code 
# Calculate CVI for each construct 
Constructs <- unique(DF$construct)
Items<-list()
Items_trimmed <- list()
Scales_full <- list()
Scales_trimmed <- list()
Item_N <- NULL
Item_N_trimmed <- NULL
for (i in 1:length(Constructs)){
  Items[[i]] <- CVI_item(DF[DF$construct==Constructs[i],2:4])
  Item_N[i] <- nrow(Items[[i]])
  Items_trimmed[[i]] <- Items[[i]][Items[[i]]$CVIFit=='Appropriate',]
  Item_N_trimmed[i] <- nrow(Items_trimmed[[i]])
  Scales_full[[i]] <- CVI_scale(Items[[i]])  
  Scales_trimmed[[i]] <- CVI_scale(Items_trimmed[[i]])  
}
Construct_table <- data.frame(Constructs = Constructs, 
                             Item_n = Item_N,
                             SCVI_Ave = unlist(Scales_full),
                             Item_n_T = Item_N_trimmed,
                             SCVI_AveT = unlist(Scales_trimmed)
)
knitr::kable(Construct_table,digits=2)
Constructs Item_n SCVI_Ave Item_n_T SCVI_AveT
Energy Control 10 0.65 3 0.95
Focus 15 0.83 11 0.94
Connection 18 0.78 9 0.94
Belonging 14 0.75 6 0.95
Relaxation 15 0.72 7 0.90
Enjoyment 11 0.76 7 0.87
Distraction 10 0.88 8 0.91
Coping 12 0.90 10 0.95
Feeling 15 0.81 10 0.95
Moved 12 0.70 6 0.90
Curiosity 13 0.60 2 0.88
Aesthetics 10 0.62 3 0.92
Diffuse 13 0.85 9 0.96
Bodily 13 0.85 8 0.93
Emotional 19 0.87 16 0.91
Associative 12 0.77 7 0.94
Structural 10 0.87 8 0.96
Reduced 10 0.77 5 0.93
Structure 13 0.69 4 0.91
Self 17 0.82 10 0.92
Source 16 0.75 7 0.89
Preference & Familiarity 17 0.76 6 0.92
Qualia 7 0.93 6 0.97

Visualise constructs

Enjoyment-Distraction-Relaxation

Enjoyment
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
EDR_Rele_1 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
EDR_Rele_2 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
EDR_Rele_3 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
EDR_Rele_4 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
EDR_Rele_5 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
EDR_Rele_6 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
EDR_Rele_7 9 13 0.69 0.38 0.66 Eliminated Good Reject
EDR_Rele_8 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
EDR_Rele_9 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
EDR_Rele_10 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
EDR_Rele_11 9 13 0.69 0.38 0.66 Eliminated Good Reject
EDR_Rele_12 2 13 0.15 -0.69 0.15 Eliminated Poor Reject
EDR_Rele_13 8 13 0.62 0.23 0.54 Eliminated Fair Reject
EDR_Rele_14 9 13 0.69 0.38 0.66 Eliminated Good Reject
EDR_Rele_15 2 13 0.15 -0.69 0.15 Eliminated Poor Reject
Distraction
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
EDR_Rele_16 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
EDR_Rele_17 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
EDR_Rele_18 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
EDR_Rele_19 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
EDR_Rele_20 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
EDR_Rele_21 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
EDR_Rele_22 9 13 0.69 0.38 0.66 Eliminated Good Reject
EDR_Rele_23 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
EDR_Rele_24 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
EDR_Rele_25 3 13 0.23 -0.54 0.20 Eliminated Poor Reject
EDR_Rele_26 7 13 0.54 0.08 0.42 Eliminated Fair Reject
Relaxation
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
EDR_Rele_27 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
EDR_Rele_28 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
EDR_Rele_29 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
EDR_Rele_30 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
EDR_Rele_31 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
EDR_Rele_32 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
EDR_Rele_33 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
EDR_Rele_34 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
EDR_Rele_35 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
EDR_Rele_36 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54

Connection-Belonging

Connection
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
CB_Rele_1 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
CB_Rele_2 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
CB_Rele_3 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
CB_Rele_4 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
CB_Rele_5 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
CB_Rele_6 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
CB_Rele_7 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
CB_Rele_8 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
CB_Rele_9 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
CB_Rele_10 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
CB_Rele_11 8 13 0.62 0.23 0.54 Eliminated Fair Reject
CB_Rele_12 9 13 0.69 0.38 0.66 Eliminated Good Reject
CB_Rele_13 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
CB_Rele_14 7 13 0.54 0.08 0.42 Eliminated Fair Reject
CB_Rele_15 8 13 0.62 0.23 0.54 Eliminated Fair Reject
CB_Rele_16 7 13 0.54 0.08 0.42 Eliminated Fair Reject
CB_Rele_17 7 13 0.54 0.08 0.42 Eliminated Fair Reject
CB_Rele_18 7 13 0.54 0.08 0.42 Eliminated Fair Reject
Belonging
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
CB_Rele_19 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
CB_Rele_20 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
CB_Rele_21 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
CB_Rele_22 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
CB_Rele_23 9 13 0.69 0.38 0.66 Eliminated Good Reject
CB_Rele_24 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
CB_Rele_25 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
CB_Rele_26 6 13 0.46 -0.08 0.32 Eliminated Poor Reject
CB_Rele_27 7 13 0.54 0.08 0.42 Eliminated Fair Reject
CB_Rele_28 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
CB_Rele_29 7 13 0.54 0.08 0.42 Eliminated Fair Reject
CB_Rele_30 9 13 0.69 0.38 0.66 Eliminated Good Reject
CB_Rele_31 6 13 0.46 -0.08 0.32 Eliminated Poor Reject
CB_Rele_32 9 13 0.69 0.38 0.66 Eliminated Good Reject

Focus-Motivation

Energy Control
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
FM_Rele_1 3 13 0.23 -0.54 0.20 Eliminated Poor Reject
FM_Rele_2 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
FM_Rele_3 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
FM_Rele_4 7 13 0.54 0.08 0.42 Eliminated Fair Reject
FM_Rele_5 8 13 0.62 0.23 0.54 Eliminated Fair Reject
FM_Rele_6 9 13 0.69 0.38 0.66 Eliminated Good Reject
FM_Rele_7 6 13 0.46 -0.08 0.32 Eliminated Poor Reject
FM_Rele_8 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
FM_Rele_9 7 13 0.54 0.08 0.42 Eliminated Fair Reject
FM_Rele_10 7 13 0.54 0.08 0.42 Eliminated Fair Reject
Focus
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
FM_Rele_11 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
FM_Rele_12 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
FM_Rele_13 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
FM_Rele_14 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
FM_Rele_15 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
FM_Rele_16 7 13 0.54 0.08 0.42 Eliminated Fair Reject
FM_Rele_17 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
FM_Rele_18 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
FM_Rele_19 5 13 0.38 -0.23 0.27 Eliminated Poor Reject
FM_Rele_20 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
FM_Rele_21 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
FM_Rele_22 7 13 0.54 0.08 0.42 Eliminated Fair Reject
FM_Rele_23 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
FM_Rele_24 9 13 0.69 0.38 0.66 Eliminated Good Reject
FM_Rele_25 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54

Personal Emotional Processing

Coping
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
PEP_Rele_1 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_2 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
PEP_Rele_3 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_4 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_5 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_6 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_7 8 13 0.62 0.23 0.54 Eliminated Fair Reject
PEP_Rele_8 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
PEP_Rele_9 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
PEP_Rele_10 8 13 0.62 0.23 0.54 Eliminated Fair Reject
PEP_Rele_11 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_12 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
Feeling
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
PEP_Rele_13 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_14 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_15 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_16 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_17 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
PEP_Rele_18 7 13 0.54 0.08 0.42 Eliminated Fair Reject
PEP_Rele_19 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_20 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
PEP_Rele_21 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
PEP_Rele_22 9 13 0.69 0.38 0.66 Eliminated Good Reject
PEP_Rele_23 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
PEP_Rele_24 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
PEP_Rele_25 7 13 0.54 0.08 0.42 Eliminated Fair Reject
PEP_Rele_26 5 13 0.38 -0.23 0.27 Eliminated Poor Reject
PEP_Rele_27 6 13 0.46 -0.08 0.32 Eliminated Poor Reject

Aesthetic–Interest–Awe

Moved
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
AIA_Rele_1 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
AIA_Rele_2 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
AIA_Rele_3 8 13 0.62 0.23 0.54 Eliminated Fair Reject
AIA_Rele_4 8 13 0.62 0.23 0.54 Eliminated Fair Reject
AIA_Rele_5 8 13 0.62 0.23 0.54 Eliminated Fair Reject
AIA_Rele_6 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
AIA_Rele_7 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
AIA_Rele_8 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
AIA_Rele_9 5 13 0.38 -0.23 0.27 Eliminated Poor Reject
AIA_Rele_10 6 13 0.46 -0.08 0.32 Eliminated Poor Reject
AIA_Rele_11 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
AIA_Rele_12 4 13 0.31 -0.38 0.24 Eliminated Poor Reject
Curiosity
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
AIA_Rele_13 7 13 0.54 0.08 0.42 Eliminated Fair Reject
AIA_Rele_14 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
AIA_Rele_15 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
AIA_Rele_16 9 13 0.69 0.38 0.66 Eliminated Good Reject
AIA_Rele_17 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
AIA_Rele_18 4 13 0.31 -0.38 0.24 Eliminated Poor Reject
AIA_Rele_19 8 13 0.62 0.23 0.54 Eliminated Fair Reject
AIA_Rele_20 9 13 0.69 0.38 0.66 Eliminated Good Reject
AIA_Rele_21 7 13 0.54 0.08 0.42 Eliminated Fair Reject
AIA_Rele_22 4 13 0.31 -0.38 0.24 Eliminated Poor Reject
AIA_Rele_23 7 13 0.54 0.08 0.42 Eliminated Fair Reject
AIA_Rele_24 6 13 0.46 -0.08 0.32 Eliminated Poor Reject
AIA_Rele_25 8 13 0.62 0.23 0.54 Eliminated Fair Reject
Aesthetics
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
AIA_Rele_26 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
AIA_Rele_27 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
AIA_Rele_28 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
AIA_Rele_29 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
AIA_Rele_30 6 13 0.46 -0.08 0.32 Eliminated Poor Reject
AIA_Rele_31 4 13 0.31 -0.38 0.24 Eliminated Poor Reject
AIA_Rele_32 4 13 0.31 -0.38 0.24 Eliminated Poor Reject
AIA_Rele_33 7 13 0.54 0.08 0.42 Eliminated Fair Reject
AIA_Rele_34 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
AIA_Rele_35 4 13 0.31 -0.38 0.24 Eliminated Poor Reject

Listening Modes and Agency

Diffuse
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
Dif_Rele_1 6 12 0.50 0.00 0.35 Eliminated Poor Reject
Dif_Rele_2 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Dif_Rele_3 8 12 0.67 0.33 0.62 Eliminated Good Reject
Dif_Rele_4 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Dif_Rele_5 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Dif_Rele_6 5 12 0.42 -0.17 0.28 Eliminated Poor Reject
Dif_Rele_7 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Dif_Rele_8 9 12 0.75 0.50 0.74 Needs revision Good Reject
Dif_Rele_9 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Dif_Rele_10 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Dif_Rele_11 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Dif_Rele_12 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Dif_Rele_13 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Bodily
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
Bod_Rele_1 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Bod_Rele_2 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Bod_Rele_3 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Bod_Rele_4 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Bod_Rele_5 9 12 0.75 0.50 0.74 Needs revision Good Reject
Bod_Rele_6 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Bod_Rele_7 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Bod_Rele_8 9 12 0.75 0.50 0.74 Needs revision Good Reject
Bod_Rele_9 9 12 0.75 0.50 0.74 Needs revision Good Reject
Bod_Rele_10 8 12 0.67 0.33 0.62 Eliminated Good Reject
Bod_Rele_11 8 12 0.67 0.33 0.62 Eliminated Good Reject
Bod_Rele_12 10 12 0.83 0.67 0.83 Appropriate Excellent Reject
Bod_Rele_13 10 12 0.83 0.67 0.83 Appropriate Excellent Reject
Emotional
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
Emo_Rele_1 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Emo_Rele_2 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Emo_Rele_3 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Emo_Rele_4 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Emo_Rele_5 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Emo_Rele_6 10 12 0.83 0.67 0.83 Appropriate Excellent Reject
Emo_Rele_7 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Emo_Rele_8 10 12 0.83 0.67 0.83 Appropriate Excellent Reject
Emo_Rele_9 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Emo_Rele_10 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Emo_Rele_11 10 12 0.83 0.67 0.83 Appropriate Excellent Reject
Emo_Rele_12 8 12 0.67 0.33 0.62 Eliminated Good Reject
Emo_Rele_13 10 12 0.83 0.67 0.83 Appropriate Excellent Reject
Emo_Rele_14 10 12 0.83 0.67 0.83 Appropriate Excellent Reject
Emo_Rele_15 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Emo_Rele_16 8 12 0.67 0.33 0.62 Eliminated Good Reject
Emo_Rele_17 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Emo_Rele_18 8 12 0.67 0.33 0.62 Eliminated Good Reject
Emo_Rele_19 10 12 0.83 0.67 0.83 Appropriate Excellent Reject
Associative
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
Ass_Rele_1 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Ass_Rele_2 6 12 0.50 0.00 0.35 Eliminated Poor Reject
Ass_Rele_3 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Ass_Rele_4 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Ass_Rele_5 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
Ass_Rele_6 7 12 0.58 0.17 0.48 Eliminated Fair Reject
Ass_Rele_7 10 12 0.83 0.67 0.83 Appropriate Excellent Reject
Ass_Rele_8 6 12 0.50 0.00 0.35 Eliminated Poor Reject
Ass_Rele_9 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Ass_Rele_10 8 12 0.67 0.33 0.62 Eliminated Good Reject
Ass_Rele_11 5 12 0.42 -0.17 0.28 Eliminated Poor Reject
Ass_Rele_12 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Structural
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
StrA_Rele_1 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
StrA_Rele_2 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
StrA_Rele_3 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
StrA_Rele_4 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
StrA_Rele_5 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
StrA_Rele_6 10 12 0.83 0.67 0.83 Appropriate Excellent Reject
StrA_Rele_7 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
StrA_Rele_8 9 12 0.75 0.50 0.74 Needs revision Good Reject
StrA_Rele_9 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
StrA_Rele_10 3 12 0.25 -0.50 0.21 Eliminated Poor Reject
Reduced
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
RCA_Rele_1 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
RCA_Rele_2 5 12 0.42 -0.17 0.28 Eliminated Poor Reject
RCA_Rele_3 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
RCA_Rele_4 10 12 0.83 0.67 0.83 Appropriate Excellent Reject
RCA_Rele_5 9 12 0.75 0.50 0.74 Needs revision Good Reject
RCA_Rele_6 12 12 1.00 1.00 1.00 Appropriate Excellent Accept > 0.67
RCA_Rele_7 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
RCA_Rele_8 9 12 0.75 0.50 0.74 Needs revision Good Reject
RCA_Rele_9 6 12 0.50 0.00 0.35 Eliminated Poor Reject
RCA_Rele_10 7 12 0.58 0.17 0.48 Eliminated Fair Reject

Musical Meanings

Structure
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
StrM_Rele_1 8 14 0.57 0.14 0.48 Eliminated Fair Reject
StrM_Rele_2 13 14 0.93 0.86 0.93 Appropriate Excellent Accept > 0.57
StrM_Rele_3 11 14 0.79 0.57 0.78 Needs revision Excellent Accept > 0.57
StrM_Rele_4 11 14 0.79 0.57 0.78 Needs revision Excellent Accept > 0.57
StrM_Rele_5 7 14 0.50 0.00 0.37 Eliminated Poor Reject
StrM_Rele_6 8 14 0.57 0.14 0.48 Eliminated Fair Reject
StrM_Rele_7 4 14 0.29 -0.43 0.24 Eliminated Poor Reject
StrM_Rele_8 5 14 0.36 -0.29 0.27 Eliminated Poor Reject
StrM_Rele_9 10 14 0.71 0.43 0.70 Needs revision Good Reject
StrM_Rele_10 14 14 1.00 1.00 1.00 Appropriate Excellent Accept > 0.57
StrM_Rele_11 12 14 0.86 0.71 0.86 Appropriate Excellent Accept > 0.57
StrM_Rele_12 12 14 0.86 0.71 0.86 Appropriate Excellent Accept > 0.57
StrM_Rele_13 10 14 0.71 0.43 0.70 Needs revision Good Reject
Self
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
Sel_Rele_1 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
Sel_Rele_2 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
Sel_Rele_3 9 13 0.69 0.38 0.66 Eliminated Good Reject
Sel_Rele_4 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
Sel_Rele_5 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
Sel_Rele_6 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
Sel_Rele_7 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
Sel_Rele_8 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
Sel_Rele_9 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
Sel_Rele_10 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
Sel_Rele_11 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
Sel_Rele_12 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
Sel_Rele_13 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
Sel_Rele_14 9 13 0.69 0.38 0.66 Eliminated Good Reject
Sel_Rele_15 7 13 0.54 0.08 0.42 Eliminated Fair Reject
Sel_Rele_16 7 13 0.54 0.08 0.42 Eliminated Fair Reject
Sel_Rele_17 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
Source
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
Sou_Rele_1 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
Sou_Rele_2 9 13 0.69 0.38 0.66 Eliminated Good Reject
Sou_Rele_3 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
Sou_Rele_4 8 13 0.62 0.23 0.54 Eliminated Fair Reject
Sou_Rele_5 8 13 0.62 0.23 0.54 Eliminated Fair Reject
Sou_Rele_6 9 13 0.69 0.38 0.66 Eliminated Good Reject
Sou_Rele_7 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
Sou_Rele_8 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
Sou_Rele_9 9 13 0.69 0.38 0.66 Eliminated Good Reject
Sou_Rele_10 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
Sou_Rele_11 9 13 0.69 0.38 0.66 Eliminated Good Reject
Sou_Rele_12 7 13 0.54 0.08 0.42 Eliminated Fair Reject
Sou_Rele_13 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
Sou_Rele_14 7 13 0.54 0.08 0.42 Eliminated Fair Reject
Sou_Rele_15 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
Sou_Rele_16 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54

Qualia, Reward and Exposure

Qualia
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
Exp_Rele_1 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
Exp_Rele_2 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
Exp_Rele_3 9 13 0.69 0.38 0.66 Eliminated Good Reject
Exp_Rele_4 11 13 0.85 0.69 0.84 Appropriate Excellent Accept > 0.54
Exp_Rele_5 8 13 0.62 0.23 0.54 Eliminated Fair Reject
Exp_Rele_6 4 13 0.31 -0.38 0.24 Eliminated Poor Reject
Exp_Rele_7 9 13 0.69 0.38 0.66 Eliminated Good Reject
Exp_Rele_8 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
Exp_Rele_9 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
Exp_Rele_10 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
Exp_Rele_11 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
Exp_Rele_12 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
Exp_Rele_13 11 12 0.92 0.83 0.92 Appropriate Excellent Accept > 0.67
Exp_Rele_14 6 13 0.46 -0.08 0.32 Eliminated Poor Reject
Exp_Rele_15 9 13 0.69 0.38 0.66 Eliminated Good Reject
Exp_Rele_16 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
Exp_Rele_17 10 13 0.77 0.54 0.76 Needs revision Excellent Reject
Preference & Familiarity
Item Sum N CVI.I CVI.R CVI.I.adj CVIFit KappaFit CVRFit
Qua_Rele_1 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
Qua_Rele_2 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
Qua_Rele_3 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
Qua_Rele_4 12 13 0.92 0.85 0.92 Appropriate Excellent Accept > 0.54
Qua_Rele_5 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
Qua_Rele_6 13 13 1.00 1.00 1.00 Appropriate Excellent Accept > 0.54
Qua_Rele_7 9 13 0.69 0.38 0.66 Eliminated Good Reject

Table: NA

Back to top