Expert Evaluations: Round 2 representativeness

This analysis details the Round 2 of the expert decisions of representativeness.

#### CHECK in progress
U_codebook <- unique(Item_CodeBook$Rele_ID)
U_data <- unique(FC2B$rating$name)
U_data2 <- unique(MM2B$rating$name)
U_data3 <- unique(LMA2B$rating$name)
U_comb <- unique(c(U_data,U_data2,U_data3))
length(U_comb) # 302

m <- U_codebook %in% U_comb
sum(m) # 302, correct!
FC2B$rating$item_text <- Item_CodeBook$Item[match(FC2B$rating$name,Item_CodeBook$Rele_ID)]

Analysis of rankings

# define new functions 
agreement <- function(data){
  U <- unique(data[!is.na(data)])
  L <- length(data[!is.na(data)])
  agreement <- 1 - length(U) / L
  return <- agreement
}

mostfreq <- function(data){
  t<-data.frame(table(data))
  Fr<-max(t$Freq) # the most freq item
  Id <- as.numeric(t$data[max(t$Freq)==t$Freq]) # the most frequent item count
  return <- list(id=Id[1], freq=Fr)
}

FC2_constructs <- add_constructs_round2(FC2B$rating,'FC2')
LMA2_constructs <- add_constructs_round2(LMA2B$rating,'LMA2')
MM2_constructs <- add_constructs_round2(MM2B$rating, 'MM2')

# grab the constructs from ratings and apply to rankings

FC2B$rankings$construct<-FC2_constructs$construct
LMA2B$rankings$construct<-LMA2_constructs$construct
MM2B$rankings$construct<-MM2_constructs$construct

DF_rank <- rbind(FC2B$rankings,LMA2B$rankings,MM2B$rankings)
#table(DF_rank$ID,DF_rank$construct)

Calculate agreement and the most representative item for each construct. For Table 3 in the manuscript.

CONSTRUCTS <- unique(DF_rank$construct)
DF_rank$name_as_nro <- as.numeric(str_replace_all(DF_rank$name, '_|0|[a-zA-Z]', ''))

Agreement <- NULL
MostFreqId <- NULL
MostFreqCo <- NULL

for (i in 1:length(CONSTRUCTS)){
  tmp<-DF_rank[DF_rank$construct==CONSTRUCTS[i],]
  nominations <- tmp$name[!is.na(tmp$value)]
  Agreement[i] <- agreement(nominations)
  x <- mostfreq(nominations)
  MostFreqId[i] <- x$id
  MostFreqCo[i] <- x$freq
}
ranking_summary<-data.frame(CONSTRUCTS,Agreement,ItemID=MostFreqId,Count=MostFreqCo)

print(knitr::kable(ranking_summary,digits=2))

CONSTRUCTS	Agreement	ItemID	Count
Energy Control	0.61	8	6
Focus	0.77	1	9
Connection	0.73	1	9
Belonging	0.69	10	6
Relaxation	0.73	1	9
Enjoyment	0.72	6	6
Distraction	0.76	1	7
Coping	0.79	5	13
Feeling	0.68	1	6
Moved	0.76	8	11
Curiosity	0.61	2	6
Aesthetics	0.75	1	9
Diffuse	0.80	8	11
Bodily	0.81	7	9
Emotional	0.78	12	9
Associative	0.83	7	11
Structural	0.84	2	11
Reduced	0.83	1	10
Structure	0.83	6	10
Self	0.81	13	9
Source	0.83	15	12
Preference & Familiarity	0.77	5	10
Qualia	0.85	1	9

Revised rankings

To report to 3 or 5 items for Table 4 in the manuscript.

CONSTRUCTS <- unique(DF_rank$construct)
DF_rank$name_as_nro <- as.numeric(str_replace_all(DF_rank$name, '_|0|[a-zA-Z]', ''))

Agreement <- NULL
Agreement_top_n <- NULL
top_n <- list()
#MostFreqId <- NULL
#MostFreqCo <- NULL

for (i in 1:length(CONSTRUCTS)){
  tmp<-DF_rank[DF_rank$construct==CONSTRUCTS[i],]
  nominations <- tmp$name[!is.na(tmp$value)]
  Agreement[i] <- agreement(nominations)
  x <- mostfreq(nominations) # this needs to contain top 5
  ranked <- dplyr::arrange(data.frame(table(nominations)),desc(Freq))
  # top 3 items, and agreement recalculated for those
  top_n[[i]] <- ranked[1:5,]
  top_n_nominations <- nominations[nominations %in% top_n[[i]]$nominations]
  Agreement_top_n[i] <- agreement(top_n_nominations)
  
#  MostFreqId[i] <- x$id
#  MostFreqCo[i] <- x$freq
}
ranking_summary<-data.frame(CONSTRUCTS,Agreement_top_n)
#print(knitr::kable(ranking_summary,digits=2))

tab<-NULL
for (k in 1:length(top_n)) {
  tab$Construct[k] <- CONSTRUCTS[k]
  tmp<-stringr::str_replace_all(paste0(as.character(top_n[[k]]$nominations),collapse = ","),'[A-Za-z]|_0_|_| ','')
#  tmp<-paste0(as.character(top3[[k]]$nominations),collapse = ",")
  tab$Items[k] <- tmp  
}
tab<-data.frame(tab,Agreement=ranking_summary$Agreement_top_n)
knitr::kable(tab,digits=2)  

Construct	Items	Agreement
Energy Control	8,2,4,10,3	0.71
Focus	11,25,13,14,18	0.84
Connection	1,11,2,10,13	0.81
Belonging	28,19,24,21,20	0.78
Relaxation	1,9,10,6,8	0.83
Enjoyment	21,16,18,19,22	0.77
Distraction	27,36,29,28,33	0.81
Coping	2,6,9,11,12	0.87
Feeling	13,14,15,23,24	0.78
Moved	6,2,1,8,11	0.86
Curiosity	14,15,13,19,21	0.76
Aesthetics	26,27,29,34,28	0.82
Diffuse	4,2,12,3,9	0.88
Bodily	3,13,1,2,4	0.87
Emotional	2,15,7,1,10	0.85
Associative	4,12,1,3,7	0.89
Structural	2,5,9,4,1	0.88
Reduced	1,3,6,7,10	0.87
Structure	2,10,4,11,12	0.88
Self	5,1,11,3,4	0.87
Source	8,15,1,13,7	0.89
Preference & Familiarity	13,2,9,5,10	0.86
Qualia	1,6,5,4,2	0.86

CVI across MAIN constructs

For Table 3 in the manuscript.

# Calculate CVI for each construct 
Constructs <- unique(DF$construct)

DF$mainconstructs <- factor(DF$construct,levels = Constructs,
                            labels=c("FM","FM","CB","CB","EDR","EDR","EDR","PEP","PEP",
                                     "AIA","AIA","AIA","LMA","LMA","LMA","LMA","LMA","LMA",
                                     "MM","MM","MM","ERQ","ERQ"))

#table(DF$construct,DF$mainconstructs)

MainConstructs <- unique(DF$mainconstructs)

Items<-list()
Items_trimmed <- list()
Scales_full <- list()
Scales_trimmed <- list()
Item_N <- NULL
Item_N_trimmed <- NULL
for (i in 1:length(MainConstructs)){
  Items[[i]] <- CVI_item(DF[DF$mainconstructs==MainConstructs[i],2:4])
  Item_N[i] <- nrow(Items[[i]])
  Items_trimmed[[i]] <- Items[[i]][Items[[i]]$CVIFit=='Appropriate',]
  Item_N_trimmed[i] <- nrow(Items_trimmed[[i]])
  Scales_full[[i]] <- CVI_scale(Items[[i]])  
  Scales_trimmed[[i]] <- CVI_scale(Items_trimmed[[i]])  
}
MainConstruct_table <- data.frame(MainConstructs = MainConstructs, 
                             Item_n = Item_N,
                             SCVI_Ave = unlist(Scales_full),
                             Item_n_T = Item_N_trimmed,
                             SCVI_AveT = unlist(Scales_trimmed)
)
knitr::kable(MainConstruct_table,digits=2)

MainConstructs	Item_n	SCVI_Ave	Item_n_T	SCVI_AveT
FM	25	0.76	14	0.94
CB	32	0.77	15	0.94
EDR	36	0.78	22	0.90
PEP	27	0.85	20	0.95
AIA	35	0.64	11	0.90
LMA	77	0.83	53	0.94
MM	46	0.76	21	0.91
ERQ	24	0.81	12	0.95