Exp. 1 - Measurement invariance

Measurement invariance across the vignettes was examined using multigroup CFA in lavaan with robust maximum likelihood estimation (MLR) and participant ID as a clustering variable. Configural, metric, and scalar invariance models were compared sequentially. Table 1 below summarises the invariance results, but generally model fit did not degrade substantially when factor loadings were constrained (ΔCFI = .001, ΔRMSEA = .008), suggesting metric invariance for FM, CB and AIA. However, when intercepts were constrained (ΔCFI < .005, ΔRMSEA < .004), all constructs failed to support the scalar invariance across vignettes.

These analysyses correspond to section CFA model fits and measurement invariance in the manuscript.

EDR

df <- readRDS(file="data/EDR_N100.rds")
tmp <- dplyr::filter(df,vignette=="E" | vignette=="D" | vignette=="R")
tmp_w <- pivot_wider(tmp, names_from = item_label, values_from = value,id_expand = FALSE,id_cols = c('ID','vignette'))
tmp_w<-data.frame(tmp_w)
rownames(tmp_w) <- 1:nrow(tmp_w)

model <- "
E =~ E1 + E6 + E7
D =~ D1 + D6 + D7
R =~ R1 + R3 + R7
"
# baseline
model.baseline <- lavaan::cfa(model, data = tmp_w,meanstructure = TRUE,estimator = "MLR")

# configural invariance
model.configural <- lavaan::cfa(model, data = tmp_w, group = "vignette",meanstructure = TRUE,estimator = "MLR")
#s<-semTools::compareFit(model.baseline,model.configural)
#summary(s,fit.measures="all")
config1 <- lavTestLRT(model.baseline,model.configural)$`Pr(>Chisq)`[2]
# metric invariance
model.metric <- lavaan::cfa(model, data = tmp_w, group = "vignette",group.equal=c("loadings"),meanstructure = TRUE,estimator = "MLR")
#s<-semTools::compareFit(model.configural, model.metric)
#summary(s,fit.measures="all")
metric1 <- lavTestLRT(model.configural,model.metric)$`Pr(>Chisq)`[2]

# scalar invariance
model.scalar <- lavaan::cfa(model, data = tmp_w, group = "vignette", group.equal=c("loadings","intercepts"),meanstructure = TRUE,estimator = "MLR")
#summary(model.scalar, standardized = TRUE, fit.measures = TRUE)
#s<-semTools::compareFit(model.metric, model.scalar)
#summary(s)
scalar1 <- lavTestLRT(model.metric,model.scalar)$`Pr(>Chisq)`[2]

s<-semTools::compareFit(model.baseline,model.configural, model.metric,model.scalar,nested = TRUE)
summary(s)
################### Nested Model Comparison #########################

Scaled Chi-Squared Difference Test (method = "satorra.bentler.2001")

lavaan->unknown():  
   lavaan NOTE: The "Chisq" column contains standard test statistics, not the 
   robust test that should be reported per model. A robust difference test is 
   a function of two standard (not robust) statistics.
                 Df    AIC    BIC   Chisq Chisq diff Df diff Pr(>Chisq)    
model.baseline   24 6264.0 6375.1  51.853                                  
model.configural 72 5943.7 6277.0  86.815     33.670      48     0.9418    
model.metric     84 5943.3 6232.2 110.381     22.222      12     0.0351 *  
model.scalar     96 5966.5 6211.0 157.653     52.143      12  5.849e-07 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

####################### Model Fit Indices ###########################
                 chisq.scaled df.scaled pvalue.scaled rmsea.robust cfi.robust
model.baseline        42.628†        24          .011        .056       .987 
model.configural      79.087         72          .265        .033†      .995†
model.metric         101.045         84          .099        .047       .987 
model.scalar         147.454         96          .001        .076       .961 
                 tli.robust  srmr       aic       bic
model.baseline        .980  .038† 6263.973  6375.087 
model.configural      .992† .046  5943.698  6277.038 
model.metric          .983  .068  5943.263† 6232.159 
model.scalar          .957  .080  5966.536  6210.986†

################## Differences in Fit Indices #######################
                                  df.scaled rmsea.robust cfi.robust tli.robust
model.configural - model.baseline        48       -0.023      0.008      0.012
model.metric - model.configural          12        0.014     -0.008     -0.009
model.scalar - model.metric              12        0.029     -0.026     -0.027
                                   srmr      aic     bic
model.configural - model.baseline 0.007 -320.275 -98.049
model.metric - model.configural   0.022   -0.434 -44.880
model.scalar - model.metric       0.012   23.272 -21.173

FM

df <- readRDS(file="data/FM_N100.rds")
df$item_label[df$item_label=="F1"]<-"M9" # relabel

tmp <- dplyr::filter(df,vignette=="F" | vignette=="M")
tmp_w <- pivot_wider(tmp, names_from = item_label, values_from = value,id_expand = FALSE,id_cols = c('ID','vignette'))
tmp_w<-data.frame(tmp_w)
rownames(tmp_w) <- 1:nrow(tmp_w)
#tmp_w <- select(tmp_w, -ID)
tmp_w$ID<-as.numeric(as.factor(tmp_w$ID))
tmp_w$vignette<-as.factor(tmp_w$vignette)

model <- '
    M =~ M1 + M3 + M9
    F =~ F3 + F6 + F10
'

# baseline
model.baseline <- lavaan::cfa(model, data = tmp_w,meanstructure = TRUE,estimator = "MLR")

# configural invariance
model.configural <- lavaan::cfa(model, data = tmp_w, group = "vignette",meanstructure = TRUE,estimator = "MLR")
#s<-semTools::compareFit(model.baseline,model.configural)
#summary(s,fit.measures="all")
config2 <- lavTestLRT(model.baseline,model.configural)$`Pr(>Chisq)`[2]
# metric invariance
model.metric <- lavaan::cfa(model, data = tmp_w, group = "vignette",group.equal=c("loadings"),meanstructure = TRUE,estimator = "MLR")
#s<-semTools::compareFit(model.configural, model.metric)
#summary(s,fit.measures="all")
metric2 <- lavTestLRT(model.configural,model.metric)$`Pr(>Chisq)`[2]

# scalar invariance
model.scalar <- lavaan::cfa(model, data = tmp_w, group = "vignette", group.equal=c("loadings","intercepts"),meanstructure = TRUE,estimator = "MLR")
#summary(model.scalar, standardized = TRUE, fit.measures = TRUE)
#s<-semTools::compareFit(model.metric, model.scalar)
#summary(s)
scalar2 <- lavTestLRT(model.metric,model.scalar)$`Pr(>Chisq)`[2]

s<-semTools::compareFit(model.baseline,model.configural, model.metric,model.scalar,nested = TRUE)
summary(s)
################### Nested Model Comparison #########################

Scaled Chi-Squared Difference Test (method = "satorra.bentler.2001")

lavaan->unknown():  
   lavaan NOTE: The "Chisq" column contains standard test statistics, not the 
   robust test that should be reported per model. A robust difference test is 
   a function of two standard (not robust) statistics.
                 Df    AIC    BIC  Chisq Chisq diff Df diff Pr(>Chisq)    
model.baseline    8 2624.0 2686.6 16.583                                  
model.configural 16 2529.5 2654.8 22.718     6.3137       8     0.6121    
model.metric     20 2523.4 2635.5 24.632     1.0851       4     0.8966    
model.scalar     24 2542.3 2641.3 51.610    27.0694       4  1.925e-05 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

####################### Model Fit Indices ###########################
                 chisq.scaled df.scaled pvalue.scaled rmsea.robust cfi.robust
model.baseline        16.272†         8          .039        .073       .992 
model.configural      22.823         16          .119        .065       .993 
model.metric          21.438         20          .372        .029†     0.998†
model.scalar          45.933         24          .005        .101       .973 
                 tli.robust  srmr       aic       bic
model.baseline        .985  .018† 2623.969  2686.637 
model.configural      .986  .025  2529.455  2654.791 
model.metric         0.997† .029  2523.368† 2635.511†
model.scalar          .967  .057  2542.347  2641.296 

################## Differences in Fit Indices #######################
                                  df.scaled rmsea.robust cfi.robust tli.robust
model.configural - model.baseline         8       -0.007      0.001      0.001
model.metric - model.configural           4       -0.036      0.006      0.011
model.scalar - model.metric               4        0.073     -0.025     -0.031
                                   srmr     aic     bic
model.configural - model.baseline 0.007 -94.514 -31.846
model.metric - model.configural   0.004  -6.087 -19.280
model.scalar - model.metric       0.028  18.978   5.785

CB

df <- readRDS(file="data/CB_N100.rds")

tmp <- dplyr::filter(df,vignette=="C" | vignette=="S")
tmp_w <- pivot_wider(tmp, names_from = item_label, values_from = value,id_expand = FALSE,id_cols = c('ID','vignette'))
tmp_w<-data.frame(tmp_w)
rownames(tmp_w) <- 1:nrow(tmp_w)
tmp_w$ID<-as.numeric(as.factor(tmp_w$ID))
tmp_w$vignette<-as.factor(tmp_w$vignette)

model <- "
    C =~ C5 + C2 + C9
    S =~ S5 + S6 + S7 
"

# baseline
model.baseline <- lavaan::cfa(model, data = tmp_w,meanstructure = TRUE,estimator = "MLR")

# configural invariance
model.configural <- lavaan::cfa(model, data = tmp_w, group = "vignette",meanstructure = TRUE,estimator = "MLR")
#s<-semTools::compareFit(model.baseline,model.configural)
#summary(s,fit.measures="all")
config3 <- lavTestLRT(model.baseline,model.configural)$`Pr(>Chisq)`[2]
# metric invariance
model.metric <- lavaan::cfa(model, data = tmp_w, group = "vignette",group.equal=c("loadings"),meanstructure = TRUE,estimator = "MLR")
#s<-semTools::compareFit(model.configural, model.metric)
#summary(s,fit.measures="all")
metric3 <- lavTestLRT(model.configural,model.metric)$`Pr(>Chisq)`[2]

# scalar invariance
model.scalar <- lavaan::cfa(model, data = tmp_w, group = "vignette", group.equal=c("loadings","intercepts"),meanstructure = TRUE,estimator = "MLR")
#summary(model.scalar, standardized = TRUE, fit.measures = TRUE)
#s<-semTools::compareFit(model.metric, model.scalar)
#summary(s)
scalar3 <- lavTestLRT(model.metric,model.scalar)$`Pr(>Chisq)`[2]

s<-semTools::compareFit(model.baseline,model.configural, model.metric,model.scalar,nested = TRUE)
summary(s)
################### Nested Model Comparison #########################

Scaled Chi-Squared Difference Test (method = "satorra.bentler.2001")

lavaan->unknown():  
   lavaan NOTE: The "Chisq" column contains standard test statistics, not the 
   robust test that should be reported per model. A robust difference test is 
   a function of two standard (not robust) statistics.
                 Df    AIC    BIC  Chisq Chisq diff Df diff Pr(>Chisq)    
model.baseline    8 3010.2 3072.9 13.853                                  
model.configural 16 2852.4 2977.7 15.319      1.340       8     0.9951    
model.metric     20 2851.0 2963.2 21.962      6.649       4     0.1557    
model.scalar     24 2884.9 2983.9 63.879     54.236       4  4.697e-11 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

####################### Model Fit Indices ###########################
                 chisq.scaled df.scaled pvalue.scaled rmsea.robust cfi.robust
model.baseline        14.346†         8          .073        .062       .992 
model.configural      14.875         16          .534        .000†     1.000†
model.metric          21.453         20          .371        .027      0.998 
model.scalar          65.054         24          .000        .130       .944 
                 tli.robust  srmr       aic       bic
model.baseline        .985  .041  3010.237  3072.905 
model.configural     1.003† .023† 2852.365  2977.701 
model.metric         0.997  .040  2851.008† 2963.151†
model.scalar          .930  .083  2884.925  2983.874 

################## Differences in Fit Indices #######################
                                  df.scaled rmsea.robust cfi.robust tli.robust
model.configural - model.baseline         8       -0.062      0.008      0.018
model.metric - model.configural           4        0.027     -0.002     -0.006
model.scalar - model.metric               4        0.102     -0.054     -0.067
                                    srmr      aic     bic
model.configural - model.baseline -0.018 -157.872 -95.204
model.metric - model.configural    0.018   -1.357 -14.550
model.scalar - model.metric        0.042   33.917  20.724

PEP

df <- readRDS(file="data/PEP_N100.rds")

tmp <- dplyr::filter(df,vignette=="U" | vignette=="P")
tmp_w <- pivot_wider(tmp, names_from = item_label, values_from = value,id_expand = FALSE,id_cols = c('ID','vignette'))
tmp_w<-data.frame(tmp_w)
rownames(tmp_w) <- 1:nrow(tmp_w)
#tmp_w <- select(tmp_w, -ID)
tmp_w$ID<-as.numeric(as.factor(tmp_w$ID))
tmp_w$vignette<-as.factor(tmp_w$vignette)

model <- "
    U =~ U2 + U3 + U6
    P =~ P4 + P8 + P9
"


# baseline
model.baseline <- lavaan::cfa(model, data = tmp_w,meanstructure = TRUE,estimator = "MLR")

# configural invariance
model.configural <- lavaan::cfa(model, data = tmp_w, group = "vignette",meanstructure = TRUE,estimator = "MLR")
#s<-semTools::compareFit(model.baseline,model.configural)
#summary(s,fit.measures="all")
config4 <- lavTestLRT(model.baseline,model.configural)$`Pr(>Chisq)`[2]
# metric invariance
model.metric <- lavaan::cfa(model, data = tmp_w, group = "vignette",group.equal=c("loadings"),meanstructure = TRUE,estimator = "MLR")
#s<-semTools::compareFit(model.configural, model.metric)
#summary(s,fit.measures="all")
metric4 <- lavTestLRT(model.configural,model.metric)$`Pr(>Chisq)`[2]

# scalar invariance
model.scalar <- lavaan::cfa(model, data = tmp_w, group = "vignette", group.equal=c("loadings","intercepts"),meanstructure = TRUE,estimator = "MLR")
#summary(model.scalar, standardized = TRUE, fit.measures = TRUE)
#s<-semTools::compareFit(model.metric, model.scalar)
#summary(s)
scalar4 <- lavTestLRT(model.metric,model.scalar)$`Pr(>Chisq)`[2]

s<-semTools::compareFit(model.baseline,model.configural, model.metric,model.scalar,nested = TRUE)
summary(s)
################### Nested Model Comparison #########################

Scaled Chi-Squared Difference Test (method = "satorra.bentler.2001")

lavaan->unknown():  
   lavaan NOTE: The "Chisq" column contains standard test statistics, not the 
   robust test that should be reported per model. A robust difference test is 
   a function of two standard (not robust) statistics.
                 Df    AIC    BIC  Chisq Chisq diff Df diff Pr(>Chisq)  
model.baseline    8 2578.4 2641.0 15.921                                
model.configural 16 2572.0 2697.3 18.223     2.1143       8    0.97731  
model.metric     20 2576.6 2688.7 30.799    12.9850       4    0.01135 *
model.scalar     24 2573.7 2672.7 35.941     5.4128       4    0.24750  
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

####################### Model Fit Indices ###########################
                 chisq.scaled df.scaled pvalue.scaled rmsea.robust cfi.robust
model.baseline        15.623†         8          .048        .070       .989 
model.configural      17.291         16          .367        .029†     0.998†
model.metric          29.706         20          .075        .071       .985 
model.scalar          35.156         24          .066        .069       .983 
                 tli.robust  srmr       aic       bic
model.baseline        .979  .027† 2578.363  2641.031†
model.configural     0.996† .030  2571.980† 2697.316 
model.metric          .978  .064  2576.556  2688.699 
model.scalar          .979  .067  2573.699  2672.648 

################## Differences in Fit Indices #######################
                                  df.scaled rmsea.robust cfi.robust tli.robust
model.configural - model.baseline         8       -0.041      0.009      0.017
model.metric - model.configural           4        0.042     -0.013     -0.019
model.scalar - model.metric               4       -0.002     -0.002      0.001
                                   srmr    aic     bic
model.configural - model.baseline 0.003 -6.383  56.285
model.metric - model.configural   0.034  4.576  -8.617
model.scalar - model.metric       0.003 -2.858 -16.051

AIA

df <- readRDS(file="data/AIA_N101.rds")

tmp <- dplyr::filter(df,vignette=="I" | vignette=="X" | vignette=="B")
tmp_w <- pivot_wider(tmp, names_from = item_label, values_from = value,id_expand = FALSE,id_cols = c('ID','vignette'))
tmp_w<-data.frame(tmp_w)
rownames(tmp_w) <- 1:nrow(tmp_w)
tmp_w$ID<-as.numeric(as.factor(tmp_w$ID))
tmp_w$vignette<-as.factor(tmp_w$vignette)

model <- "
    I =~ I3 + I4 + I5
    X =~ X1 + X2
    B =~ B1 + B3 + B5
"


# baseline
model.baseline <- lavaan::cfa(model, data = tmp_w,meanstructure = TRUE,estimator = "MLR")

# configural invariance
model.configural <- lavaan::cfa(model, data = tmp_w, group = "vignette",meanstructure = TRUE,estimator = "MLR")
#s<-semTools::compareFit(model.baseline,model.configural)
#summary(s,fit.measures="all")
config5 <- lavTestLRT(model.baseline,model.configural)$`Pr(>Chisq)`[2]
# metric invariance
model.metric <- lavaan::cfa(model, data = tmp_w, group = "vignette",group.equal=c("loadings"),meanstructure = TRUE,estimator = "MLR")
#s<-semTools::compareFit(model.configural, model.metric)
#summary(s,fit.measures="all")
metric5 <- lavTestLRT(model.configural,model.metric)$`Pr(>Chisq)`[2]

# scalar invariance
model.scalar <- lavaan::cfa(model, data = tmp_w, group = "vignette", group.equal=c("loadings","intercepts"),meanstructure = TRUE,estimator = "MLR")
#summary(model.scalar, standardized = TRUE, fit.measures = TRUE)
#s<-semTools::compareFit(model.metric, model.scalar)
#summary(s)
scalar5 <- lavTestLRT(model.metric,model.scalar)$`Pr(>Chisq)`[2]

s<-semTools::compareFit(model.baseline,model.configural, model.metric,model.scalar,nested = TRUE)
summary(s)
################### Nested Model Comparison #########################

Scaled Chi-Squared Difference Test (method = "satorra.bentler.2001")

lavaan->unknown():  
   lavaan NOTE: The "Chisq" column contains standard test statistics, not the 
   robust test that should be reported per model. A robust difference test is 
   a function of two standard (not robust) statistics.
                 Df    AIC    BIC   Chisq Chisq diff Df diff Pr(>Chisq)    
model.baseline   17 5294.4 5394.7  54.764                                  
model.configural 51 5227.1 5527.9  97.533     43.811      34     0.1209    
model.metric     61 5214.9 5478.5 105.338      8.148      10     0.6144    
model.scalar     71 5258.1 5484.6 168.575     67.439      10  1.381e-10 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

####################### Model Fit Indices ###########################
                 chisq.scaled df.scaled pvalue.scaled rmsea.robust cfi.robust
model.baseline        43.154†        17          .000        .080†      .982†
model.configural      90.827         51          .001        .091       .976 
model.metric          99.864         61          .001        .082       .977 
model.scalar         162.353         71          .000        .115       .947 
                 tli.robust  srmr       aic       bic
model.baseline        .970† .051† 5294.415  5394.686†
model.configural      .961  .058  5227.055  5527.867 
model.metric          .969  .062  5214.860† 5478.535 
model.scalar          .938  .078  5258.096  5484.634 

################## Differences in Fit Indices #######################
                                  df.scaled rmsea.robust cfi.robust tli.robust
model.configural - model.baseline        34        0.011     -0.005     -0.009
model.metric - model.configural          10       -0.010      0.001      0.008
model.scalar - model.metric              10        0.033     -0.030     -0.031
                                   srmr     aic     bic
model.configural - model.baseline 0.006 -67.360 133.181
model.metric - model.configural   0.004 -12.195 -49.332
model.scalar - model.metric       0.017  43.236   6.099

Summary

Measurement Invariance p-values for robust CFA analysis across constructs.
Configural Metric Scalar
EDR 0.942 0.035 0.000
FM 0.612 0.897 0.000
CB 0.995 0.156 0.000
PEP 0.977 0.011 0.248
AIA 0.121 0.614 0.000
Back to top