CoE Meta-Analysis

Last update: 04 May 2020

Meta-Analysis of Change of Evidence Measures

In these meta-analyses, we will analyze the overall significance of the change of evidence-measures in all experimental and control studies. Since we don’t have effect sizes for these measures, we will combine the empirical p-values of the studies, this means, we will analyze the amount of simulations that show an equal or higher value in the maxbf, energybf, and frequency tests than the empirical BF.

The resulting p-values will be analyzed using the inverse chi-square method, which converts each p-value into a chi-square statistic and then sums them up. The method allows for the incorporation of different degrees of freedom, making it particularly useful when the p-values come from tests with varying sample sizes.

In this analysis all Micro-Pk studies of the LMU Micro-Pk Lab are included that hypothesize the presence of an effect. In studies comparing experimental to control conditions, only the experimental conditions are included.

coe <- readRDS("data/coe.rds")

# Select studies / conditions
data <- subset(coe, coe$Experimental == TRUE)

knitr::kable(data)
Study Experimental N MaxBF MaxBF p Energy Energy p FFT Amplitude sum FFT p Direction Labstudy
1 Loving Kind TRUE 171 4.16 (n=32) 0.113 17.332366 0.155 3.223027 0.120 greater TRUE
2 Prayer TRUE 431 1.44 (n=17) 0.481 -269.852281 0.723 1.415288 0.481 greater FALSE
3 Monks T1 Exp TRUE 23 1.98 (n=22) 0.150 6.443314 0.106 1.813206 0.133 greater TRUE
5 Monks T2 Exp TRUE 23 1.74 (n=14) 0.196 4.114508 0.145 1.535508 0.195 greater TRUE
8 Meditation T2 TRUE 48 1.74 (n=4) 0.248 -13.477604 0.571 1.336472 0.324 greater FALSE
9 Sound preference TRUE 70 1.27 (n=53) 0.455 -1.286672 0.230 1.344490 0.361 greater TRUE
10 All-will-be-good TRUE 38 1.77 (n=5) 0.233 -9.020942 0.539 1.431673 0.267 greater TRUE
11 One-armed Bandit TRUE 40 4.15 (n=24) 0.051 44.754281 0.031 3.872203 0.041 greater TRUE
12 Coin Toss TRUE 40 1.15 (n=17) 0.512 -2.651081 0.310 1.198054 0.418 greater TRUE
13 Incongruence Exp TRUE 236 10.72 (n=153) 0.038 564.279423 0.018 9.885522 0.029 greater TRUE
15 Smokers 1 Exp TRUE 122 66.06 (n=122) 0.008 497.532426 0.012 44.307712 0.009 different TRUE
17 Smokers 2 Exp TRUE 175 10.19 (n=7) 0.052 -59.180724 0.119 6.991435 0.055 less TRUE
19 Smokers 3 TRUE 203 1.89 (n=3) 0.283 -130.013155 0.289 1.306952 0.340 less TRUE
20 Psyscanner Style 1 Exp TRUE 1400 12.92 (n=825) 0.041 4219.990983 0.017 17.408978 0.024 greater FALSE
22 Psyscanner Style 2 Exp TRUE 1308 3.2 (n=355) 0.234 -100.586634 0.185 3.625750 0.202 greater FALSE
24 Psyscanner Style 3 Exp TRUE 1462 2.39 (n=187) 0.341 -335.853817 0.241 2.955509 0.261 greater FALSE
26 Relaxation TRUE 12571 7.44 (n=380) 0.110 -6613.797389 0.110 7.517862 0.093 greater FALSE
27 Priming 1 Exp TRUE 4092 28.78 (n=1680) 0.023 13019.985027 0.019 32.312257 0.020 greater FALSE
29 Priming 2 Exp TRUE 2063 1.15 (n=21) 0.767 -1698.465345 0.903 1.166660 0.868 greater FALSE
31 Priming 3 Exp TRUE 6099 1.01 (n=28) 0.868 -5344.888154 0.826 1.222472 0.844 greater FALSE
33 Priming 4 Exp TRUE 4060 1.65 (n=11) 0.539 -3516.542596 0.884 1.501313 0.656 greater FALSE
35 Erotic Images 1 TRUE 241 1.39 (n=111) 0.487 -54.619823 0.282 1.426101 0.429 greater FALSE
36 Erotic Images 2 TRUE 678 5.24 (n=363) 0.115 482.409201 0.059 5.292795 0.099 greater FALSE 
library(metap)
istwo <- ifelse(data$Direction == "different", TRUE, FALSE) # two-tailed test
df <- data$N-1 # degrees of freedom

Max BF

pvals <- two2one(data$`MaxBF p`, istwo)

res <- invchisq(pvals, k=df)
print(res)
chisq =  36155.12  with df =  35571  p =  0.01460616 
plot(res)

BF Energy

pvals <- two2one(data$`Energy p`, istwo)

res <- invchisq(pvals, k=df)
print(res)
chisq =  36096.14  with df =  35571  p =  0.0248961 
plot(res)

FFT Amplitude Sum

pvals <- two2one(data$`FFT p`, istwo)

res <- invchisq(pvals, k=df)
print(res)
chisq =  36174.83  with df =  35571  p =  0.01210754 
plot(res)

Robustness to Sample Sizes

In comparison to the inverse chi-square method, which weights each p-value by its degree of freedom and therefor its sample size, Fisher’s method combines p-values by taking the sum of their logarithms and then comparing them to a chi-square distribution. This method is less sensitive to sample sizes and can be used as a robustness check countering arguments of selectively continuing data collection of studies with a noticeable BF.

#Max BF
pvals <- two2one(data$`MaxBF p`, istwo)
sumlog(pvals)
chisq =  84.68508  with df =  46  p =  0.0004450107 
#Energy BF
pvals <- two2one(data$`Energy p`, istwo)
sumlog(pvals)
chisq =  88.05687  with df =  46  p =  0.000187993 
#FFT Amplitude Sum
pvals <- two2one(data$`FFT p`, istwo)
sumlog(pvals)
chisq =  87.62367  with df =  46  p =  0.0002103507

In this analysis all Micro-Pk studies of the LMU Micro-Pk Lab are included that hypothesize the absence of an effect. In studies comparing experimental to control conditions, only the control conditions are included.

# Select studies / conditions
data <- subset(coe, coe$Experimental == FALSE)
knitr::kable(data)
Study Experimental N MaxBF MaxBF p Energy Energy p FFT Amplitude sum FFT p Direction Labstudy
4 Monks T1 Con FALSE 29 1 (n=1) 1.000 -9.659564 0.792 0.9418921 0.895 greater TRUE
6 Monks T2 Con FALSE 29 1.18 (n=7) 0.445 -5.080576 0.472 1.0954834 0.475 greater TRUE
7 Meditation T1 FALSE 40 1 (n=1) 1.000 -17.858023 0.938 0.8598144 1.000 greater FALSE
14 Incongruence Con FALSE 271 1.17 (n=125) 0.616 -89.808184 0.405 1.2501113 0.525 different TRUE
16 Smokers 1 Con FALSE 132 1 (n=1) 1.000 -99.068118 0.986 0.8035244 1.000 different TRUE
18 Smokers 2 Con FALSE 220 3.56 (n=10) 0.155 -138.717494 0.285 2.2931728 0.177 different TRUE
21 Psyscanner Style 1 Con FALSE 1003 1 (n=1) 1.000 -506.646162 0.570 1.0942438 0.977 different FALSE
23 Psyscanner Style 2 Con FALSE 1095 5.91 (n=936) 0.105 1012.370693 0.054 6.7424289 0.080 different FALSE
25 Psyscanner Style 3 Con FALSE 941 2.91 (n=17) 0.234 -530.178690 0.823 1.9538020 0.355 different FALSE
28 Priming 1 Con FALSE 4092 1.38 (n=58) 0.648 -3306.742875 0.687 1.3074276 0.768 greater FALSE
30 Priming 2 Con FALSE 2063 1 (n=1) 1.000 -1611.962028 0.801 0.9568926 0.994 greater FALSE
32 Priming 3 Con FALSE 6099 1 (n=1) 1.000 -5561.551895 0.949 1.0836289 0.955 greater FALSE
34 Priming 4 Con FALSE 4060 1.56 (n=5) 0.562 -3668.203466 0.970 1.3702124 0.732 greater FALSE 
library(metap)
istwo <- ifelse(data$Direction == "different", TRUE, FALSE) # two-tailed test
df <- data$N-1 # degrees of freedom

Max BF

pvals <- two2one(data$`MaxBF p`, istwo)
res <- invchisq(pvals, k=df)
print(res)
chisq =  11953.7  with df =  20061  p =  1 
plot(res)

BF Energy

pvals <- two2one(data$`Energy p`, istwo)
res <- invchisq(pvals, k=df)
print(res)
chisq =  19762.9  with df =  20061  p =  0.9321976 
plot(res)

FFT Amplitude Sum

pvals <- two2one(data$`FFT p`, istwo)
res <- invchisq(pvals, k=df)
print(res)
chisq =  19712.76  with df =  20061  p =  0.9595416 
plot(res)

Robustness to Sample Sizes

#Max BF
pvals <- two2one(data$`MaxBF p`, istwo)
sumlog(pvals)
chisq =  24.0675  with df =  26  p =  0.5721054 
#Energy BF
pvals <- two2one(data$`Energy p`, istwo)
sumlog(pvals)
chisq =  23.4718  with df =  26  p =  0.60614 
#FFT Amplitude Sum
pvals <- two2one(data$`FFT p`, istwo)
sumlog(pvals)
chisq =  23.20571  with df =  26  p =  0.6212849