We still use the lalonde data from the MatchIt package and use the
propensity score model that we found out in R example 5
library("MatchIt")
data("lalonde")
model <- glm(treat ~ re74 + race + married + I(re74^2) + re74:race, data = lalonde, family = "binomial")
eps <- predict(model, type = "response")
lalonde
Check weights and covariates balancing before trimming
Then we perform a first check of the weights by drawing the histogram
or boxplot of the weights. At this moment, the figures are not very
informative as there are units with extremely large weights.
## Calculate the raw weights (before normalization)
## Weights to estimate ATE
n.treated <- sum(lalonde$treat == 1)
n.control <- sum(lalonde$treat == 0)
weights <- ifelse(lalonde$treat == 1, 1/eps, 1/(1 - eps))
## Check the weights histogram
library(ggplot2)
temp.data <- data.frame(weights = weights, treated = as.factor(lalonde$treat))
ggplot(temp.data, aes(x = weights, fill = treated, color = treated)) +
geom_histogram(alpha = 0.5, position = "identity") +
xlab("Weights")
## We can also draw boxplots
ggplot(temp.data, aes(x = treated, y = weights, color = treated)) +
geom_boxplot()
- Check covariate balancing. If we do not perform any trimming and
just look at the full data, we can check covariate balancing from the
love plot or summary data of the standardized mean difference. Here, we
need to write our own function to draw the love plot.
## Need to change race (categorical) into indicators (numerical)
lalonde$black <- lalonde$race == "black"
lalonde$hispan <- lalonde$race == "hispan"
lalonde$white <- lalonde$race == "white"
## Draw love plot
love.plot = function(cov, treat, ## cov is the matrix of covariates and treat is a vector of treatment assignment
weights = rep(1, length(treat)),
plot = F)
{
## mean with normalized weights \sum w_i x_i / (\sum w_i)
treat.means <- colSums(cov[treat == 1,] * weights[treat == 1])/sum(weights[treat == 1])
treat.var <- colSums(t(t(cov[treat == 1,]) - treat.means)^2 *
weights[treat == 1])/sum(weights[treat == 1])
control.means <- colSums(cov[treat == 0,] * weights[treat == 0])/sum(weights[treat == 0])
control.var <- colSums(t(t(cov[treat == 0,]) - control.means)^2 *
weights[treat == 0])/sum(weights[treat == 0])
## the standardized mean differences for every covariate
smd <- (treat.means - control.means)/sqrt((treat.var + control.var)/2)
names(smd) <- colnames(cov)
if (plot == T) {
plot.data <- data.frame(smd = smd, covariates = names(smd))
range <- max(abs(smd))
ggplot(plot.data) + geom_point(aes(x = as.numeric(smd), y = covariates)) +
geom_vline(xintercept = 0) + xlim(-range, range) +
labs(x = 'Standardized Difference in Means')
}
return(smd)
}
colnames(lalonde)
[1] "treat" "age" "educ" "race" "married" "nodegree" "re74"
[8] "re75" "re78" "black" "hispan" "white"
raw.smd <- love.plot(lalonde[, c(2:3, 5:9, 10:12)], lalonde$treat)
weighted.smd <- love.plot(lalonde[, c(2:3, 5:9, 10:12)], lalonde$treat, weights = weights)
plot.data <- data.frame(smd = c(raw.smd, weighted.smd),
covariates = c(names(raw.smd), names(weighted.smd)),
category = c(rep("Original", length(raw.smd)), rep("IPW", length(weighted.smd))))
range <- max(abs(plot.data$smd))
ggplot(plot.data) + geom_point(aes(x = as.numeric(smd), y = covariates, color = category)) +
geom_vline(xintercept = c(-0.1, -0.05, 0, 0.05, 0.1),
linetype = c("solid", "dashed", "solid", "dashed", "solid")) +
xlim(-range, range) +
labs(x = 'Standardized Difference in Means')
Check weights and covariates balancing after trimming
Now we perform trimming and check covariate balancing again
## Histogram of estimated propensity score
temp.data <- data.frame(eps = eps, treated = as.factor(lalonde$treat))
ggplot(temp.data, aes(x = eps, fill = treated, color = treated)) +
geom_histogram(alpha = 0.5, position = "identity") + xlim(c(0, 1)) +
ggtitle("Histogram of eps before trimming")
rm.idx <- which(eps < 0.1 | eps > 0.9)
## remove control units
length(rm.idx)
[1] 237
## Check the histogram of eps again
ggplot(temp.data[-rm.idx, ], aes(x = eps, fill = treated, color = treated)) +
geom_histogram(alpha = 0.5, position = "identity") + xlim(c(0, 1)) +
ggtitle("Histogram of eps after trimming")
temp.data <- data.frame(weights = weights, treated = as.factor(lalonde$treat))
## We can also draw boxplots
ggplot(temp.data[-rm.idx, ], aes(x = treated, y = weights, color = treated)) +
geom_boxplot()
Check covariate balancing after trimming. Covariate balancing is much
better.
raw.smd <- love.plot(lalonde[-rm.idx, c(2:3, 5:9, 10:12)], lalonde$treat[-rm.idx])
weighted.smd <- love.plot(lalonde[-rm.idx, c(2:3, 5:9, 10:12)], lalonde$treat[-rm.idx],
weights = weights[-rm.idx])
plot.data <- data.frame(smd = c(raw.smd, weighted.smd),
covariates = c(names(raw.smd), names(weighted.smd)),
category = c(rep("Original", length(raw.smd)), rep("IPW", length(weighted.smd))))
range <- max(abs(plot.data$smd))
ggplot(plot.data) + geom_point(aes(x = as.numeric(smd), y = covariates, color = category)) +
geom_vline(xintercept = c(-0.1, -0.05, 0, 0.05, 0.1),
linetype = c("solid", "dashed", "solid", "dashed", "solid")) +
xlim(-range, range) +
labs(x = 'Standardized Difference in Means')
Estimate causal effect by weighted least square regression and
estimate the variance by Sandwich estimator
lm.result <- lm(re78 ~ treat, weights = weights[-rm.idx], data = lalonde[-rm.idx, ])
summary(lm.result)
Call:
lm(formula = re78 ~ treat, data = lalonde[-rm.idx, ], weights = weights[-rm.idx])
Weighted Residuals:
Min 1Q Median 3Q Max
-17233 -6644 -2553 4239 64304
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) 5135.9 480.0 10.700 <2e-16 ***
treat 1203.2 676.4 1.779 0.0761 .
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 9335 on 375 degrees of freedom
Multiple R-squared: 0.008368, Adjusted R-squared: 0.005723
F-statistic: 3.164 on 1 and 375 DF, p-value: 0.07607
library(sandwich)
tau_hat <- lm.result$coefficients[2]
SE <- sqrt(diag(vcovHC(lm.result, type = "HC2")))[2]
## get the 95% CI
result <- c(tau_hat, SE, c(tau_hat- 1.96 * SE, tau_hat + 1.96 * SE))
names(result) <- c("est", "sd", "CI_lower", "CI_upper")
result
est sd CI_lower CI_upper
1203.2270 768.3267 -302.6934 2709.1475
LS0tCnRpdGxlOiAiUiBFeGFtcGxlIDc6IEludmVyc2UgcHJvcGVuc2l0eSBzY29yZSB3ZWlnaHRpbmciCm91dHB1dDogaHRtbF9ub3RlYm9vawotLS0KCgoKV2Ugc3RpbGwgdXNlIHRoZSBsYWxvbmRlIGRhdGEgZnJvbSB0aGUgTWF0Y2hJdCBwYWNrYWdlIGFuZCB1c2UgdGhlIHByb3BlbnNpdHkgc2NvcmUgbW9kZWwgdGhhdCB3ZSBmb3VuZCBvdXQgaW4gUiBleGFtcGxlIDUKCmBgYHtyfQpsaWJyYXJ5KCJNYXRjaEl0IikKZGF0YSgibGFsb25kZSIpCgptb2RlbCA8LSBnbG0odHJlYXQgfiByZTc0ICsgcmFjZSArIG1hcnJpZWQgKyBJKHJlNzReMikgKyByZTc0OnJhY2UsIGRhdGEgPSBsYWxvbmRlLCBmYW1pbHkgPSAiYmlub21pYWwiKQplcHMgPC0gcHJlZGljdChtb2RlbCwgdHlwZSA9ICJyZXNwb25zZSIpCgpsYWxvbmRlCmBgYAoKIyMgQ2hlY2sgd2VpZ2h0cyBhbmQgY292YXJpYXRlcyBiYWxhbmNpbmcgYmVmb3JlIHRyaW1taW5nCgpUaGVuIHdlIHBlcmZvcm0gYSBmaXJzdCBjaGVjayBvZiB0aGUgd2VpZ2h0cyBieSBkcmF3aW5nIHRoZSBoaXN0b2dyYW0gb3IgYm94cGxvdCBvZiB0aGUgd2VpZ2h0cy4gQXQgdGhpcyBtb21lbnQsIHRoZSBmaWd1cmVzIGFyZSBub3QgdmVyeSBpbmZvcm1hdGl2ZSBhcyB0aGVyZSBhcmUgdW5pdHMgd2l0aCBleHRyZW1lbHkgbGFyZ2Ugd2VpZ2h0cy4KYGBge3J9CiMjIENhbGN1bGF0ZSB0aGUgcmF3IHdlaWdodHMgKGJlZm9yZSBub3JtYWxpemF0aW9uKQojIyBXZWlnaHRzIHRvIGVzdGltYXRlIEFURQpuLnRyZWF0ZWQgPC0gc3VtKGxhbG9uZGUkdHJlYXQgPT0gMSkKbi5jb250cm9sIDwtIHN1bShsYWxvbmRlJHRyZWF0ID09IDApCndlaWdodHMgPC0gaWZlbHNlKGxhbG9uZGUkdHJlYXQgPT0gMSwgMS9lcHMsIDEvKDEgLSBlcHMpKQoKIyMgQ2hlY2sgdGhlIHdlaWdodHMgaGlzdG9ncmFtCmxpYnJhcnkoZ2dwbG90MikKdGVtcC5kYXRhIDwtIGRhdGEuZnJhbWUod2VpZ2h0cyA9IHdlaWdodHMsIHRyZWF0ZWQgPSBhcy5mYWN0b3IobGFsb25kZSR0cmVhdCkpCmdncGxvdCh0ZW1wLmRhdGEsIGFlcyh4ID0gd2VpZ2h0cywgZmlsbCA9IHRyZWF0ZWQsIGNvbG9yID0gdHJlYXRlZCkpICsgCiAgZ2VvbV9oaXN0b2dyYW0oYWxwaGEgPSAwLjUsIHBvc2l0aW9uID0gImlkZW50aXR5IikgKyAKICB4bGFiKCJXZWlnaHRzIikgCgojIyBXZSBjYW4gYWxzbyBkcmF3IGJveHBsb3RzCmdncGxvdCh0ZW1wLmRhdGEsIGFlcyh4ID0gdHJlYXRlZCwgeSA9IHdlaWdodHMsIGNvbG9yID0gdHJlYXRlZCkpICsgCiAgZ2VvbV9ib3hwbG90KCkgCmBgYAoKLSBDaGVjayBjb3ZhcmlhdGUgYmFsYW5jaW5nLgpJZiB3ZSBkbyBub3QgcGVyZm9ybSBhbnkgdHJpbW1pbmcgYW5kIGp1c3QgbG9vayBhdCB0aGUgZnVsbCBkYXRhLCB3ZSBjYW4gY2hlY2sgY292YXJpYXRlIGJhbGFuY2luZyBmcm9tIHRoZSBsb3ZlIHBsb3Qgb3Igc3VtbWFyeSBkYXRhIG9mIHRoZSBzdGFuZGFyZGl6ZWQgbWVhbiBkaWZmZXJlbmNlLiBIZXJlLCB3ZSBuZWVkIHRvIHdyaXRlIG91ciBvd24gZnVuY3Rpb24gdG8gZHJhdyB0aGUgbG92ZSBwbG90LiAKYGBge3J9CiMjIE5lZWQgdG8gY2hhbmdlIHJhY2UgKGNhdGVnb3JpY2FsKSBpbnRvIGluZGljYXRvcnMgKG51bWVyaWNhbCkKbGFsb25kZSRibGFjayA8LSBsYWxvbmRlJHJhY2UgPT0gImJsYWNrIgpsYWxvbmRlJGhpc3BhbiA8LSBsYWxvbmRlJHJhY2UgPT0gImhpc3BhbiIKbGFsb25kZSR3aGl0ZSA8LSBsYWxvbmRlJHJhY2UgPT0gIndoaXRlIgoKIyMgRHJhdyBsb3ZlIHBsb3QKbG92ZS5wbG90ID0gZnVuY3Rpb24oY292LCB0cmVhdCwgICMjIGNvdiBpcyB0aGUgbWF0cml4IG9mIGNvdmFyaWF0ZXMgYW5kIHRyZWF0IGlzIGEgdmVjdG9yIG9mIHRyZWF0bWVudCBhc3NpZ25tZW50CiAgICAgICAgICAgICAgICAgICAgIHdlaWdodHMgPSByZXAoMSwgbGVuZ3RoKHRyZWF0KSksCiAgICAgICAgICAgICAgICAgICAgIHBsb3QgPSBGKSAKewogICAgCiAgICAjIyBtZWFuIHdpdGggbm9ybWFsaXplZCB3ZWlnaHRzIFxzdW0gd19pIHhfaSAvIChcc3VtIHdfaSkKICB0cmVhdC5tZWFucyA8LSBjb2xTdW1zKGNvdlt0cmVhdCA9PSAxLF0gKiB3ZWlnaHRzW3RyZWF0ID09IDFdKS9zdW0od2VpZ2h0c1t0cmVhdCA9PSAxXSkKICB0cmVhdC52YXIgPC0gY29sU3Vtcyh0KHQoY292W3RyZWF0ID09IDEsXSkgLSB0cmVhdC5tZWFucyleMiAqCiAgICAgICAgICAgICAgICAgICAgICAgICAgd2VpZ2h0c1t0cmVhdCA9PSAxXSkvc3VtKHdlaWdodHNbdHJlYXQgPT0gMV0pCiAgCiAgY29udHJvbC5tZWFucyA8LSBjb2xTdW1zKGNvdlt0cmVhdCA9PSAwLF0gKiB3ZWlnaHRzW3RyZWF0ID09IDBdKS9zdW0od2VpZ2h0c1t0cmVhdCA9PSAwXSkKICBjb250cm9sLnZhciA8LSBjb2xTdW1zKHQodChjb3ZbdHJlYXQgPT0gMCxdKSAtIGNvbnRyb2wubWVhbnMpXjIgKgogICAgICAgICAgICAgICAgICAgICAgICAgIHdlaWdodHNbdHJlYXQgPT0gMF0pL3N1bSh3ZWlnaHRzW3RyZWF0ID09IDBdKQogIAogICMjIHRoZSBzdGFuZGFyZGl6ZWQgbWVhbiBkaWZmZXJlbmNlcyBmb3IgZXZlcnkgY292YXJpYXRlCiAgc21kIDwtICh0cmVhdC5tZWFucyAtIGNvbnRyb2wubWVhbnMpL3NxcnQoKHRyZWF0LnZhciArIGNvbnRyb2wudmFyKS8yKQogIG5hbWVzKHNtZCkgPC0gY29sbmFtZXMoY292KQogIAogIGlmIChwbG90ID09IFQpIHsKICAgIHBsb3QuZGF0YSA8LSBkYXRhLmZyYW1lKHNtZCA9IHNtZCwgY292YXJpYXRlcyA9IG5hbWVzKHNtZCkpCiAgICByYW5nZSA8LSBtYXgoYWJzKHNtZCkpCiAgICBnZ3Bsb3QocGxvdC5kYXRhKSArIGdlb21fcG9pbnQoYWVzKHggPSBhcy5udW1lcmljKHNtZCksIHkgPSBjb3ZhcmlhdGVzKSkgKwogICAgICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSAwKSArIHhsaW0oLXJhbmdlLCByYW5nZSkgKwogICAgICBsYWJzKHggPSAnU3RhbmRhcmRpemVkIERpZmZlcmVuY2UgaW4gTWVhbnMnKQogIH0KICByZXR1cm4oc21kKQp9Cgpjb2xuYW1lcyhsYWxvbmRlKQpyYXcuc21kIDwtIGxvdmUucGxvdChsYWxvbmRlWywgYygyOjMsIDU6OSwgMTA6MTIpXSwgbGFsb25kZSR0cmVhdCkKd2VpZ2h0ZWQuc21kIDwtIGxvdmUucGxvdChsYWxvbmRlWywgYygyOjMsIDU6OSwgMTA6MTIpXSwgbGFsb25kZSR0cmVhdCwgd2VpZ2h0cyA9IHdlaWdodHMpCgoKcGxvdC5kYXRhIDwtIGRhdGEuZnJhbWUoc21kID0gYyhyYXcuc21kLCB3ZWlnaHRlZC5zbWQpLCAKICAgICAgICAgICAgICAgICAgICAgICAgY292YXJpYXRlcyA9IGMobmFtZXMocmF3LnNtZCksIG5hbWVzKHdlaWdodGVkLnNtZCkpLAogICAgICAgICAgICAgICAgICAgICAgICBjYXRlZ29yeSA9IGMocmVwKCJPcmlnaW5hbCIsIGxlbmd0aChyYXcuc21kKSksIHJlcCgiSVBXIiwgbGVuZ3RoKHdlaWdodGVkLnNtZCkpKSkKcmFuZ2UgPC0gbWF4KGFicyhwbG90LmRhdGEkc21kKSkKCmdncGxvdChwbG90LmRhdGEpICsgZ2VvbV9wb2ludChhZXMoeCA9IGFzLm51bWVyaWMoc21kKSwgeSA9IGNvdmFyaWF0ZXMsIGNvbG9yID0gY2F0ZWdvcnkpKSArCiAgICAgIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IGMoLTAuMSwgLTAuMDUsIDAsIDAuMDUsIDAuMSksCiAgICAgICAgICAgICAgICAgbGluZXR5cGUgPSBjKCJzb2xpZCIsICJkYXNoZWQiLCAic29saWQiLCAiZGFzaGVkIiwgInNvbGlkIikpICsgCiAgICAgIHhsaW0oLXJhbmdlLCByYW5nZSkgKwogICAgICBsYWJzKHggPSAnU3RhbmRhcmRpemVkIERpZmZlcmVuY2UgaW4gTWVhbnMnKQpgYGAKCiMjIENoZWNrIHdlaWdodHMgYW5kIGNvdmFyaWF0ZXMgYmFsYW5jaW5nIGFmdGVyIHRyaW1taW5nCgpOb3cgd2UgcGVyZm9ybSB0cmltbWluZyBhbmQgY2hlY2sgY292YXJpYXRlIGJhbGFuY2luZyBhZ2FpbgpgYGB7cn0KIyMgSGlzdG9ncmFtIG9mIGVzdGltYXRlZCBwcm9wZW5zaXR5IHNjb3JlCnRlbXAuZGF0YSA8LSBkYXRhLmZyYW1lKGVwcyA9IGVwcywgdHJlYXRlZCA9IGFzLmZhY3RvcihsYWxvbmRlJHRyZWF0KSkKZ2dwbG90KHRlbXAuZGF0YSwgYWVzKHggPSBlcHMsIGZpbGwgPSB0cmVhdGVkLCBjb2xvciA9IHRyZWF0ZWQpKSArIAogIGdlb21faGlzdG9ncmFtKGFscGhhID0gMC41LCBwb3NpdGlvbiA9ICJpZGVudGl0eSIpICsgeGxpbShjKDAsIDEpKSArCiAgZ2d0aXRsZSgiSGlzdG9ncmFtIG9mIGVwcyBiZWZvcmUgdHJpbW1pbmciKQoKcm0uaWR4IDwtIHdoaWNoKGVwcyA8IDAuMSB8IGVwcyA+IDAuOSkKIyMgcmVtb3ZlIGNvbnRyb2wgdW5pdHMKbGVuZ3RoKHJtLmlkeCkKCiMjIENoZWNrIHRoZSBoaXN0b2dyYW0gb2YgZXBzIGFnYWluCmdncGxvdCh0ZW1wLmRhdGFbLXJtLmlkeCwgXSwgYWVzKHggPSBlcHMsIGZpbGwgPSB0cmVhdGVkLCBjb2xvciA9IHRyZWF0ZWQpKSArIAogIGdlb21faGlzdG9ncmFtKGFscGhhID0gMC41LCBwb3NpdGlvbiA9ICJpZGVudGl0eSIpICsgeGxpbShjKDAsIDEpKSArCiAgZ2d0aXRsZSgiSGlzdG9ncmFtIG9mIGVwcyBhZnRlciB0cmltbWluZyIpCgoKdGVtcC5kYXRhIDwtIGRhdGEuZnJhbWUod2VpZ2h0cyA9IHdlaWdodHMsIHRyZWF0ZWQgPSBhcy5mYWN0b3IobGFsb25kZSR0cmVhdCkpCiMjIFdlIGNhbiBhbHNvIGRyYXcgYm94cGxvdHMKZ2dwbG90KHRlbXAuZGF0YVstcm0uaWR4LCBdLCBhZXMoeCA9IHRyZWF0ZWQsIHkgPSB3ZWlnaHRzLCBjb2xvciA9IHRyZWF0ZWQpKSArIAogIGdlb21fYm94cGxvdCgpIApgYGAKCkNoZWNrIGNvdmFyaWF0ZSBiYWxhbmNpbmcgYWZ0ZXIgdHJpbW1pbmcuIENvdmFyaWF0ZSBiYWxhbmNpbmcgaXMgbXVjaCBiZXR0ZXIuCmBgYHtyfQpyYXcuc21kIDwtIGxvdmUucGxvdChsYWxvbmRlWy1ybS5pZHgsIGMoMjozLCA1OjksIDEwOjEyKV0sIGxhbG9uZGUkdHJlYXRbLXJtLmlkeF0pCndlaWdodGVkLnNtZCA8LSBsb3ZlLnBsb3QobGFsb25kZVstcm0uaWR4LCBjKDI6MywgNTo5LCAxMDoxMildLCBsYWxvbmRlJHRyZWF0Wy1ybS5pZHhdLCAKICAgICAgICAgICAgICAgICAgICAgICAgICB3ZWlnaHRzID0gd2VpZ2h0c1stcm0uaWR4XSkKcGxvdC5kYXRhIDwtIGRhdGEuZnJhbWUoc21kID0gYyhyYXcuc21kLCB3ZWlnaHRlZC5zbWQpLCAKICAgICAgICAgICAgICAgICAgICAgICAgY292YXJpYXRlcyA9IGMobmFtZXMocmF3LnNtZCksIG5hbWVzKHdlaWdodGVkLnNtZCkpLAogICAgICAgICAgICAgICAgICAgICAgICBjYXRlZ29yeSA9IGMocmVwKCJPcmlnaW5hbCIsIGxlbmd0aChyYXcuc21kKSksIHJlcCgiSVBXIiwgbGVuZ3RoKHdlaWdodGVkLnNtZCkpKSkKcmFuZ2UgPC0gbWF4KGFicyhwbG90LmRhdGEkc21kKSkKZ2dwbG90KHBsb3QuZGF0YSkgKyBnZW9tX3BvaW50KGFlcyh4ID0gYXMubnVtZXJpYyhzbWQpLCB5ID0gY292YXJpYXRlcywgY29sb3IgPSBjYXRlZ29yeSkpICsKICAgICAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gYygtMC4xLCAtMC4wNSwgMCwgMC4wNSwgMC4xKSwKICAgICAgICAgICAgICAgICBsaW5ldHlwZSA9IGMoInNvbGlkIiwgImRhc2hlZCIsICJzb2xpZCIsICJkYXNoZWQiLCAic29saWQiKSkgKyAKICAgICAgeGxpbSgtcmFuZ2UsIHJhbmdlKSArCiAgICAgIGxhYnMoeCA9ICdTdGFuZGFyZGl6ZWQgRGlmZmVyZW5jZSBpbiBNZWFucycpCmBgYAoKIyMgRXN0aW1hdGUgY2F1c2FsIGVmZmVjdCBieSB3ZWlnaHRlZCBsZWFzdCBzcXVhcmUgcmVncmVzc2lvbiBhbmQgZXN0aW1hdGUgdGhlIHZhcmlhbmNlIGJ5IFNhbmR3aWNoIGVzdGltYXRvcgoKYGBge3J9CmxtLnJlc3VsdCA8LSBsbShyZTc4IH4gdHJlYXQsIHdlaWdodHMgPSB3ZWlnaHRzWy1ybS5pZHhdLCBkYXRhID0gbGFsb25kZVstcm0uaWR4LCBdKQpzdW1tYXJ5KGxtLnJlc3VsdCkKCmxpYnJhcnkoc2FuZHdpY2gpCnRhdV9oYXQgPC0gbG0ucmVzdWx0JGNvZWZmaWNpZW50c1syXQpTRSA8LSBzcXJ0KGRpYWcodmNvdkhDKGxtLnJlc3VsdCwgdHlwZSA9ICJIQzIiKSkpWzJdCgojIyBnZXQgdGhlIDk1JSBDSQpyZXN1bHQgPC0gYyh0YXVfaGF0LCBTRSwgYyh0YXVfaGF0LSAxLjk2ICogU0UsIHRhdV9oYXQgKyAxLjk2ICogU0UpKQpuYW1lcyhyZXN1bHQpIDwtIGMoImVzdCIsICJzZCIsICJDSV9sb3dlciIsICJDSV91cHBlciIpCnJlc3VsdApgYGA=