E3 — AI vs Teacher Solutions (Study 3, K2 campaign)

Author

EduStories / habilitace-2

Published

October 13, 2026

Formalises the completed E3 analysis (PLAN.md §5, frozen v1.0). Authoritative data = OSF package of the AIED manuscript (data/externi/osf-aied-ndtkz/), column Reseni_gen_AI: 0 = AI, 1 = teacher. This notebook (a) reproduces the manuscript’s LMMs (Table 3), (b) runs the pre-specified CLMM sensitivity, (c) computes inter-rater agreement with cluster-bootstrap 95% CIs, and (d) writes the numbers manifest vystupy/tabulky/kap7_k2_shoda_cisla.csv.

suppressPackageStartupMessages({
  library(dplyr); library(tidyr); library(lmerTest); library(ordinal); library(irr); library(digest)
})
SEED <- 20260714
set.seed(SEED)
ROOT <- normalizePath(file.path(getwd(), ".."))          # analyzy/
H2   <- normalizePath(file.path(ROOT, ".."))             # habilitace-2/
CSV  <- file.path(H2, "data/externi/osf-aied-ndtkz/anotace_final_data_bezdiakritiky.csv")
PUB  <- file.path(H2, "data/processed/kap7_hodnoceni_pseudo.csv")
DATA_SHA <- "ba464aaa3aa7223ba6a14009ec38dfb576db9af250079ed821bb6f9992478927"
# Studie 3 potřebuje k výpočtu jen HODNOCENÍ, ne texty kazuistik. Když je surový
# OSF soubor přítomen (autor), pseudonymizujeme anotátorky, ponecháme jen
# analytické sloupce (BEZ textů Popis/Anamneza/Řešení) a zapíšeme veřejnou
# tabulku; když chybí (Binder), čteme ji. Náhrada jmen 1:1 nemění žádný výsledek
# (anotátorka vstupuje jen jako náhodný efekt), texty korpusu zůstávají privátní.
KEEP <- c("Id", "anotator_name", "Otazka", "Hodnoceni", "Reseni_gen_AI",
          "Pristup.I", "Pristup.II", "Pristup.III")
df <- read.csv(PUB, header = TRUE, sep = ",", encoding = "UTF-8")
DATA_SHA <- digest::digest(file = PUB, algo = "sha256")
stopifnot(!anyDuplicated(df[c("Id", "anotator_name", "Otazka", "Reseni_gen_AI")]))
stopifnot(length(unique(df$Id)) == 324,
          all(sort(unique(na.omit(df$Hodnoceni))) == -2:2))
dims <- c(vhodnost     = "Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci.",
          reaktivni    = "Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni.",
          humanisticke = "Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane.",
          systemove    = "Jak.moc.je.popisovane.reseni.systemove.nebo.situacni.")
manifest <- list()
put <- function(metric, value) manifest[[length(manifest) + 1]] <<-
  data.frame(metric = metric, value = round(as.numeric(value), 4))
put("n_kazuistik", length(unique(df$Id)))
put("n_anotatorek", length(unique(df$anotator_name)))
put("n_odpovedi_dimenze", nrow(df))
put("n_missing_dimenze", sum(is.na(df$Hodnoceni)))

Descriptives by source

desc <- df |>
  filter(!is.na(Hodnoceni)) |>
  mutate(dim = names(dims)[match(Otazka, dims)],
         src = ifelse(Reseni_gen_AI == 0, "AI", "teacher")) |>
  group_by(dim, src) |>
  summarise(n = n(), mean = mean(Hodnoceni), sd = sd(Hodnoceni), .groups = "drop")
knitr::kable(desc, digits = 2)
dim src n mean sd
humanisticke AI 1727 -0.44 1.00
humanisticke teacher 1737 0.34 1.25
reaktivni AI 1730 0.69 1.02
reaktivni teacher 1741 -0.32 1.24
systemove AI 1730 0.35 1.07
systemove teacher 1741 0.17 1.39
vhodnost AI 1729 1.56 0.61
vhodnost teacher 1741 0.71 0.95 
for (i in seq_len(nrow(desc)))
  put(paste0("mean_", desc$src[i], "_", desc$dim[i]), desc$mean[i])

Primary models: LMM (manuscript reproduction)

lmm_rows <- lapply(names(dims), function(nm) {
  d <- df[df$Otazka == dims[[nm]], ]
  m <- lmer(Hodnoceni ~ Reseni_gen_AI + (1 | Id) + (1 | anotator_name),
            data = d, REML = FALSE)
  fe <- summary(m)$coefficients
  ci <- confint(m, parm = "beta_", method = "Wald")
  put(paste0("lmm_b0_AI_", nm), fe[1, 1])
  put(paste0("lmm_b1_ucitel_", nm), fe[2, 1])
  put(paste0("lmm_b1_se_", nm), fe[2, 2])
  put(paste0("lmm_b1_ci_lo_", nm), ci[2, 1])
  put(paste0("lmm_b1_ci_hi_", nm), ci[2, 2])
  data.frame(dim = nm, b0_AI = fe[1, 1], b1_teacher = fe[2, 1],
             se = fe[2, 2], ci_lo = ci[2, 1], ci_hi = ci[2, 2], p = fe[2, 5])
})
knitr::kable(bind_rows(lmm_rows), digits = 3)
dim b0_AI b1_teacher se ci_lo ci_hi p
vhodnost 1.547 -0.855 0.022 -0.898 -0.811 0
reaktivni 0.683 -1.010 0.029 -1.066 -0.954 0
humanisticke -0.410 0.772 0.029 0.714 0.830 0
systemove 0.321 -0.172 0.032 -0.233 -0.110 0

Expected (AIED Table 3): appropriateness −0.85; reactive −1.01; humanistic +0.77; systemic −0.17 — reproduced exactly.

Sensitivity: CLMM (pre-specified)

clmm_rows <- lapply(names(dims), function(nm) {
  d <- df[df$Otazka == dims[[nm]] & !is.na(df$Hodnoceni), ]
  m <- clmm(factor(Hodnoceni, ordered = TRUE) ~ Reseni_gen_AI +
              (1 | Id) + (1 | anotator_name), data = d)
  co <- summary(m)$coefficients
  b <- co[grep("Reseni_gen_AI", rownames(co)), , drop = FALSE]
  put(paste0("clmm_b1_ucitel_", nm), b[1, 1])
  put(paste0("clmm_or_ucitel_", nm), exp(b[1, 1]))
  # Wald 95% CI na škále OR (pro tabulku 7.2 a figuru)
  lo <- b[1, 1] - 1.96 * b[1, 2]; hi <- b[1, 1] + 1.96 * b[1, 2]
  put(paste0("clmm_or_ci_lo_", nm), exp(lo))
  put(paste0("clmm_or_ci_hi_", nm), exp(hi))
  data.frame(dim = nm, b1_teacher = b[1, 1], SE = b[1, 2],
             OR = exp(b[1, 1]), OR_lo = exp(lo), OR_hi = exp(hi), p = b[1, 4])
})
knitr::kable(bind_rows(clmm_rows), digits = 3)
dim b1_teacher SE OR OR_lo OR_hi p
vhodnost -2.790 0.089 0.061 0.052 0.073 0
reaktivni -2.252 0.077 0.105 0.091 0.122 0
humanisticke 1.683 0.071 5.383 4.684 6.187 0
systemove -0.256 0.065 0.775 0.682 0.880 0

Sensitivity check: CLMM must agree in direction with LMM for all four dimensions.

Sensitivity: LMM with a random slope for source by case

The design is paired within case (both solutions of the same case are rated), so the source effect may vary across cases. The random-intercept-only model assumes a constant contrast; here we relax it with (1 + source | Id) and check whether the fixed effect and its CI move (opponent finding, 2026-07-17).

slope_rows <- lapply(names(dims), function(nm) {
  d <- df[df$Otazka == dims[[nm]], ]
  m <- suppressWarnings(
    lmer(Hodnoceni ~ Reseni_gen_AI + (1 + Reseni_gen_AI | Id) +
           (1 | anotator_name), data = d, REML = FALSE))
  fe <- summary(m)$coefficients
  ci <- suppressWarnings(confint(m, parm = "beta_", method = "Wald"))
  put(paste0("lmm_slope_b1_ucitel_", nm), fe[2, 1])
  put(paste0("lmm_slope_ci_lo_", nm), ci[2, 1])
  put(paste0("lmm_slope_ci_hi_", nm), ci[2, 2])
  put(paste0("lmm_slope_singular_", nm), as.integer(isSingular(m)))
  data.frame(dim = nm, b1_teacher = fe[2, 1],
             ci_lo = ci[2, 1], ci_hi = ci[2, 2], singular = isSingular(m))
})
knitr::kable(bind_rows(slope_rows), digits = 3)
dim b1_teacher ci_lo ci_hi singular
vhodnost -0.857 -0.932 -0.781 FALSE
reaktivni -1.019 -1.125 -0.913 FALSE
humanisticke 0.767 0.668 0.867 FALSE
systemove -0.164 -0.275 -0.053 FALSE

Sensitivity: source effect varying across annotators

The primary models let annotators differ in overall severity (random intercept), not in how strongly they separate the two sources. Two checks (independent audit, 2026-10-13): (a) a random slope for source by annotator, (b) a fixed source × annotator interaction that gives each annotator’s own teacher − AI contrast (min, max and sign count across the six annotators).

ann_rows <- lapply(names(dims), function(nm) {
  d <- df[df$Otazka == dims[[nm]] & !is.na(df$Hodnoceni), ]
  m <- suppressWarnings(
    lmer(Hodnoceni ~ Reseni_gen_AI + (1 | Id) + (1 + Reseni_gen_AI | anotator_name),
         data = d, REML = FALSE))
  fe <- summary(m)$coefficients
  ci <- suppressWarnings(confint(m, parm = "beta_", method = "Wald"))
  put(paste0("lmm_annslope_b1_ucitel_", nm), fe[2, 1])
  put(paste0("lmm_annslope_ci_lo_", nm), ci[2, 1])
  put(paste0("lmm_annslope_ci_hi_", nm), ci[2, 2])
  put(paste0("lmm_annslope_p_", nm), fe[2, 5])
  put(paste0("lmm_annslope_singular_", nm), as.integer(isSingular(m)))
  d$ann <- factor(d$anotator_name)
  mi <- lmer(Hodnoceni ~ Reseni_gen_AI * ann + (1 | Id), data = d, REML = FALSE)
  b <- lme4::fixef(mi)
  per_ann <- sapply(levels(d$ann), function(a) {
    k <- paste0("Reseni_gen_AI:ann", a)
    b[["Reseni_gen_AI"]] + if (k %in% names(b)) b[[k]] else 0
  })
  put(paste0("lmm_ann_b1_min_", nm), min(per_ann))
  put(paste0("lmm_ann_b1_max_", nm), max(per_ann))
  put(paste0("lmm_ann_n_kladny_", nm), sum(per_ann > 0))
  put(paste0("lmm_ann_n_zaporny_", nm), sum(per_ann < 0))
  data.frame(dim = nm, b1_slope_model = fe[2, 1], ci_lo = ci[2, 1], ci_hi = ci[2, 2],
             p = fe[2, 5], singular = isSingular(m),
             per_annotator_min = min(per_ann), per_annotator_max = max(per_ann),
             n_positive = sum(per_ann > 0), n_negative = sum(per_ann < 0))
})
knitr::kable(bind_rows(ann_rows), digits = 3)
dim b1_slope_model ci_lo ci_hi p singular per_annotator_min per_annotator_max n_positive n_negative
vhodnost -0.862 -0.948 -0.775 0.0 FALSE -0.986 -0.682 0 6
reaktivni -1.003 -1.123 -0.883 0.0 FALSE -1.237 -0.845 0 6
humanisticke 0.773 0.715 0.831 0.0 TRUE 0.671 0.963 6 0
systemove -0.138 -0.438 0.161 0.4 FALSE -0.870 0.391 2 4

Missingness by source, including structurally absent rows

Blinding hides explicit source labels, not all textual source cues. It does not establish ignorable missingness. Separate missing ratings within observed rows from absent rows in the intended case × rater × dimension × source grid.

mis <- df |>
  mutate(src = ifelse(Reseni_gen_AI == 0, "AI", "ucitel")) |>
  group_by(src) |>
  summarise(n = n(), n_na = sum(is.na(Hodnoceni)),
            podil_na = mean(is.na(Hodnoceni)), .groups = "drop")
knitr::kable(mis, digits = 4)
src n n_na podil_na
AI 7776 860 0.1106
ucitel 7700 740 0.0961 
for (i in seq_len(nrow(mis)))
  put(paste0("missing_podil_", mis$src[i]), mis$podil_na[i])
rating_grid <- tidyr::expand_grid(Id = unique(df$Id), anotator_name = unique(df$anotator_name),
                                Otazka = unname(dims), Reseni_gen_AI = c(0, 1)) |>
  left_join(mutate(df, observed_row = TRUE), by = c("Id", "anotator_name", "Otazka", "Reseni_gen_AI"))
put("n_ocekavanych_bunek", nrow(rating_grid))
put("n_chybejicich_radku", sum(is.na(rating_grid$observed_row)))
put("n_chybejicich_bunek_celkem", sum(is.na(rating_grid$Hodnoceni)))
missing_grid <- rating_grid |>
  group_by(Reseni_gen_AI, Otazka, anotator_name) |>
  summarise(expected = n(), absent = sum(is.na(observed_row)),
            observed_missing = sum(!is.na(observed_row) & is.na(Hodnoceni)), .groups = "drop")
write.csv(missing_grid, file.path(H2, "vystupy/tabulky/kap7_chybeni.csv"), row.names = FALSE)
knitr::kable(missing_grid)
Reseni_gen_AI Otazka anotator_name expected absent observed_missing
0 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A1 324 0 3
0 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A2 324 0 0
0 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A3 324 0 178
0 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A4 324 0 34
0 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A5 324 0 2
0 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A6 324 0 0
0 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A1 324 0 2
0 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A2 324 0 0
0 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A3 324 0 178
0 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A4 324 0 33
0 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A5 324 0 1
0 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A6 324 0 0
0 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A1 324 0 2
0 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A2 324 0 0
0 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A3 324 0 178
0 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A4 324 0 33
0 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A5 324 0 1
0 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A6 324 0 0
0 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A1 324 0 2
0 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A2 324 0 0
0 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A3 324 0 179
0 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A4 324 0 33
0 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A5 324 0 1
0 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A6 324 0 0
1 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A1 324 0 2
1 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A2 324 0 2
1 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A3 324 0 151
1 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A4 324 19 33
1 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A5 324 0 0
1 Jak.moc.je.popisovane.reseni.humanisticky.nebo.behavioralne.orientovane. A6 324 0 0
1 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A1 324 0 1
1 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A2 324 0 2
1 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A3 324 0 150
1 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A4 324 19 31
1 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A5 324 0 0
1 Jak.moc.je.popisovane.reseni.reaktivni.nebo.proaktivni. A6 324 0 0
1 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A1 324 0 1
1 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A2 324 0 2
1 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A3 324 0 150
1 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A4 324 19 31
1 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A5 324 0 0
1 Jak.moc.je.popisovane.reseni.systemove.nebo.situacni. A6 324 0 0
1 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A1 324 0 1
1 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A2 324 0 2
1 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A3 324 0 150
1 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A4 324 19 31
1 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A5 324 0 0
1 Jak.moc.je.vhodna.toto.reseni.pro.danou.situaci. A6 324 0 0

Post-audit sensitivity: complete AI–teacher pairs per rater and dimension

complete_keys <- df |>
  group_by(Id, anotator_name, Otazka) |>
  summarise(complete = n_distinct(Reseni_gen_AI) == 2 && all(!is.na(Hodnoceni)), .groups = "drop") |>
  filter(complete)
paired_ratings <- semi_join(df, complete_keys, by = c("Id", "anotator_name", "Otazka"))
paired_results <- bind_rows(lapply(names(dims), function(dimension) {
  selected <- paired_ratings[paired_ratings$Otazka == dims[[dimension]], ]
  fitted <- lmer(Hodnoceni ~ Reseni_gen_AI + (1 | Id) + (1 | anotator_name), data = selected, REML = FALSE)
  estimate <- summary(fitted)$coefficients["Reseni_gen_AI", "Estimate"]
  interval <- confint(fitted, parm = "Reseni_gen_AI", method = "Wald")
  put(paste0("paired_n_", dimension), nrow(selected))
  put(paste0("paired_b1_", dimension), estimate)
  put(paste0("paired_ci_lo_", dimension), interval[1, 1])
  put(paste0("paired_ci_hi_", dimension), interval[1, 2])
  data.frame(dimension = dimension, n = nrow(selected), estimate = estimate,
             ci_lo = interval[1, 1], ci_hi = interval[1, 2])
}))
write.csv(paired_results, file.path(H2, "vystupy/tabulky/kap7_uplne_dvojice.csv"), row.names = FALSE)
knitr::kable(paired_results, digits = 4)
dimension n estimate ci_lo ci_hi
vhodnost 3314 -0.8588 -0.9024 -0.8152
reaktivni 3314 -1.0109 -1.0679 -0.9538
humanisticke 3306 0.7623 0.7037 0.8208
systemove 3314 -0.1871 -0.2494 -0.1248

Inter-rater agreement with cluster-bootstrap 95% CIs

Unit = (Id × source); Krippendorff’s alpha (ordinal); bootstrap resamples the 324 case Ids with replacement (B = 1000, seed 2.0260714^{7}).

make_wide <- function(d) {
  w <- d |> mutate(item = paste0(Id, "_", Reseni_gen_AI)) |>
    select(anotator_name, item, Hodnoceni) |>
    distinct(anotator_name, item, .keep_all = TRUE) |>
    pivot_wider(names_from = item, values_from = Hodnoceni)
  m <- as.matrix(w[, -1]); rownames(m) <- w[[1]]
  m[, colSums(!is.na(m)) >= 2, drop = FALSE]
}
alpha_of <- function(m) suppressWarnings(irr::kripp.alpha(m, method = "ordinal")$value)

B <- 1000
ids <- unique(df$Id)
alpha_rows <- lapply(names(dims), function(nm) {
  d <- df[df$Otazka == dims[[nm]] & !is.na(df$Hodnoceni), ]
  a <- alpha_of(make_wide(d))
  full_matrix <- make_wide(d)
  columns_by_id <- lapply(as.character(ids), function(case_id) {
    which(colnames(full_matrix) %in% paste0(case_id, "_", c(0, 1)))
  })
  boots <- vapply(seq_len(B), function(b) {
    selected_cases <- sample(seq_along(ids), replace = TRUE)
    selected_columns <- unlist(columns_by_id[selected_cases], use.names = FALSE)
    alpha_of(full_matrix[, selected_columns, drop = FALSE])
  }, numeric(1))
  ci <- quantile(boots, c(.025, .975), na.rm = TRUE)
  put(paste0("alpha_ord_final_", nm), a)
  put(paste0("alpha_ci_lo_", nm), ci[1])
  put(paste0("alpha_ci_hi_", nm), ci[2])
  data.frame(dim = nm, alpha = a, ci_lo = ci[1], ci_hi = ci[2])
})
knitr::kable(bind_rows(alpha_rows), digits = 3)
dim alpha ci_lo ci_hi
2.5%…1 vhodnost 0.527 0.485 0.566
2.5%…2 reaktivni 0.645 0.610 0.677
2.5%…3 humanisticke 0.529 0.491 0.565
2.5%…4 systemove 0.556 0.520 0.592

Approach labels: Jaccard (both conventions)

pr <- unique(df[, c("Id", "anotator_name", "Reseni_gen_AI",
                    "Pristup.I", "Pristup.II", "Pristup.III")])
for (c_ in c("Pristup.I", "Pristup.II", "Pristup.III"))
  pr[[c_]][pr[[c_]] %in% c("", "nic")] <- NA
pr$item <- paste0(pr$Id, "_", pr$Reseni_gen_AI)
sets <- pr |> rowwise() |>
  mutate(s = list(na.omit(c(Pristup.I, Pristup.II, Pristup.III)))) |> ungroup() |>
  select(anotator_name, item, s) |> distinct(anotator_name, item, .keep_all = TRUE)
rat <- unique(sets$anotator_name)
jac <- function(sets) {
  inc <- c(); exc <- c()
  for (i in seq_along(rat)) for (j in seq_along(rat)) if (i < j) {
    a <- sets |> filter(anotator_name == rat[i])
    b <- sets |> filter(anotator_name == rat[j])
    m2 <- inner_join(a, b, by = "item", suffix = c("_a", "_b"))
    inc <- c(inc, mean(mapply(function(x, y) {
      if (!length(x) && !length(y)) return(1)
      length(intersect(x, y)) / length(union(x, y)) }, m2$s_a, m2$s_b)))
    exc <- c(exc, mean(mapply(function(x, y) {
      if (!length(x) && !length(y)) return(NA_real_)
      length(intersect(x, y)) / length(union(x, y)) }, m2$s_a, m2$s_b), na.rm = TRUE))
  }
  c(incl = mean(inc), excl = mean(exc))
}
j <- jac(sets)
put("jaccard_pristupy_obe_prazdne_shoda", j["incl"])
put("jaccard_pristupy_bez_prazdnych", j["excl"])
knitr::kable(data.frame(konvence = c("both-empty = agreement (paper)", "exclude both-empty"),
                        jaccard = round(j, 3)))
konvence jaccard
incl both-empty = agreement (paper) 0.725
excl exclude both-empty 0.123

Approach labels: prevalence and GLMM (RQ2 of the manuscript)

Binary outcome: any framework label assigned to a solution by a rater. GLMM with logit link, random intercepts for case and annotator (as in the manuscript’s analyses.R; expected OR ≈ 0.31 for teacher solutions).

pr2 <- pr |> rowwise() |>
  mutate(n_filled = sum(!is.na(c_across(starts_with("Pristup"))))) |> ungroup() |>
  group_by(Id, Reseni_gen_AI, anotator_name) |>
  summarise(n_filled = max(n_filled), .groups = "drop") |>
  mutate(filled_binary = as.integer(n_filled > 0))
n_labels <- sum(!is.na(pr$Pristup.I)) + sum(!is.na(pr$Pristup.II)) + sum(!is.na(pr$Pristup.III))
lab_by_src <- pr |> pivot_longer(starts_with("Pristup"), values_to = "lab") |>
  filter(!is.na(lab)) |> count(Reseni_gen_AI)
podil_AI <- lab_by_src$n[lab_by_src$Reseni_gen_AI == 0] / sum(lab_by_src$n)
gm <- lme4::glmer(filled_binary ~ Reseni_gen_AI + (1 | Id) + (1 | anotator_name),
                  data = pr2, family = binomial)
gs <- summary(gm)$coefficients
gci <- exp(confint(gm, parm = "beta_", method = "Wald"))
put("ramce_n_labelu_celkem", n_labels)
put("ramce_podil_labelu_AI", podil_AI)
put("glmm_or_ucitel_ramce", exp(gs[2, 1]))
put("glmm_or_ci_lo_ramce", gci[2, 1])
put("glmm_or_ci_hi_ramce", gci[2, 2])
knitr::kable(data.frame(
  n_labels_total = n_labels, share_AI = round(podil_AI, 3),
  OR_teacher = round(exp(gs[2, 1]), 3),
  ci = sprintf("[%.3f; %.3f]", gci[2, 1], gci[2, 2]), p = signif(gs[2, 4], 3)))
n_labels_total share_AI OR_teacher ci p
874 0.717 0.313 [0.259; 0.377] 0

Freeze: write manifest

out <- bind_rows(manifest) |>
  mutate(skript = "analyzy/notebooks/30_ai_vs_ucitel.qmd",
         data_sha = DATA_SHA, seed = SEED, datum = format(Sys.Date()))
dir.create(file.path(H2, "vystupy/tabulky"), showWarnings = FALSE, recursive = TRUE)
write.csv(out, file.path(H2, "vystupy/tabulky/kap7_k2_shoda_cisla.csv"), row.names = FALSE)
cat("Manifest:", nrow(out), "metrik → vystupy/tabulky/kap7_k2_shoda_cisla.csv\n")
Manifest: 140 metrik → vystupy/tabulky/kap7_k2_shoda_cisla.csv
sessionInfo()$R.version$version.string
[1] "R version 4.6.1 (2026-06-24 ucrt)"