{ "_id": "6a3e3a37a319ef94144853a6", "Package": "mixedsubjectsirt", "Title": "Item Response Theory Calibration with a Mixed Subjects Design", "Version": "1.0.0", "Authors@R": "person(\"Klint\", \"Kanopka\", , \"klint.kanopka@nyu.edu\", role = c(\"aut\", \"cre\"),\ncomment = c(ORCID = \"0000-0003-3196-9538\"))", "Description": "Integrates large language model generated item responses\ninto psychometric calibration studies through a mixed-subjects\ndesign for unidimensional two-parameter and one-parameter\nlogistic item response theory models. Human pilot responses are\naugmented with model-generated responses using a\nprediction-powered inference estimator (Angelopoulos, Bates,\nFannjiang, Jordan and Zrnic (2023)\n; Angelopoulos, Duchi and Zrnic\n(2023) ) adapted to marginal\nmaximum-likelihood estimation, following the mixed-subjects\ndesign of Broska, Howes and van Loon (2025)\n. The estimator is anchored to\nthe human responses and is asymptotically unbiased for the\nhuman item parameters at any tuning weight; the weight on the\nsynthetic responses is chosen to minimize propagated\nability-score risk, down-weighting uninformative or biased\ngenerated responses. Louis-corrected sandwich standard errors,\nability scoring, cross-fitted tuning, and scale linking are\nalso provided.", "License": "MIT + file LICENSE", "Encoding": "UTF-8", "Language": "en-US", "Roxygen": "list(markdown = TRUE)", "RoxygenNote": "7.3.3", "VignetteBuilder": "knitr", "Config/testthat/edition": "3", "URL": "https://klintkanopka.com/mixedsubjectsirt/,\nhttps://github.com/klintkanopka/mixedsubjectsirt", "BugReports": "https://github.com/klintkanopka/mixedsubjectsirt/issues", "Config/pak/sysreqs": "cmake make libuv1-dev libx11-dev", "Repository": "https://klintkanopka.r-universe.dev", "Date/Publication": "2026-06-25 17:38:01 UTC", "RemoteUrl": "https://github.com/klintkanopka/mixedsubjectsirt", "RemoteRef": "HEAD", "RemoteSha": "291a1d21ceffc69aa272ac5c2c7b91145d39f841", "NeedsCompilation": "no", "Packaged": { "Date": "2026-06-26 08:33:05 UTC", "User": "root" }, "Author": "Klint Kanopka [aut, cre] (ORCID:\n)", "Maintainer": "Klint Kanopka ", "MD5sum": "220d7b477888935fc0e58a11027ee0b4", "_user": "klintkanopka", "_type": "src", "_file": "mixedsubjectsirt_1.0.0.tar.gz", "_fileid": "5229ede372d2cadd0ccbf78ae04ba3178d583203a2b71344add9cf0b264e20df", "_filesize": 925280, "_sha256": "5229ede372d2cadd0ccbf78ae04ba3178d583203a2b71344add9cf0b264e20df", "_created": "2026-06-26T08:33:05.000Z", "_published": "2026-06-26T08:37:11.922Z", "_distro": "resolute", "_jobs": [ { "job": 83620811963, "time": 209, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "7900887647" }, { "job": 83620811987, "time": 205, "config": "linux-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "7900886530" }, { "job": 83620811984, "time": 211, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "7900878126" }, { "job": 83620811956, "time": 203, "config": "macos-release-arm64", "r": "4.6.0", "check": "OK", "artifact": "7900875665" }, { "job": 83619837081, "time": 376, "config": "source", "r": "4.6.0", "check": "OK", "artifact": "7900816695" }, { "job": 83620811974, "time": 152, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "7900868631" }, { "job": 83620812032, "time": 147, "config": "windows-devel", "r": "4.7.0", "check": "OK", "artifact": "7900866898" }, { "job": 83620812076, "time": 164, "config": "windows-oldrel", "r": "4.5.3", "check": "OK", "artifact": "7900872881" }, { "job": 83620812042, "time": 164, "config": "windows-release", "r": "4.6.0", "check": "OK", "artifact": "7900872581" } ], "_buildurl": "https://github.com/r-universe/klintkanopka/actions/runs/28226552438", "_status": "success", "_host": "GitHub-Actions", "_upstream": "https://github.com/klintkanopka/mixedsubjectsirt", "_commit": { "id": "291a1d21ceffc69aa272ac5c2c7b91145d39f841", "author": "Klint Kanopka ", "committer": "Klint Kanopka ", "message": "badge fix\n", "time": 1782409081 }, "_maintainer": { "name": "Klint Kanopka", "email": "klint.kanopka@nyu.edu", "login": "klintkanopka", "bluesky": "@klint.bsky.social", "orcid": "0000-0003-3196-9538", "twitter": "@KlintKanopka", "description": "Assitant Professor of Applied Statistics at NYU ASH. Interested in measurement, psychometrics, networks, and machine learning.", "uuid": 28941934 }, "_registered": true, "_dependencies": [ { "package": "mirt", "role": "Imports" }, { "package": "rmutil", "role": "Imports" }, { "package": "ggplot2", "role": "Suggests" }, { "package": "knitr", "role": "Suggests" }, { "package": "rmarkdown", "role": "Suggests" }, { "package": "testthat", "version": ">= 3.0.0", "role": "Suggests" } ], "_owner": "klintkanopka", "_selfowned": true, "_usedby": 0, "_updates": [ { "week": "2026-16", "n": 2 }, { "week": "2026-23", "n": 12 }, { "week": "2026-24", "n": 6 }, { "week": "2026-26", "n": 7 } ], "_tags": [ { "name": "v0.1.0", "date": "2026-06-11" }, { "name": "v1.0.0", "date": "2026-06-22" } ], "_topics": [ "data-augmentation", "psychometrics" ], "_stars": 0, "_contributors": [ { "user": "klintkanopka", "count": 28, "uuid": 28941934 } ], "_userbio": { "uuid": 28941934, "type": "user", "name": "Klint Kanopka", "followers": 21, "description": "Assitant Professor of Applied Statistics at NYU ASH. Interested in measurement, psychometrics, networks, and machine learning." }, "_downloads": { "count": 0, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/mixedsubjectsirt" }, "_devurl": "https://github.com/klintkanopka/mixedsubjectsirt", "_pkgdown": "https://klintkanopka.com/mixedsubjectsirt/", "_searchresults": 0, "_rbuild": "4.6.0", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/mixedsubjectsirt.html", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md", "LICENSE", "manual.pdf" ], "_homeurl": "https://github.com/klintkanopka/mixedsubjectsirt", "_realowner": "klintkanopka", "_cranurl": true, "_releases": [ { "version": "1.0.0", "date": "2026-06-25" } ], "_exports": [ "ability_gradient", "ability_gradient_1pl", "ability_risk", "ability_risk_1pl", "diagnose_lambda_grid", "fit_1pl", "fit_2pl", "fit_mixed_subjects", "fit_mixed_subjects_1pl", "fit_mixed_subjects_from_quadrature", "fit_mixed_subjects_iterative", "fit_mixed_subjects_mml", "fit_mixed_subjects_mml_1pl", "fit_mixed_subjects_split", "link_item_parameters", "make_quadrature", "mixed_subjects_loss", "mixed_subjects_quadrature", "posterior_weights_2pl", "score_theta", "simulate_2pl", "summarize_expected_counts", "tune_lambda_ability_risk", "tune_lambda_ability_risk_1pl", "tune_lambda_ability_risk_crossfit", "tune_lambda_ability_risk_item", "tune_lambda_ppi_score", "tune_lambda_ppi_score_1pl", "tune_lambda_ppi_score_item", "vcov_mixed_subjects", "vcov_mixed_subjects_1pl", "vcov_mixed_subjects_mml" ], "_help": [ { "page": "ability_gradient", "title": "Gradient of ML ability scores with respect to item parameters", "topics": [ "ability_gradient" ] }, { "page": "ability_gradient_1pl", "title": "Gradient of ML ability scores w.r.t. 1PL item parameters", "topics": [ "ability_gradient_1pl" ] }, { "page": "ability_risk", "title": "Propagated ability risk from item-parameter uncertainty", "topics": [ "ability_risk" ] }, { "page": "ability_risk_1pl", "title": "Propagated ability risk for a 1PL fit", "topics": [ "ability_risk_1pl" ] }, { "page": "diagnose_lambda_grid", "title": "Diagnose lambda values over a grid", "topics": [ "diagnose_lambda_grid" ] }, { "page": "fit_1pl", "title": "Fit a 1PL (one-parameter logistic) model", "topics": [ "fit_1pl" ] }, { "page": "fit_2pl", "title": "Fit a unidimensional 2PL IRT model", "topics": [ "fit_2pl" ] }, { "page": "fit_mixed_subjects", "title": "Fit a mixed-subjects 2PL calibration", "topics": [ "fit_mixed_subjects" ] }, { "page": "fit_mixed_subjects_1pl", "title": "Fit a mixed-subjects 1PL calibration (frozen expected-count)", "topics": [ "fit_mixed_subjects_1pl" ] }, { "page": "fit_mixed_subjects_from_quadrature", "title": "Fit from precomputed quadrature summaries", "topics": [ "fit_mixed_subjects_from_quadrature" ] }, { "page": "fit_mixed_subjects_iterative", "title": "Fit a mixed-subjects 2PL calibration with iterative EM", "topics": [ "fit_mixed_subjects_iterative" ] }, { "page": "fit_mixed_subjects_mml", "title": "Fit a mixed-subjects 2PL calibration via marginal maximum likelihood", "topics": [ "fit_mixed_subjects_mml" ] }, { "page": "fit_mixed_subjects_mml_1pl", "title": "Fit a mixed-subjects 1PL calibration via marginal maximum likelihood", "topics": [ "fit_mixed_subjects_mml_1pl" ] }, { "page": "fit_mixed_subjects_split", "title": "Fit a split-sample mixed-subjects 2PL calibration", "topics": [ "fit_mixed_subjects_split" ] }, { "page": "link_item_parameters", "title": "Link item parameters onto a target scale", "topics": [ "link_item_parameters" ] }, { "page": "make_quadrature", "title": "Create a standard-normal Gauss-Hermite quadrature grid", "topics": [ "make_quadrature" ] }, { "page": "mixed_subjects_loss", "title": "Mixed-subjects objective function", "topics": [ "mixed_subjects_loss" ] }, { "page": "mixed_subjects_quadrature", "title": "Convert responses to quadrature form", "topics": [ "mixed_subjects_quadrature" ] }, { "page": "posterior_weights_2pl", "title": "Compute posterior quadrature weights for a 2PL model", "topics": [ "posterior_weights_2pl" ] }, { "page": "score_theta", "title": "Estimate ability scores from a 2PL calibration", "topics": [ "score_theta" ] }, { "page": "simulate_2pl", "title": "Simulate 2PL item responses", "topics": [ "simulate_2pl" ] }, { "page": "summarize_expected_counts", "title": "Summarize response data as expected quadrature counts", "topics": [ "summarize_expected_counts" ] }, { "page": "tune_lambda_ability_risk", "title": "Tune lambda by downstream ability-score risk", "topics": [ "tune_lambda_ability_risk" ] }, { "page": "tune_lambda_ability_risk_1pl", "title": "Tune lambda by downstream ability-score risk for a 1PL model", "topics": [ "tune_lambda_ability_risk_1pl" ] }, { "page": "tune_lambda_ability_risk_crossfit", "title": "Cross-fit ability-score-risk lambda tuning", "topics": [ "tune_lambda_ability_risk_crossfit" ] }, { "page": "tune_lambda_ability_risk_item", "title": "Per-item ability-risk lambda tuning via coordinate descent", "topics": [ "tune_lambda_ability_risk_item" ] }, { "page": "tune_lambda_ppi_score", "title": "Plug-in PPI++ optimal tuning parameter", "topics": [ "tune_lambda_ppi_score" ] }, { "page": "tune_lambda_ppi_score_1pl", "title": "Plug-in PPI++ optimal tuning parameter for a 1PL model", "topics": [ "tune_lambda_ppi_score_1pl" ] }, { "page": "tune_lambda_ppi_score_item", "title": "Per-item PPI++ optimal tuning parameters", "topics": [ "tune_lambda_ppi_score_item" ] }, { "page": "vcov_mixed_subjects", "title": "Sandwich covariance for a mixed-subjects fit", "topics": [ "vcov_mixed_subjects" ] }, { "page": "vcov_mixed_subjects_1pl", "title": "Sandwich covariance for a 1PL mixed-subjects fit", "topics": [ "vcov_mixed_subjects_1pl" ] }, { "page": "vcov_mixed_subjects_mml", "title": "Marginal-MML sandwich covariance for a mixed-subjects fit", "topics": [ "vcov_mixed_subjects_mml" ] } ], "_readme": "https://github.com/klintkanopka/mixedsubjectsirt/raw/HEAD/README.md", "_rundeps": [ "audio", "beepr", "brio", "callr", "class", "cli", "clipr", "cluster", "codetools", "crayon", "dcurver", "Deriv", "desc", "diffobj", "digest", "dplyr", "e1071", "evaluate", "fs", "future", "future.apply", "generics", "globals", "glue", "GPArotation", "gridExtra", "gtable", "jsonlite", "lattice", "lifecycle", "listenv", "magrittr", "MASS", "Matrix", "mgcv", "mirai", "mirt", "nanonext", "nlme", "otel", "parallelly", "pbapply", "permute", "pillar", "pkgbuild", "pkgconfig", "pkgload", "praise", "processx", "progressr", "proxy", "ps", "qs2", "R.methodsS3", "R.oo", "R.utils", "R6", "Rcpp", "RcppArmadillo", "RcppParallel", "rlang", "rmutil", "rprojroot", "sessioninfo", "SimDesign", "splines2", "stringfish", "testthat", "tibble", "tidyselect", "utf8", "vctrs", "vegan", "waldo", "withr" ], "_vignettes": [ { "source": "weakly-informative-llm.Rmd", "filename": "weakly-informative-llm.html", "title": "Calibrating with a Weakly-Informative, Biased LLM", "engine": "knitr::rmarkdown", "headings": [ "The setup", "Naive pooling inherits the bias", "$\\lambda$ moves efficiency, not bias", "Choosing $\\lambda$", "Takeaways", "Reproducing" ], "created": "2026-06-11 03:50:58", "modified": "2026-06-25 17:38:01", "commits": 4 }, { "source": "lambda-tuning.Rmd", "filename": "lambda-tuning.html", "title": "Choosing Lambda in Mixed-Subjects IRT", "engine": "knitr::rmarkdown", "headings": [ "Two objectives, two estimators", "Example data", "Ability-risk tuning: Minimizing $\\mathbb{E}[g'\\Sigma_\\gamma g]$", "Cross-fit $\\lambda$ tuning (recommended workflow)", "Frozen expected-count estimator (fast approximation)", "Minimizing $\\text{Tr}\\big[\\Sigma_\\gamma\\big]$ (diagnostic only)", "Choosing a procedure" ], "created": "2026-06-01 19:53:48", "modified": "2026-06-25 17:38:01", "commits": 9 }, { "source": "linking-comparison.Rmd", "filename": "linking-comparison.html", "title": "IRT Linking and Gradient Asymmetry: Diagnostic Guide", "engine": "knitr::rmarkdown", "headings": [ "Background", "Background (frozen expected-count estimator)", "Linking implementations", "Simulation", "Fitting human and LLM models", "Applying the three methods", "Parameter alignment after linking", "TCC alignment", "Gradient asymmetry: what linking fixes and what it does not", "Lambda sweep: how $\\lambda$ interacts with linking quality", "The role of power tuning", "Validation: what does $\\lambda^*$ measure?", "Test A — Perfect paired surrogate ($F = Y$)", "Test B — Partially overlapping predictions", "Test C — Stochastic LLM predictions (practical baseline)", "Summary: PPI++ score vs. ability risk", "Summary of findings", "Recommendation", "The marginal-MML fix" ], "created": "2026-06-03 18:33:27", "modified": "2026-06-25 17:38:01", "commits": 10 }, { "source": "mixed-subjects-1pl.Rmd", "filename": "mixed-subjects-1pl.html", "title": "Mixed-Subjects 1PL Calibration", "engine": "knitr::rmarkdown", "headings": [ "Simulate a 1PL test", "Step 1: Fit the 1PL baseline", "Step 2: Fit mixed-subjects MML (1PL)", "Step 3: Correct covariance — $(J+1) \\times (J+1)$ sandwich", "Step 4: Ability-score risk and lambda tuning", "Step 5: Verify — F = Y gives lambda > 0", "Compare 1PL and 2PL", "Ability-score risk: 1PL vs 2PL parameterization" ], "created": "2026-06-05 00:54:16", "modified": "2026-06-11 03:50:58", "commits": 3 }, { "source": "mixed-subjects-workflow.Rmd", "filename": "mixed-subjects-workflow.html", "title": "Mixed-Subjects IRT Calibration", "engine": "knitr::rmarkdown", "headings": [ "Simulate example data", "Step 1: Fit the human baseline", "Step 2: Fit the MML mixed-subjects model", "Step 3: Select $\\lambda$ by ability-score risk", "Step 3b (recommended workflow): cross-fit $\\lambda$ tuning", "Step 4: Inspect the covariance", "Compare calibrations", "When the LLM is uninformative", "Validation" ], "created": "2026-06-01 19:09:42", "modified": "2026-06-25 17:38:01", "commits": 10 }, { "source": "lambda-tuning-item.Rmd", "filename": "lambda-tuning-item.html", "title": "Per-Item Lambda (Experimental)", "engine": "knitr::rmarkdown", "headings": [ "Why per-item lambda?", "Simulate a heterogeneous test", "Step 1: Fit 2PL baseline and get global scalar lambda", "Step 2: PPI++ score per item (fast diagnostic)", "Step 3: Per-item ability-risk tuning", "Step 4: Compare scalar vs. per-item parameter recovery", "Important note on initialization", "Approximation caveat" ], "created": "2026-06-05 00:54:16", "modified": "2026-06-25 17:38:01", "commits": 5 }, { "source": "simulation-validation.Rmd", "filename": "simulation-validation.html", "title": "Simulation Validation of the Mixed-Subjects MML Estimator", "engine": "knitr::rmarkdown", "headings": [ "Design", "Does $\\lambda$-selection track predictor quality?", "Do standard errors achieve appropriate coverage?", "Does the method improve downstream scoring?", "What is the role of cross-fitting?", "Is coverage valid at the tuned $\\lambda$?", "Summary", "Reproducing these results" ], "created": "2026-06-09 14:29:52", "modified": "2026-06-25 17:38:01", "commits": 6 }, { "source": "understanding-ability-risk.Rmd", "filename": "understanding-ability-risk.html", "title": "Understanding Ability-Risk Tuning", "engine": "knitr::rmarkdown", "headings": [ "Why this vignette exists", "Key Intuition", "The three response matrices", "1. Observed human responses: $O$", "2. Paired LLM-predicted human responses: $P$", "3. Additional LLM-generated responses: $G$", "The mixed-subjects IRT objective", "What lambda is learning", "$$L_O^", "Ability-risk tuning", "The approximate target is$$\\widehat R(\\lambda)", "Why row alignment matters", "Case A: perfect paired prediction", "$$\\lambda_", "\\frac", "Case B: row-shuffled perfect predictions", "Case C: same DGP, fresh Bernoulli draw", "$$\\operatorname{Cov}(O_{ij},P_{ij})", "$$\\operatorname{Var}(P_{ij})", "What kind of LLM data produces higher lambda?", "One approach to row alignment: leave-one-item-out prediction", "Another approach: covariate-based prediction", "Something that probably won't work: item-text-only generation", "How to generate $G$", "Summary", "Technical Explanation", "Overview: four objects, one objective", "1. The estimator and its estimating equation", "2. The sandwich covariance of $\\hat\\gamma$", "3. Ability scoring and the implicit gradient", "4. Delta-method propagation and the risk", "5. Why this differs from the PPI++ trace objective" ], "created": "2026-06-25 17:38:01", "modified": "2026-06-25 17:38:01", "commits": 1 } ], "_score": 5.204119982655925, "_indexed": true, "_nocasepkg": "mixedsubjectsirt", "_universes": [ "klintkanopka" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "1.0.0", "date": "2026-06-26T08:35:28.000Z", "distro": "resolute", "commit": "291a1d21ceffc69aa272ac5c2c7b91145d39f841", "fileid": "e07469d01e2408e9473f0f1542234a3a0dc4773966aa8ec644993c3b9a7fff97", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/klintkanopka/actions/runs/28226552438" }, { "r": "4.6.1", "os": "linux", "version": "1.0.0", "date": "2026-06-26T08:35:28.000Z", "distro": "resolute", "commit": "291a1d21ceffc69aa272ac5c2c7b91145d39f841", "fileid": "89c48833e36e74f11e06178311d3fd3255f7878b9ccde4ee5f6da6e7c52e8c8a", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/klintkanopka/actions/runs/28226552438" }, { "r": "4.5.3", "os": "mac", "version": "1.0.0", "date": "2026-06-26T08:35:28.000Z", "commit": "291a1d21ceffc69aa272ac5c2c7b91145d39f841", "fileid": "1feb7ab4ca3c9ef5e24757e9d41d6639e3f3a806409e54df3f66d0bb8411c8ea", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/klintkanopka/actions/runs/28226552438" }, { "r": "4.6.0", "os": "mac", "version": "1.0.0", "date": "2026-06-26T08:35:24.000Z", "commit": "291a1d21ceffc69aa272ac5c2c7b91145d39f841", "fileid": "7f5c7364b9d7f4f70a29bdba19f85e6036cbfd4106fe7b5c0c2f9c2e6d1197a5", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/klintkanopka/actions/runs/28226552438" }, { "r": "4.6.0", "os": "wasm", "version": "1.0.0", "date": "2026-06-26T08:35:58.000Z", "commit": "291a1d21ceffc69aa272ac5c2c7b91145d39f841", "fileid": "b639940e206f0314d6900193495abdc95ad13b4a43410bab532406f3dd362376", "status": "success", "buildurl": "https://github.com/r-universe/klintkanopka/actions/runs/28226552438" }, { "r": "4.7.0", "os": "win", "version": "1.0.0", "date": "2026-06-26T08:34:20.000Z", "commit": "291a1d21ceffc69aa272ac5c2c7b91145d39f841", "fileid": "de67f0f54019c72468ceeddfc66e790d1dc4785c7e5129b915e6b78582ee38d2", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/klintkanopka/actions/runs/28226552438" }, { "r": "4.5.3", "os": "win", "version": "1.0.0", "date": "2026-06-26T08:34:34.000Z", "commit": "291a1d21ceffc69aa272ac5c2c7b91145d39f841", "fileid": "400fe4ea8d11be9b6c3a7c44893f60d7d269e79b55637fdff87cc7e27ae3068e", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/klintkanopka/actions/runs/28226552438" }, { "r": "4.6.0", "os": "win", "version": "1.0.0", "date": "2026-06-26T08:34:26.000Z", "commit": "291a1d21ceffc69aa272ac5c2c7b91145d39f841", "fileid": "a75290595e5acaa6bec2479d334709fbeb1aa2b231f1add399f045d4de813f72", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/klintkanopka/actions/runs/28226552438" } ] }