{ "_id": "6aafa6633d719c1f8408035a", "Package": "aiEvalR", "Type": "Package", "Title": "Statistical and Psychometric Evaluation of AI Systems", "Version": "0.1.0", "Authors@R": "person(\"Subir\", \"Hait\", email = \"haitsubi@msu.edu\",\nrole = c(\"aut\", \"cre\"), comment = c(ORCID = \"0009-0004-9871-9677\"))", "Description": "Evaluates artificial intelligence (AI) systems as\nmeasurement instruments using psychometric methods. Provides\nmulti-facet generalizability theory (G-study and D-study) via\n'lme4', reliability via the intraclass correlation coefficient\n(ICC), calibration via the expected calibration error (ECE) and\nBrier score, robustness stress testing, and group disparity\ndiagnostics. Item-level differential item functioning (DIF)\nbased on item response theory (IRT) is delegated to the 'aiDIF'\npackage. Methods follow Cronbach, Gleser, Nanda and Rajaratnam\n(1972, ) and Brennan (2001)\n.", "License": "MIT + file LICENSE", "Encoding": "UTF-8", "Language": "en-US", "Roxygen": "list(markdown = TRUE)", "Config/testthat/edition": "3", "URL": "https://github.com/causalfragility-lab/aiEvalR", "BugReports": "https://github.com/causalfragility-lab/aiEvalR/issues", "VignetteBuilder": "knitr", "NeedsCompilation": "no", "Packaged": { "Date": "2026-09-20 09:21:37 UTC", "User": "root" }, "Author": "Subir Hait [aut, cre] ()", "Maintainer": "Subir Hait ", "Config/roxygen2/version": "8.0.0", "Repository": "https://subirhait.r-universe.dev", "Date/Publication": "2026-08-06 20:25:11 UTC", "RemoteUrl": "https://github.com/subirhait/aievalr", "RemoteRef": "HEAD", "RemoteSha": "79c93e72ed8db61a7a8a91ce4d2cb6e19bfb9b52", "_user": "subirhait", "_type": "src", "_file": "aiEvalR_0.1.0.tar.gz", "_fileid": "https://r2.ropensci.org/f320fa2bf1b013dd616828ecd22d2da8be3137b0d8ce3af4f28b4617af7e5226", "_filesize": 188418, "_sha256": "f320fa2bf1b013dd616828ecd22d2da8be3137b0d8ce3af4f28b4617af7e5226", "_expires": "2026-12-29T09:24:49.000Z", "_created": "2026-09-20T09:21:37.000Z", "_published": "2026-09-20T09:24:51.335Z", "_jobs": [ { "job": 106055570515, "time": 149, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "10602328855" }, { "job": 106055570505, "time": 150, "config": "linux-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "10602418744" }, { "job": 106055570542, "time": 136, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "10602252401" }, { "job": 106055570477, "time": 152, "config": "macos-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "10602119137" }, { "job": 106055158788, "time": 187, "config": "source", "r": "4.6.1", "check": "OK", "artifact": "10602781586" }, { "job": 106055570448, "time": 112, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "10602293868" }, { "job": 106055570506, "time": 84, "config": "windows-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "10601934426" }, { "job": 106055570493, "time": 77, "config": "windows-oldrel-x86_64", "r": "4.5.3", "check": "OK", "artifact": "10601819827" }, { "job": 106055570487, "time": 79, "config": "windows-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "10602916564" } ], "_host": "GitHub-Actions", "_buildurl": "https://github.com/r-universe/subirhait/actions/runs/35501931746", "_status": "success", "_upstream": "https://github.com/subirhait/aievalr", "_commit": { "id": "79c93e72ed8db61a7a8a91ce4d2cb6e19bfb9b52", "author": "Subir Hait <264290991+causalfragility-lab@users.noreply.github.com>", "committer": "Subir Hait <264290991+causalfragility-lab@users.noreply.github.com>", "message": "Address CRAN review: unwrap dontrun example, add examples, update wordlist\n", "time": 1786047911 }, "_maintainer": { "name": "Subir Hait", "email": "haitsubi@msu.edu", "login": "subirhait", "linkedin": "in/subirhait", "description": "PhD Candidate in Measurement & Quantitative Methods at Michigan State University | Causal inference, sensitivity analysis, and statistical software", "uuid": 264290991, "orcid": "0009-0004-9871-9677" }, "_distro": "resolute", "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 4.1.0", "role": "Depends" }, { "package": "stats", "role": "Imports" }, { "package": "utils", "role": "Imports" }, { "package": "testthat", "version": ">= 3.0.0", "role": "Suggests" }, { "package": "knitr", "role": "Suggests" }, { "package": "rmarkdown", "role": "Suggests" }, { "package": "covr", "role": "Suggests" }, { "package": "boot", "role": "Suggests" }, { "package": "lme4", "role": "Suggests" }, { "package": "dplyr", "role": "Suggests" }, { "package": "ggplot2", "role": "Suggests" }, { "package": "aiDIF", "role": "Suggests" }, { "package": "spelling", "role": "Suggests" } ], "_owner": "subirhait", "_selfowned": true, "_usedby": 0, "_updates": [ { "week": "2026-31", "n": 5 }, { "week": "2026-32", "n": 1 } ], "_tags": [], "_stars": 0, "_contributors": [ { "user": "subirhait", "count": 6, "uuid": 264290991 } ], "_userbio": { "uuid": 264290991, "type": "user", "name": "Subir Hait", "followers": 4, "description": "PhD Candidate in Measurement & Quantitative Methods at Michigan State University | Causal inference, sensitivity analysis, and statistical software" }, "_downloads": { "count": 265, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/aiEvalR" }, "_devurl": "https://github.com/causalfragility-lab/aievalr", "_rbuild": "4.6.1", "_assets": [ "extra/aiEvalR.html", "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/manual.pdf", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md", "LICENSE" ], "_homeurl": "https://github.com/subirhait/aievalr", "_realowner": "subirhait", "_cranurl": true, "_releases": [ { "version": "0.1.0", "date": "2026-09-08" } ], "_exports": [ "adversarial_stability", "ai_bootstrap_reliability", "ai_calibration", "ai_conditional_sem", "ai_dashboard", "ai_decision_consistency", "ai_dif_mh", "ai_dstudy", "ai_fairness", "ai_generalizability", "ai_group_disparity", "ai_internal_consistency", "ai_reliability", "ai_robustness", "ai_test_retest", "brier_score", "calibration_curve", "citation_accuracy", "ece", "empirical_output_interval", "fairness_dashboard", "hallucination_rate", "lexical_overlap", "prompt_sensitivity", "stress_test" ], "_help": [ { "page": "adversarial_stability", "title": "Adversarial stability", "topics": [ "adversarial_stability" ] }, { "page": "ai_bootstrap_reliability", "title": "Bootstrap confidence interval for AI reliability estimates", "topics": [ "ai_bootstrap_reliability" ] }, { "page": "ai_calibration", "title": "Overall calibration summary for AI system outputs", "topics": [ "ai_calibration" ] }, { "page": "ai_conditional_sem", "title": "Conditional standard error of measurement for AI outputs", "topics": [ "ai_conditional_sem" ] }, { "page": "ai_dashboard", "title": "Integrated AI Evaluation Dashboard", "topics": [ "ai_dashboard" ] }, { "page": "ai_decision_consistency", "title": "Decision consistency for AI classification/scoring decisions", "topics": [ "ai_decision_consistency" ] }, { "page": "ai_dif_mh", "title": "Mantel-Haenszel differential functioning test for AI-scored binary items", "topics": [ "ai_dif_mh" ] }, { "page": "ai_dstudy", "title": "D-study: project generalizability/dependability for given facet sizes", "topics": [ "ai_dstudy" ] }, { "page": "ai_fairness", "title": "Overall fairness summary for AI system outputs", "topics": [ "ai_fairness" ] }, { "page": "ai_generalizability", "title": "Multi-facet generalizability (G-)study for AI outputs", "topics": [ "ai_generalizability" ] }, { "page": "ai_group_disparity", "title": "Group-level disparity in AI outcomes", "topics": [ "ai_group_disparity" ] }, { "page": "ai_internal_consistency", "title": "Internal consistency of AI responses across parallel items", "topics": [ "ai_internal_consistency" ] }, { "page": "ai_reliability", "title": "Overall reliability summary for AI system outputs", "topics": [ "ai_reliability" ] }, { "page": "ai_robustness", "title": "Overall robustness summary for AI system outputs", "topics": [ "ai_robustness" ] }, { "page": "ai_test_retest", "title": "Test-retest reliability of AI responses", "topics": [ "ai_test_retest" ] }, { "page": "brier_score", "title": "Brier score", "topics": [ "brier_score" ] }, { "page": "calibration_curve", "title": "Calibration curve (reliability diagram data)", "topics": [ "calibration_curve" ] }, { "page": "citation_accuracy", "title": "Citation accuracy", "topics": [ "citation_accuracy" ] }, { "page": "ece", "title": "Expected Calibration Error", "topics": [ "ece" ] }, { "page": "empirical_output_interval", "title": "Empirical output interval for AI numeric outputs", "topics": [ "empirical_output_interval" ] }, { "page": "fairness_dashboard", "title": "Fairness dashboard summary (text form)", "topics": [ "fairness_dashboard" ] }, { "page": "hallucination_rate", "title": "Hallucination rate across AI outputs", "topics": [ "hallucination_rate" ] }, { "page": "lexical_overlap", "title": "Lexical overlap between an AI output and a reference source", "topics": [ "lexical_overlap" ] }, { "page": "prompt_sensitivity", "title": "Prompt sensitivity analysis", "topics": [ "prompt_sensitivity" ] }, { "page": "stress_test", "title": "Stress test AI responses under a set of perturbations", "topics": [ "stress_test" ] } ], "_readme": "https://github.com/subirhait/aievalr/raw/HEAD/README.md", "_rundeps": [], "_vignettes": [ { "source": "getting-started.Rmd", "filename": "getting-started.html", "title": "Getting Started with aiEvalR", "engine": "knitr::rmarkdown", "headings": [ "Why evaluate an AI system like a test?", "Reliability: is the system consistent?", "Robustness: does the answer survive small changes?", "Fairness: do outcomes differ by group?", "Calibration: are stated confidences trustworthy?", "Hallucination: aggregating adjudicated labels", "An integrated dashboard", "Where to next" ], "created": "2026-07-27 22:27:43", "modified": "2026-07-27 22:27:43", "commits": 1 }, { "source": "psychometric-core.Rmd", "filename": "psychometric-core.html", "title": "The Generalizability-Theory Core", "engine": "knitr::rmarkdown", "headings": [ "From one reliability number to a variance decomposition", "A G-study", "A D-study: how many conditions do you need?", "Conditional error and decision consistency", "References" ], "created": "2026-07-27 22:27:43", "modified": "2026-07-27 22:27:43", "commits": 1 } ], "_score": 4.301029995663981, "_indexed": true, "_nocasepkg": "aievalr", "_universes": [ "subirhait", "causalfragility-lab" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "0.1.0", "date": "2026-09-20T09:24:03.000Z", "distro": "resolute", "commit": "79c93e72ed8db61a7a8a91ce4d2cb6e19bfb9b52", "fileid": "https://r2.ropensci.org/3b2a77e7fb381394240b9131ff364fe3df045e1dac43172c2dba02340894ef0c", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/subirhait/actions/runs/35501931746" }, { "r": "4.6.1", "os": "linux", "version": "0.1.0", "date": "2026-09-20T09:24:05.000Z", "distro": "resolute", "commit": "79c93e72ed8db61a7a8a91ce4d2cb6e19bfb9b52", "fileid": "https://r2.ropensci.org/bb82d1c792bbc4af47a1c7217a3450781b2612930dbe57c8ce14af810b6e4240", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/subirhait/actions/runs/35501931746" }, { "r": "4.5.3", "os": "mac", "version": "0.1.0", "date": "2026-09-20T09:23:44.000Z", "commit": "79c93e72ed8db61a7a8a91ce4d2cb6e19bfb9b52", "fileid": "https://r2.ropensci.org/416771fc740e65c0177932cb62ce89005745fc9e5de93ad13a5088cf251ed0c6", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/subirhait/actions/runs/35501931746" }, { "r": "4.6.1", "os": "mac", "version": "0.1.0", "date": "2026-09-20T09:23:49.000Z", "commit": "79c93e72ed8db61a7a8a91ce4d2cb6e19bfb9b52", "fileid": "https://r2.ropensci.org/f11676648dfe67eb7a03adb6f6a868b296733a3d01009f694e899558dbf957f7", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/subirhait/actions/runs/35501931746" }, { "r": "4.6.0", "os": "wasm", "version": "0.1.0", "date": "2026-09-20T09:23:42.000Z", "commit": "79c93e72ed8db61a7a8a91ce4d2cb6e19bfb9b52", "fileid": "https://r2.ropensci.org/8b2807794a67405d8067be2285d32900a4df61b424834e59063e368f85631f96", "status": "success", "buildurl": "https://github.com/r-universe/subirhait/actions/runs/35501931746" }, { "r": "4.7.0", "os": "win", "version": "0.1.0", "date": "2026-09-20T09:22:54.000Z", "commit": "79c93e72ed8db61a7a8a91ce4d2cb6e19bfb9b52", "fileid": "https://r2.ropensci.org/b4df6610f65a2bd19c70026ce75afb50c8abe0ce8159b948d0af46bfa68b02ba", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/subirhait/actions/runs/35501931746" }, { "r": "4.5.3", "os": "win", "version": "0.1.0", "date": "2026-09-20T09:22:49.000Z", "commit": "79c93e72ed8db61a7a8a91ce4d2cb6e19bfb9b52", "fileid": "https://r2.ropensci.org/941080a125ed528c9f933b10ab78ff1c39e6034f4f65700a679692c73ff875fc", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/subirhait/actions/runs/35501931746" }, { "r": "4.6.1", "os": "win", "version": "0.1.0", "date": "2026-09-20T09:22:50.000Z", "commit": "79c93e72ed8db61a7a8a91ce4d2cb6e19bfb9b52", "fileid": "https://r2.ropensci.org/25dc2053990659c24b118d97ef44029b06ed5a6694035853a50832c3949e8f78", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/subirhait/actions/runs/35501931746" } ] }