classroom.yml

name: Grade Submission
# This workflow runs in submission repos (e.g. DataScience4Psych/2026-lab_02),
# which are created from this template (DataScience4Psych/actions).
#
# When a student opens an issue here with a link to their work repo, this
# workflow clones their code, runs the matching testthat suite, and posts
# results back as a comment on the issue.
#
# Required secret: GRADING_PAT — a PAT with read access to student repos
# (only needed if student repos are private).
#
# ── Reusable workflow ────────────────────────────────────────────────────────
# Other submission repos can call this workflow instead of copying it:
#
#   jobs:
#     grade:
#       uses: DataScience4Psych/actions/.github/workflows/classroom.yml@main
#       secrets: inherit
#
# That single call replaces the entire workflow body and automatically picks
# up any future changes made here.
# ─────────────────────────────────────────────────────────────────────────────
on:
  workflow_call:
    inputs:
      lab:
        description: 'Override which lab to grade (e.g. lab02). Leave blank for auto-detection.'
        required: false
        type: string
        default: ''
    secrets:
      GRADING_PAT:
        required: false
  workflow_dispatch:
    inputs:
      lab:
        description: 'Override which lab to grade (e.g. lab02). Leave blank for auto-detection.'
        required: false
        type: string
        default: ''
  issues:
    types: [opened]
  issue_comment:
    types: [created]

permissions:
  issues: write
  contents: read

# Cancel earlier in-progress runs when a new grading request arrives for the
# same issue, so only the latest attempt consumes runner minutes.
# concurrency:
#  group: ${{ github.repository }}-grade-${{ github.event.issue.number || github.run_id }}
#  cancel-in-progress: true

jobs:
  grade:
    runs-on: ubuntu-latest
    timeout-minutes: 45
    # On issue open: always run (except bots).
    # On comment: only run if the comment body is exactly /grade or /regrade.
    # On workflow_call: always run (caller decides when to invoke).
    if: |
      github.actor != 'github-classroom[bot]' && (
        github.event_name == 'workflow_call' ||
        github.event_name == 'workflow_dispatch' ||
        github.event_name == 'issues' ||
        (github.event_name == 'issue_comment' && (
          startsWith(github.event.comment.body, '/grade') ||
          startsWith(github.event.comment.body, '/regrade') ||
          startsWith(github.event.comment.body, '/feedback') ||
          startsWith(github.event.comment.body, '/update')
        ))
      )
    env:
      GH_TOKEN: ${{ github.token }}

    steps:
    - name: Acknowledge submission
      uses: actions/github-script@v7
      with:
        script: |
          const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
          const { data: comment } = await github.rest.issues.createComment({
            owner: context.repo.owner,
            repo: context.repo.repo,
            issue_number: context.issue.number,
            body: `:hourglass_flowing_sand: Grading request received from **${context.actor}** — [running now](${runUrl}). Results will appear here when complete.`
          });
          require('fs').writeFileSync('/tmp/ack_comment_id.txt', String(comment.id));
    # Extract the student's GitHub repo URL from the issue body.
    # On /grade rerun, the URL is in the original issue body (not the comment).
    # Fallback: if no URL is provided, try {issue-author}/{this-repo-name}.
    - name: Extract student repo URL
      id: extract
      env:
        ISSUE_BODY: ${{ github.event.issue.body }}
      run: |
        repo_url=$(echo "$ISSUE_BODY" \
          | grep -oP 'https?://github\.com/[A-Za-z0-9._-]+/[A-Za-z0-9._-]+' \
          | head -1 \
          | sed 's|\.git$||')
        if [ -n "$repo_url" ]; then
          repo_path=$(echo "$repo_url" | sed 's|https://github.com/||')
        else
          author="${{ github.event.issue.user.login }}"
          repo_path="${author}/${{ github.event.repository.name }}"
          _ack_id=$(cat /tmp/ack_comment_id.txt 2>/dev/null || echo '')
          _msg=":mag: No repo URL found in your submission — trying \`${repo_path}\`. Results will appear here when grading completes."
          if [ -n "$_ack_id" ]; then
            gh api "repos/${{ github.repository }}/issues/comments/$_ack_id" -X PATCH -f body="$_msg"
          else
            gh issue comment ${{ github.event.issue.number }} --body "$_msg" --repo ${{ github.repository }}
          fi
        fi
        echo "repo_path=$repo_path" >> $GITHUB_OUTPUT
        echo "Student repo: $repo_path"

    # Detect which lab to grade.
    # Priority 1: explicit override via workflow input (inputs.lab)
    # Priority 2: lab identifier in comment body, e.g. "/grade lab02"
    # Priority 3: lab identifier in issue body, e.g. a line "lab: lab02"
    # Priority 4: student's repo name, e.g. "lab-02-plastic-waste" -> "lab02"
    # Fallback:   submission repo name, e.g. "2026-lab_02"          -> "lab02"
    - name: Detect lab
      id: detect-lab
      env:
        COMMENT_BODY: ${{ github.event.comment.body }}
        ISSUE_BODY: ${{ github.event.issue.body }}
      run: |
        normalize_lab() {
          raw="$1"
          num=$(echo "$raw" | grep -oP '\d+' | head -1)
          if [ -n "$num" ]; then
            printf 'lab%02d\n' "$((10#$num))"
          fi
        }

        # ── Priority 1: explicit override via workflow input ──────────────────
        input_lab="${{ inputs.lab }}"
        if [ -n "$input_lab" ]; then
          lab=$(normalize_lab "$input_lab")
          lab="${lab:-$input_lab}"
          echo "Lab overridden by workflow input: $lab"
          echo "lab=$lab" >> $GITHUB_OUTPUT
          echo "Detected lab: $lab"
          exit 0
        fi

        # ── Priority 2: lab override in comment body ("/grade lab02" etc.) ────
        if [ -n "$COMMENT_BODY" ]; then
          comment_override=$(echo "$COMMENT_BODY" | head -1 | grep -oiP 'labs?[_-]?\d+' | head -1)
          if [ -n "$comment_override" ]; then
            lab=$(normalize_lab "$comment_override")
            if [ -n "$lab" ]; then
              echo "Lab overridden by comment body: $lab"
              echo "lab=$lab" >> $GITHUB_OUTPUT
              echo "Detected lab: $lab"
              exit 0
            fi
          fi
        fi

        # ── Priority 3: lab override in issue body ("lab: lab02") ─────────────
        if [ -n "$ISSUE_BODY" ]; then
          body_override=$(echo "$ISSUE_BODY" | grep -iP '^lab:' | grep -oiP 'labs?[_-]?\d+' | head -1)
          if [ -n "$body_override" ]; then
            lab=$(normalize_lab "$body_override")
            if [ -n "$lab" ]; then
              echo "Lab overridden by issue body: $lab"
              echo "lab=$lab" >> $GITHUB_OUTPUT
              echo "Detected lab: $lab"
              exit 0
            fi
          fi
        fi

        # ── Priority 4: auto-detect from student's repo name ──────────────────
        student_repo=$(echo "${{ steps.extract.outputs.repo_path }}" | cut -d/ -f2)
        # Accept forms like:
        #   lab1, lab01, lab-1, lab_1, labs1, labs-01, etc.
        raw_lab=$(echo "$student_repo" | grep -oiP 'labs?[_-]?\d+' | head -1)

        if [ -n "$raw_lab" ]; then
          lab=$(normalize_lab "$raw_lab")
        fi

        if [ -z "$lab" ]; then
          # Fallback: use the submission repo name
          submission_repo="${{ github.event.repository.name }}"
          lab=$(echo "$submission_repo" | grep -oiP 'lab[_]?\d+' | head -1 | tr -d '_' | tr '[:upper:]' '[:lower:]')
          if [ -z "$lab" ]; then
            num=$(echo "$submission_repo" | grep -oP '\d+' | tail -1)
            if [ -n "$num" ]; then
              lab="lab$(printf '%02d' "$((10#$num))")"
            fi
          fi
          if [ -z "$lab" ]; then
            _ack_id=$(cat /tmp/ack_comment_id.txt 2>/dev/null || echo '')
            _msg=":x: Could not detect a lab number from your repo name or the submission repo name. Please include your repo URL in the issue body."
            if [ -n "$_ack_id" ]; then
              gh api "repos/${{ github.repository }}/issues/comments/$_ack_id" -X PATCH -f body="$_msg"
            else
              gh issue comment ${{ github.event.issue.number }} --body "$_msg" --repo ${{ github.repository }}
            fi
            exit 1
          fi
          echo "::warning::Lab inferred from submission repo '$submission_repo': $lab"
        fi
        echo "lab=$lab" >> $GITHUB_OUTPUT
        echo "Detected lab: $lab"

    - name: Checkout student code
      uses: actions/checkout@v4
      with:
        repository: ${{ steps.extract.outputs.repo_path }}
        token: ${{ secrets.GRADING_PAT || github.token }}
        path: student-code

    # Check that the student pushed a knitted output file (md or html) and that
    # every locally-referenced image inside it is present in the repository.
    # Remote URLs (http/https/data:) are skipped for the image check.
    # Results are written to /tmp/missing_output.txt and /tmp/missing_images.txt
    # so the Post grading results step can include them in the comment.
    - name: Check for missing knitted output and images
      working-directory: student-code
      run: |
        # ── 1. Check that a knitted output file exists ────────────────────────
        knitted=$(find . -maxdepth 1 \( -iname "*.md" -o -iname "*.html" \) \
          | grep -iv 'template\|solution\|example' | head -1)
        if [ -z "$knitted" ]; then
          echo "::warning::No knitted md or html file found in the student repo."
          echo "missing" > /tmp/missing_output.txt
          exit 0
        fi
        echo "Found knitted output: $knitted"

        # ── 2. Scan it for locally-referenced images ──────────────────────────
        echo "Checking image references in: $knitted"

        # Collect image paths from:
        #   1. Markdown syntax : ![alt](path)
        #   2. HTML img tags   : <img src="path"> / <img src='path'>
        {
          grep -oP '!\[[^\]]*\]\(\K[^)]+' "$knitted" 2>/dev/null || true
          grep -oP '<img[^>]+src="\K[^"]+' "$knitted" 2>/dev/null || true
          grep -oP "<img[^>]+src='\K[^']+" "$knitted" 2>/dev/null || true
        } | sort -u | while IFS= read -r img_path; do
          # Skip remote URLs and data URIs
          echo "$img_path" | grep -qiP '^https?://|^data:' && continue
          # Strip surrounding whitespace
          img_path=$(echo "$img_path" | xargs 2>/dev/null || echo "$img_path")
          [ -z "$img_path" ] && continue
          [ ! -f "$img_path" ] && echo "$img_path"
        done > /tmp/missing_images.txt

        if [ -s /tmp/missing_images.txt ]; then
          echo "::warning::Missing images detected in student submission:"
          while IFS= read -r f; do
            echo "::warning::Missing image: $f"
          done < /tmp/missing_images.txt
        else
          echo "All referenced images are present in the repository."
          rm -f /tmp/missing_images.txt
        fi

    # Tests live in DataScience4Psych/DataScience4Psych at tests/<lab>/testthat/
    - name: Checkout autograding tests
      uses: actions/checkout@v4
      with:
        repository: DataScience4Psych/DataScience4Psych
        path: autograding
        sparse-checkout: tests

    - name: Verify lab test directory exists
      run: |
        test_path="$GITHUB_WORKSPACE/autograding/tests/${{ steps.detect-lab.outputs.lab }}/testthat"
        if [ ! -d "$test_path" ]; then
          _ack_id=$(cat /tmp/ack_comment_id.txt 2>/dev/null || echo '')
          _msg=":x: No autograding tests found for \`${{ steps.detect-lab.outputs.lab }}\`. Please contact your instructor @smasongarrison."
          if [ -n "$_ack_id" ]; then
            gh api "repos/${{ github.repository }}/issues/comments/$_ack_id" -X PATCH -f body="$_msg"
          else
            gh issue comment ${{ github.event.issue.number }} --body "$_msg" --repo ${{ github.repository }}
          fi
          echo "::error::No tests found at $test_path"
          exit 1
        fi
        n=$(find "$test_path" -name 'test-*.R' | wc -l)
        echo "Found $n test file(s) in $test_path"

    - name: Set up R
      uses: r-lib/actions/setup-r@v2
      with:
        use-public-rspm: true

    - name: Install R dependencies
      uses: r-lib/actions/setup-r-dependencies@v2
      with:
        packages: |
          any::testthat
          any::tidyverse
          any::stringr
          any::knitr
          any::pak

    # Scan the student's Rmd for library()/require() calls and PACKAGE:: usage
    # and install any packages not already available, so labs with different
    # dependencies work.
    - name: Install student package dependencies
      working-directory: student-code
      run: |
        Rscript --vanilla -e "
          rmd <- list.files('.', pattern = '\\\\.Rmd$', ignore.case = TRUE,
                            recursive = FALSE, full.names = TRUE)
          rmd <- rmd[!grepl('(template|solution|example)', basename(rmd), ignore.case = TRUE)]
          if (length(rmd) == 0) quit(status = 0)
          txt <- readLines(rmd[1], warn = FALSE)
          pkgs1 <- regmatches(txt, gregexpr('(?<=library\\\\(|require\\\\()([A-Za-z0-9_.]+)', txt, perl = TRUE))
          pkgs2 <- regmatches(txt, gregexpr('(?<!:)[A-Za-z0-9_.]+(?=:::?)', txt, perl = TRUE))
          pkgs <- unique(c(unlist(pkgs1), unlist(pkgs2)))
          pkgs <- pkgs[!vapply(pkgs, requireNamespace, logical(1), quietly = TRUE)]
          if('dsbox' %in% pkgs){
          pkgs[pkgs == 'dsbox'] <- 'tidyverse/dsbox'
          }
          if('emo' %in% pkgs){
           pkgs[pkgs == 'emo'] <- 'hadley/emo'
          }
          if (length(pkgs) > 0) {
            message('Installing missing packages: ', paste(pkgs, collapse = ', '))
            pak::pak(pkgs)
          } else {
            message('All student packages already installed.')
          }
        "

    # Run tests with student code as working directory so relative file paths
    # inside test files resolve correctly.
    # Student Rmd is purled+sourced into .GlobalEnv first so all objects the
    # tests reference (variables, functions, plots) are available.
    # Proportional score = passing_expectations / total * 100.
    - name: Run automatic tests
      id: run-tests
      working-directory: student-code
      env:
        STUDENT_CODE_DIR: ${{ github.workspace }}/student-code
      run: |
        lab="${{ steps.detect-lab.outputs.lab }}"
        test_path="$GITHUB_WORKSPACE/autograding/tests/${lab}/testthat"

        Rscript --vanilla -e "
          # Source student Rmd into .GlobalEnv so test assertions can see objects.
          # Use STUDENT_CODE_DIR so this works regardless of testthat's cwd.
          student_dir <- Sys.getenv('STUDENT_CODE_DIR', unset = '.')
          rmd <- list.files(student_dir, pattern = '\\\\.Rmd$', ignore.case = TRUE,
                            recursive = FALSE, full.names = TRUE)
          rmd <- rmd[!grepl('(template|solution|example)', basename(rmd), ignore.case = TRUE)]
          if (length(rmd) > 0) {
            r_script <- tempfile(fileext = '.R')
            knitr::purl(rmd[1], output = r_script, quiet = TRUE)
            code_err <- tryCatch(
              { source(r_script, local = FALSE); NULL },
              error = function(e) e
            )
            if (!is.null(code_err)) {
              writeLines(conditionMessage(code_err), '/tmp/code_error.txt')
              message('Student code error (grading continues): ', conditionMessage(code_err))
            }
          } else {
            message('No student Rmd found; tests will run without student environment.')
          }
          # Read .R files from subfolders into .r_script_content so tests
          # can check code patterns in R scripts separately from Rmd content.
          .r_script_files <- list.files(student_dir, pattern = '\\\\.R$',
                                        ignore.case = TRUE, recursive = TRUE,
                                        full.names = TRUE)
          .r_script_files <- .r_script_files[!grepl('(template|solution|example)',
                                                     basename(.r_script_files),
                                                     ignore.case = TRUE)]
          .r_script_content <- if (length(.r_script_files) > 0) {
            message('Loaded content from ', length(.r_script_files), ' .R file(s) into .r_script_content')
            unlist(lapply(.r_script_files, function(f) {
              tryCatch(readLines(f, warn = FALSE), error = function(e) character(0))
            }))
          } else {
            character(0)
          }


          
          shared_test_dir <- normalizePath(file.path('${test_path}', '..', '..'), mustWork = FALSE)
          source(file.path(shared_test_dir, 'shared-helper-load.R'), local = FALSE)

          universal_test_file <- file.path(shared_test_dir, 'test-universal-rmd-code.R')
          result_universal <- NULL
          tap_lines_universal <- character(0)
          if (file.exists(universal_test_file)) {
            tap_lines_universal <- tryCatch(
              capture.output(
                result_universal <- testthat::test_file(universal_test_file, reporter = 'tap',
                                                        stop_on_failure = FALSE),
                type = 'output'
              ),
              error = function(e) {
                message('Error running universal test file (', universal_test_file, '): ', conditionMessage(e))
                character(0)
              }
            )
          } else {
            message('Universal test file not found: ', universal_test_file)
          }

          tap_lines_lab <- capture.output(
            result_lab <-  testthat::test_dir('${test_path}', reporter = 'tap',
                                         stop_on_failure = FALSE),
            type = 'output'
          )

          df <- if (!is.null(result_universal)) {
            rbind(as.data.frame(result_universal), as.data.frame(result_lab))
          } else {
            as.data.frame(result_lab)
          }
          passing <- sum(df\$passed)
          total   <- passing + sum(df\$failed) + sum(df\$error)
          score   <- if (total > 0L) as.integer(round(passing / total * 100)) else 0L
          status  <- if (passing == total && total > 0L) 'pass' else 'fail'

          tap_lines <- c(tap_lines_universal, tap_lines_lab)

          skip_symbol <- sample(c('\u2205','\u26A0','\u2753','\u2754','\u29BB','\u2B55'),1)
          success_symbol <- sample(c('\u2611','\u2611','\u2705','\u2713','\u2714'),1)
          unsuccess_symbol <- sample(c('\u2573','\u26D2','\u2715','\u2716','\u2717','\u2718','\u274C','\u274E','\u292B','\u292C'),1)
          
          rows <- character(0)
          i <- 1L
          while (i <= length(tap_lines)) {
            line <- tap_lines[i]
            if (grepl('^ok ', line)) {
              desc <- sub('^ok [0-9]+ - ', '', line)
              if(grepl('[[:space:]]*#[[:space:]]*SKIP', line)) {
              rows <- c(rows, paste0('| ',skip_symbol,' | ', desc, ' | |'))
              }else{
              rows <- c(rows, paste0('| ',success_symbol,' | ', desc, ' | |'))
              }
              i <- i + 1L
            } else if (grepl('^not ok ', line)) {
              desc <- sub('^not ok [0-9]+ - ', '', line)
              msg_lines <- character(0)
              i <- i + 1L
              if (i <= length(tap_lines) && trimws(tap_lines[i]) == '---') {
                i <- i + 1L
                while (i <= length(tap_lines) && trimws(tap_lines[i]) != '...') {
                  msg_lines <- c(msg_lines, tap_lines[i])
                  i <- i + 1L
                }
                i <- i + 1L
              }
              msg_idx <- grep('^[[:space:]]*message:', msg_lines)
              detail <- if (length(msg_idx) > 0) {
                raw <- sub('^[[:space:]]*message:[[:space:]]*', '', msg_lines[msg_idx[1]])
                raw <- sub('^[|>][-+]?[0-9]?[[:space:]]*$', '', raw)
                msg_indent <- nchar(regmatches(msg_lines[msg_idx[1]],
                                               regexpr('^[[:space:]]*', msg_lines[msg_idx[1]])))
                j <- msg_idx[1] + 1L
                while (j <= length(msg_lines) &&
                       nchar(regmatches(msg_lines[j],
                                        regexpr('^[[:space:]]*', msg_lines[j]))) > msg_indent) {
                  content <- trimws(msg_lines[j])
                  if (nchar(content) > 0) {
                    raw <- if (nchar(raw) == 0) content else paste0(raw, ' ', content)
                  }
                  j <- j + 1L
                }
                raw
              } else ''
              rows <- c(rows, paste0('| ',unsuccess_symbol,' | ', desc, ' | ', detail, ' |'))
            } else {
              i <- i + 1L
            }
          }

          header <- if (score == 100) {
            paste0('> \U1F389 All expectations passing!')
          } else {
            paste0('> ', passing, ' of ', total,
                   ' expectations passing \u2014 ', score, '/100')
          }

          md <- c(
            paste0('## ', toupper('${lab}'), ' autograding results'),
            '',
            header,
            '',
            '| | Expectation | Failure message |',
            '|---|-------------|-----------------|',
            rows
          )
          writeLines(md, '/tmp/summary.md')

          writeLines(
            c(paste0('SCORE=',   score),
              paste0('PASSING=', passing),
              paste0('TOTAL=',   total),
              paste0('STATUS=',  status)),
            '/tmp/test_metrics.txt'
          )
        " || true

        if [ ! -f /tmp/test_metrics.txt ]; then
          echo "::error::Tests failed to run (R error before metrics were written)"
        fi
        if [ -f /tmp/code_error.txt ]; then
          echo "CODE_FAILED=true" >> $GITHUB_ENV
        fi
    # Try to source the student's Rmd and warn if it fails.
    # Does NOT stop grading — formative feedback should show as many
    # test results as possible even when code has errors.
    # The warning is folded into the final grading comment (see Post grading results)
    # so the student gets only one notification rather than a separate comment.

    - name: Post grading results
      if: always()
      uses: actions/github-script@v7
      with:
        script: |
          const fs = require('fs');
          const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;

          // Fold code-error warning into the body so no extra comment/notification is needed.
          let codeErrorWarning = '';
          if (fs.existsSync('/tmp/code_error.txt')) {
            const err = fs.readFileSync('/tmp/code_error.txt', 'utf8').trim();
            codeErrorWarning = `\n\n:exploding_head: **Your code had errors when sourced** — some tests below may fail as a result. Fix these before resubmitting:\`\`\`${err}\`\`\`Check the [workflow logs](${runUrl}) for full details.`;
          }

          // Warn if the student did not push a knitted output file at all.
          let missingOutputWarning = '';
          if (fs.existsSync('/tmp/missing_output.txt')) {
            missingOutputWarning = `\n\n:warning: **No knitted output file found** — your repository does not contain a \`.md\` or \`.html\` file. Make sure to knit your Rmd and commit the output before submitting.`;
          }

          // Warn about image files referenced in the Rmd but missing from the repo.
          let missingImagesWarning = '';
          if (fs.existsSync('/tmp/missing_images.txt')) {
            const imgs = fs.readFileSync('/tmp/missing_images.txt', 'utf8').trim();
            if (imgs) {
              const imgList = imgs.split('\n').map(i => `- \`${i}\``).join('\n');
              missingImagesWarning = `\n\n:framed_picture: **Missing images detected** — the following image files are referenced in your knitted document but were not found in your repository. Make sure to commit and push them:\n${imgList}`;
            }
          }

          let body;
          if (fs.existsSync('/tmp/summary.md')) {
            body = '';
            body += codeErrorWarning;
            body += missingOutputWarning;
            body += fs.readFileSync('/tmp/summary.md', 'utf8').trim();
            body += missingImagesWarning;
            body += `\n\n[View full workflow run](${runUrl})`;
          } else {
            body = `:x: Grading failed to run — check the [workflow logs](${runUrl}) for R errors.`;
          }

          // Update the acknowledgment comment rather than posting a new one,
          // so the student receives only one notification per submission.
          let commentId = null;
          if (fs.existsSync('/tmp/ack_comment_id.txt')) {
            commentId = parseInt(fs.readFileSync('/tmp/ack_comment_id.txt', 'utf8').trim(), 10);
          }

          if (commentId) {
            await github.rest.issues.updateComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: commentId,
              body: body
            });
          } else {
            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              body: body
            });
          }