Rubric AI feedback delivery modes

app/controllers/course/admin/assessment_settings_controller.rb

@@ -81,6 +81,7 @@ def category_params
       # :allow_randomization,
       :allow_mrq_options_randomization,
       :programming_max_time_limit,
+      :rubric_grading_feedback_workflow,
       :rubric_grading_prompt_enabled,
       :rubric_grading_prompt,
       assessment_categories_attributes: [

app/jobs/course/rubric/apply_evaluations_job.rb

@@ -23,7 +23,10 @@ def apply(rubric, answer)
     grading = Course::Rubric::GradingEvaluationMirrorService.mirror(answer, evaluation)
     grade = Course::Rubric::GradingEvaluationMirrorService.total_grade(grading, answer.question.maximum_grade)
     answer.update_column(:grade, grade)
-    Course::Assessment::Answer::AiGeneratedPostService.new(answer, grading.feedback).create_ai_generated_draft_post
+    # force_draft: applying from the playground is a staff action over a whole class of answers, so it
+    # drafts even where the course publishes student-triggered feedback automatically.
+    Course::Assessment::Answer::AiGeneratedPostService.
+      new(answer, grading.feedback, force_draft: true).create_ai_generated_draft_post
   end
 
   # Runs the LLM once for an answer with no (visible) playground evaluation for this rubric, populating a

app/models/concerns/course/assessment/questions_concern.rb

@@ -75,7 +75,13 @@ def correctly_answered_questions(submission)
     where(id: correctly_answered_question_ids(submission))
   end
 
+  # A rubric-graded question counts as answered once the student has submitted it at least once: rubric grading
+  # has no notion of a wrong answer, and a student should not be held on a step by how long grading takes, or by
+  # whether it succeeds at all.
   def correctly_answered_question_ids(submission)
-    submission.answers.where(correct: true).select(:question_id)
+    submitted_rubric_answers = submission.answers.where(question_id: where(grading_mode: :rubric).select(:id)).
+                               where.not(workflow_state: :attempting)
+
+    submission.answers.where(correct: true).or(submitted_rubric_answers).select(:question_id)
   end
 end

app/models/course.rb

@@ -40,6 +40,7 @@ class Course < ApplicationRecord # rubocop:disable Metrics/ClassLength
   validates :conditional_satisfiability_evaluation_time, presence: true
   validates :ssid_folder_id, uniqueness: { if: :ssid_folder_id_changed? }, allow_nil: true
   validate :validate_rubric_grading_model
+  validate :validate_rubric_grading_feedback_workflow
   validate :validate_rubric_grading_model_options
   validate :validate_rubric_grading_system_prompt
 
@@ -290,6 +291,18 @@ def rubric_grading_prompt=(prompt)
     settings(:course_assessments_component).rubric_grading_prompt = prompt.presence
   end
 
+  # How generated rubric feedback reaches students -- drafted for staff approval (the default), published
+  # to the student directly, or not generated at all. See
+  # Course::Assessment::Answer::AiGeneratedPostService::FEEDBACK_WORKFLOWS for the modes.
+  def rubric_grading_feedback_workflow
+    settings(:course_assessments_component).rubric_grading_feedback_workflow.presence ||
+      Course::Assessment::Answer::AiGeneratedPostService::DEFAULT_FEEDBACK_WORKFLOW
+  end
+
+  def rubric_grading_feedback_workflow=(workflow)
+    settings(:course_assessments_component).rubric_grading_feedback_workflow = workflow.presence
+  end
+
   # The model rubric grading runs on, blank meaning the LlmAdapter's DEFAULT_MODEL. Admin-only (see
   # Course::AssessmentsAbilityComponent), as are the two settings below.
   def rubric_grading_model
@@ -481,6 +494,14 @@ def nearest_forum_discussions(query_embedding, limit: 3)
 
   private
 
+  def validate_rubric_grading_feedback_workflow
+    workflow = settings(:course_assessments_component).rubric_grading_feedback_workflow
+    return if workflow.blank?
+    return if Course::Assessment::Answer::AiGeneratedPostService::FEEDBACK_WORKFLOWS.include?(workflow)
+
+    errors.add(:rubric_grading_feedback_workflow, "is not a supported feedback workflow: #{workflow}")
+  end
+
   def validate_rubric_grading_model
     return if rubric_grading_model.blank?
     return if Course::Rubric::LlmService::LlmAdapter.available_models.include?(rubric_grading_model)

app/models/course/assessment/submission.rb

@@ -22,6 +22,7 @@ class Course::Assessment::Submission < ApplicationRecord
 
   after_save :auto_grade_submission, if: :submitted?
   after_save :retrieve_codaveri_feedback, if: :submitted?
+  after_save :publish_ai_generated_feedback, if: :submitted?
   after_create :schedule_force_submission, if: :attempting?
 
   workflow do
@@ -447,6 +448,31 @@ def auto_grade_submission
     end
   end
 
+  # Releases the AI rubric feedback drafted while this submission was being attempted, for courses that hold
+  # it back until the student finalises (see AiGeneratedPostService::PUBLISH_ON_FINALISE). Comments generated
+  # *by* finalising are published as they are created -- this callback runs before those grading jobs, so it
+  # only ever sees the earlier drafts.
+  def publish_ai_generated_feedback
+    return unless saved_change_to_workflow_state?
+    return unless assessment.course.rubric_grading_feedback_workflow ==
+                  Course::Assessment::Answer::AiGeneratedPostService::PUBLISH_ON_FINALISE
+
+    ActiveRecord.after_all_transactions_commit do
+      drafts = Course::Discussion::Post.where(topic: submission_questions.map(&:discussion_topic)).
+               where(is_ai_generated: true, workflow_state: 'draft').to_a
+      # One at a time with update! -- not the workflow event, and not update_all -- so each post's own callbacks
+      # fire, which snapshot the final text into the feedback rating.
+      drafts.each { |post| post.update!(workflow_state: 'published') }
+
+      # Nobody is waiting to approve them any more, so their topics leave the staff pending queues. Topic has no
+      # save callbacks, so one statement is equivalent to unmark_as_pending on each -- including the updated_at
+      # bump, which read tracking keys off (acts_as_readable on: :updated_at). Only these drafts' topics: one
+      # whose comment staff accepted earlier may be pending again because the student has since replied.
+      Course::Discussion::Topic.where(id: drafts.map(&:topic_id), pending_staff_reply: true).
+        update_all(pending_staff_reply: false, updated_at: Time.current)
+    end
+  end
+
   # Retrieve codaveri feedback only for current answers of codaveri programming question type
   # for finalised submissions.
   def retrieve_codaveri_feedback

app/services/course/assessment/answer/ai_generated_post_service.rb

@@ -1,16 +1,41 @@
 # frozen_string_literal: true
 
 class Course::Assessment::Answer::AiGeneratedPostService
+  # How a course delivers generated rubric feedback to its students (Course#rubric_grading_feedback_workflow,
+  # configured under Course Settings > Assessments). Mirrors the Codaveri component's feedback_workflow, with
+  # the publish case split in two: rubric answers are graded both when a student submits a single answer and
+  # when they finalise the submission.
+  #
+  #   * +none+                     -- grade the answer, but draft no feedback comment at all.
+  #   * +draft+                    -- staff accept or reject the comment before the student sees it (default).
+  #   * +publish_on_answer_submit+ -- the student sees the comment as soon as the answer is graded.
+  #   * +publish_on_finalise+      -- drafted while the submission is being attempted, published (see
+  #                                   Course::Assessment::Submission#publish_ai_generated_feedback) once it
+  #                                   is finalised.
+  NO_FEEDBACK = 'none'
+  DRAFT_FEEDBACK = 'draft'
+  PUBLISH_ON_ANSWER_SUBMIT = 'publish_on_answer_submit'
+  PUBLISH_ON_FINALISE = 'publish_on_finalise'
+  FEEDBACK_WORKFLOWS = [NO_FEEDBACK, DRAFT_FEEDBACK, PUBLISH_ON_ANSWER_SUBMIT, PUBLISH_ON_FINALISE].freeze
+  DEFAULT_FEEDBACK_WORKFLOW = DRAFT_FEEDBACK
+
   # @param [Course::Assessment::Answer] answer The answer to create/update the post for
   # @param [String] feedback The feedback text to include in the post
-  def initialize(answer, content)
+  # @param [Boolean] force_draft Whether to draft the comment regardless of the course's workflow. Set by
+  #   staff-initiated generation (applying playground evaluations), which may publish to a whole class at
+  #   once and so is never delivered to students without a person deciding to.
+  def initialize(answer, content, force_draft: false)
     @answer = answer
     @content = content
+    @force_draft = force_draft
   end
 
-  # Creates or updates AI-generated draft feedback post for the answer
+  # Creates or updates the AI-generated feedback post for the answer, drafted or published according to the
+  # course's feedback workflow.
   # @return [void]
   def create_ai_generated_draft_post
+    return if feedback_workflow == NO_FEEDBACK
+
     submission_question = @answer.submission.submission_questions.find_by(question_id: @answer.question_id)
     return unless submission_question
 
@@ -26,6 +51,26 @@ def create_ai_generated_draft_post
 
   private
 
+  # @return [String] one of FEEDBACK_WORKFLOWS
+  def feedback_workflow
+    return DRAFT_FEEDBACK if @force_draft
+
+    @feedback_workflow ||= @answer.submission.assessment.course.rubric_grading_feedback_workflow
+  end
+
+  # Whether this comment reaches the student without a staff decision. +publish_on_finalise+ waits: a
+  # comment drafted mid-attempt stays a draft, and the submission publishes it when it is finalised, while
+  # a comment generated after that point (finalising triggers grading of the remaining answers) is published
+  # straight away.
+  # @return [Boolean]
+  def publish_immediately?
+    case feedback_workflow
+    when PUBLISH_ON_ANSWER_SUBMIT then true
+    when PUBLISH_ON_FINALISE then !@answer.submission.attempting?
+    else false
+    end
+  end
+
   # Builds a draft post with AI-generated feedback
   # @param [Course::Assessment::SubmissionQuestion] submission_question The submission question
   # @return [Course::Discussion::Post] The built post
@@ -35,7 +80,7 @@ def build_draft_post(submission_question)
       updater: User.system,
       text: @content,
       is_ai_generated: true,
-      workflow_state: 'draft',
+      workflow_state: publish_immediately? ? 'published' : 'draft',
       title: @answer.submission.assessment.title
     )
   end
@@ -51,9 +96,12 @@ def save_draft_post(submission_question, post)
       end
       post.save!
       submission_question.save!
+      # The rating is initialized whichever way the comment is delivered, so the generated text is always
+      # snapshotted. A published comment carries no rating UI today (the card is draft-only), so its rating
+      # stays unrated -- the same trade the Codaveri publish workflow makes.
       initialize_rating(post)
       create_topic_subscription(post.topic)
-      post.topic.mark_as_pending
+      post.topic.mark_as_pending unless post.published?
     end
   end
 
@@ -128,7 +176,9 @@ def create_topic_subscription(discussion_topic)
     end
   end
 
-  # Finds the latest AI-generated draft post for the submission question
+  # Finds the latest AI-generated draft post for the submission question. Only drafts are updated in place,
+  # so under a publish workflow a re-graded answer gains a new comment instead of having one the student may
+  # already have read rewritten underneath them.
   # @param [Course::Assessment::SubmissionQuestion] submission_question The submission question
   # @return [Course::Discussion::Post, nil] The latest AI-generated draft post or nil if none exists
   def find_existing_ai_draft_post(submission_question)

app/views/course/admin/assessment_settings/edit.json.jbuilder

@@ -6,6 +6,9 @@ json.showStdoutAndStderr current_course.show_stdout_and_stderr || false
 json.allowRandomization current_course.allow_randomization || false
 json.allowMrqOptionsRandomization current_course.allow_mrq_options_randomization || false
 json.maxProgrammingTimeLimit current_course.programming_max_time_limit if can?(:manage, :all)
+# How generated rubric feedback reaches students. Unlike the model configuration below, this is a teaching
+# decision, so any course manager may change it.
+json.rubricGradingFeedbackWorkflow current_course.rubric_grading_feedback_workflow
 json.rubricGradingPromptEnabled current_course.rubric_grading_prompt_enabled
 json.rubricGradingPrompt current_course.rubric_grading_prompt || ''
 

app/views/course/assessment/answer/_ai_generated_comment.json.jbuilder

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+# The latest AI feedback comment on this answer's question, so the client can show it as soon as grading
+# finishes rather than on the next full page load. Which state that is depends on the course's feedback
+# workflow: a draft for staff to review, or already published to the student. Visibility follows the
+# submission's own comment rules (see _topics.json.jbuilder): staff see every state, students only what has
+# been published.
+posts = answer.submission.submission_questions.find_by(question_id: answer.question_id)&.discussion_topic&.posts
+ai_generated_comment = posts&.select do |post|
+  post.is_ai_generated && (can_grade || post.published?)
+end&.last
+if ai_generated_comment
+  json.aiGeneratedComment do
+    json.partial! ai_generated_comment
+  end
+end

app/views/course/assessment/answer/_rubric_explanation.json.jbuilder

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+# The step-by-step gate for a rubric-graded answer: the Continue and Finalise buttons read +correct+, as does the
+# server's maxStep (see Course::Assessment::QuestionsConcern#correctly_answered_question_ids). Rubric grading has
+# no notion of a wrong answer, so +correct+ means "submitted at least once" -- the student may move on as soon as
+# they have submitted, however long grading takes or whether it succeeds. Null until then, which hides the panel.
+#
+# Unlike other answer types this is not gated on can_read_grade?: it carries no grade, only whether the student
+# has submitted, which they already know.
+submitted = last_attempt.present? && !last_attempt.attempting?
+json.explanation do
+  json.correct submitted ? true : nil
+  json.explanations []
+end

app/views/course/assessment/answer/forum_post_responses/_forum_post_response.json.jbuilder

@@ -19,9 +19,9 @@ last_attempt = last_attempt(answer)
 # the plain grade field, so emitting an (empty) breakdown would misroute their save through the rubric path.
 if answer.question.grading_mode_rubric?
   json.partial! 'course/assessment/answer/rubric_category_grades', answer: answer, can_grade: can_grade
-end
-
-if answer.can_read_grade?(current_ability)
+  json.partial! 'course/assessment/answer/ai_generated_comment', answer: answer, can_grade: can_grade
+  json.partial! 'course/assessment/answer/rubric_explanation', last_attempt: last_attempt
+elsif answer.can_read_grade?(current_ability)
   json.explanation do
     json.correct last_attempt&.correct
     json.explanations []

app/views/course/assessment/answer/rubric_based_responses/_rubric_based_response.json.jbuilder

@@ -32,24 +32,8 @@ end
 
 json.partial! 'course/assessment/answer/rubric_category_grades', answer: answer, can_grade: can_grade
 
-if can_grade
-  posts = answer.submission.submission_questions.find_by(question_id: answer.question_id)&.discussion_topic&.posts
-  ai_generated_comment = posts&.select do |post|
-    post.is_ai_generated && post.workflow_state == 'draft'
-  end&.last
-  if ai_generated_comment
-    json.aiGeneratedComment do
-      json.partial! ai_generated_comment
-    end
-  end
-end
-
-if answer.can_read_grade?(current_ability)
-  json.explanation do
-    json.correct last_attempt&.correct
-    json.explanations []
-  end
-end
+json.partial! 'course/assessment/answer/ai_generated_comment', answer: answer, can_grade: can_grade
+json.partial! 'course/assessment/answer/rubric_explanation', last_attempt: last_attempt
 
 if answer.current_answer? && !last_attempt.current_answer?
   json.latestAnswer do

app/views/course/assessment/question/rubric_based_responses/_rubric_based_response.json.jbuilder

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 json.aiGradingEnabled question.ai_grading_enabled? if can_grade
 
-json.autogradable false
+json.autogradable question.auto_gradable?
 json.templateText question.template_text
 
 json.partial! 'course/assessment/question/rubric_categories',

app/views/course/assessment/submission/submissions/_question.json.jbuilder

@@ -12,5 +12,7 @@ end
 
 json.canViewHistory question.history_viewable?
 json.type question.question_type
+# 'rubric' questions have no notion of a wrong answer, which the client needs to word the answer panel.
+json.gradingMode question.grading_mode
 
 json.partial! question, question: question.specific, can_grade: can_grade, answer: answer

app/views/course/discussion/posts/_post.json.jbuilder

@@ -36,7 +36,10 @@ end
 # Lazily backfills the rating for pre-feature (or straggler) AI drafts so they render with the rateable card.
 post.ensure_generated_rating! if post.is_ai_generated && post.workflow_state == 'draft'
 ai_feedback_rating = post.is_ai_generated ? post.ai_feedback_rating : nil
-if ai_feedback_rating
+# Staff-only. The rating holds the AI's original text (which staff may have edited before publishing), the edited
+# text, and the staff member's score -- none of which is for the student the comment is addressed to. Gated on the
+# ability the rating endpoint authorizes, so exactly the users who can rate the comment receive it.
+if ai_feedback_rating && can?(:update, ai_feedback_rating)
   json.generatedRating do
     # `type` discriminates which rating endpoint the client should call (see RateableGeneratedCommentCard).
     json.type 'rubric_feedback'

app/views/course/forum/posts/_post_list_data.json.jbuilder

@@ -17,7 +17,10 @@ json.isAiGenerated post.is_ai_generated
 # Lazily backfills the rating for pre-feature (or straggler) AI drafts so they render with the rateable card.
 post.ensure_generated_rating! if post.is_ai_generated && post.workflow_state == 'draft'
 rag_wise_rating = post.is_ai_generated ? post.rag_wise_rating : nil
-if rag_wise_rating
+# Staff-only. The rating holds the AI's original answer (which staff may have edited before publishing), the edited
+# text, the staff member's score and the answer's quality scores -- none of which is for the students reading the
+# forum. Gated on the ability the rating endpoint authorizes, so exactly the users who can rate it receive it.
+if rag_wise_rating && can?(:update, rag_wise_rating)
   json.generatedRating do
     # `type` discriminates which rating endpoint the client should call (see RateableGeneratedCommentCard).
     json.type 'rag_wise'