Class: Raif::PromptStudioBatchRunItem

Inherits:
ApplicationRecord show all
Includes:
ActionView::RecordIdentifier
Defined in:
app/models/raif/prompt_studio_batch_run_item.rb

Constant Summary collapse

STATUSES = 
%w[pending running judging completed failed].freeze

Instance Method Summary collapse

Methods inherited from ApplicationRecord

table_name_prefix, where_json_not_blank

Instance Method Details

#comparative_winner_label(winner_letter) ⇒ Object 



109
110
111
112
113
114
115
116
117
118
 
# File 'app/models/raif/prompt_studio_batch_run_item.rb', line 109

def comparative_winner_label(winner_letter)
  new_response_letter = metadata&.dig("new_response_letter")
  return winner_letter unless new_response_letter

  if winner_letter == new_response_letter
    I18n.t("raif.admin.prompt_studio.batch_runs.judge.new_response")
  else
    I18n.t("raif.admin.prompt_studio.batch_runs.judge.original_response")
  end
end

#execute! ⇒ Object 



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'app/models/raif/prompt_studio_batch_run_item.rb', line 58

def execute!
  update!(status: "running")
  broadcast_item

  new_task = create_and_run_task
  run_judge_if_configured(new_task)

  update!(status: "completed")
rescue StandardError => e
  Rails.logger.error "Error running batch run item ##{id}: #{e.message}"
  Rails.logger.error e.backtrace&.join("\n")

  update!(status: "failed")
ensure
  broadcast_item
  batch_run.check_completion!
  broadcast_progress
end

#judge_reasoning ⇒ Object 



100
101
102
103
104
105
106
107
 
# File 'app/models/raif/prompt_studio_batch_run_item.rb', line 100

def judge_reasoning
  return unless judge_task&.completed?

  parsed = judge_task.parsed_response
  return unless parsed.is_a?(Hash)

  parsed["reasoning"]
end

#judge_summary ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'app/models/raif/prompt_studio_batch_run_item.rb', line 77

def judge_summary
  return unless judge_task&.completed?

  parsed = judge_task.parsed_response
  return unless parsed.is_a?(Hash)

  case batch_run.judge_type
  when "Raif::Evals::LlmJudges::Binary"
    parsed["passes"] ? "PASS" : "FAIL"
  when "Raif::Evals::LlmJudges::Scored"
    "Score: #{parsed["score"]}"
  when "Raif::Evals::LlmJudges::Comparative"
    if parsed["winner"] == "tie"
      I18n.t("raif.admin.prompt_studio.batch_runs.judge.tie")
    else
      winner_label = comparative_winner_label(parsed["winner"])
      I18n.t("raif.admin.prompt_studio.batch_runs.judge.winner", name: winner_label)
    end
  when "Raif::Evals::LlmJudges::Summarization"
    "Overall: #{parsed.dig("overall", "score")}/5"
  end
end