Class: Raif::Evals::LlmJudge

Inherits:

Task

• Object
• ApplicationRecord
• Task
• Raif::Evals::LlmJudge

Defined in:

lib/raif/evals/llm_judge.rb

Direct Known Subclasses

Raif::Evals::LlmJudges::Binary, Raif::Evals::LlmJudges::Comparative, Raif::Evals::LlmJudges::Scored, Raif::Evals::LlmJudges::Summarization

Constant Summary

Constants included from Concerns::LlmResponseParsing

Concerns::LlmResponseParsing::ASCII_CONTROL_CHARS

Instance Attribute Summary

Attributes inherited from Task

#files, #images

Instance Method Summary

• #default_llm_model_key ⇒ Object
• #judgment_confidence ⇒ Object
• #judgment_reasoning ⇒ Object
• #low_confidence? ⇒ Boolean

Methods inherited from Task

#build_prompt, #build_system_prompt, json_response_schema, prompt, #re_run, run, #run, #status, system_prompt

Methods included from Concerns::LlmResponseParsing

#parse_html_response, #parse_json_response, #parsed_response

Methods included from Concerns::HasAvailableModelTools

#available_model_tools_map

Methods included from Concerns::HasRequestedLanguage

#requested_language_name, #system_prompt_language_preference

Methods included from Concerns::HasLlm

#llm

Methods inherited from ApplicationRecord

table_name_prefix

Instance Method Details

#default_llm_model_key ⇒ Object

# File 'lib/raif/evals/llm_judge.rb', line 15

def default_llm_model_key
  Raif.config.evals_default_llm_judge_model_key || super
end

#judgment_confidence ⇒ Object

# File 'lib/raif/evals/llm_judge.rb', line 23

def judgment_confidence
  parsed_response["confidence"] if completed?
end

#judgment_reasoning ⇒ Object

# File 'lib/raif/evals/llm_judge.rb', line 19

def judgment_reasoning
  parsed_response["reasoning"] if completed?
end

#low_confidence? ⇒ Boolean

Returns:

• (Boolean)

# File 'lib/raif/evals/llm_judge.rb', line 27

def low_confidence?
  judgment_confidence && judgment_confidence < 0.5
end