Class: Raif::Evals::LlmJudges::Scored

Inherits:
Raif::Evals::LlmJudge show all
Defined in:
app/models/raif/evals/llm_judges/scored.rb

Constant Summary

Constants included from Concerns::LlmResponseParsing

Concerns::LlmResponseParsing::ASCII_CONTROL_CHARS

Instance Attribute Summary

Attributes inherited from Task

#files, #images

Instance Method Summary collapse

Methods inherited from Raif::Evals::LlmJudge

#default_llm_model_key, #judgment_confidence, #judgment_reasoning, #low_confidence?

Methods inherited from Task

json_response_schema, #json_response_schema, #messages, prompt, #prompt_studio_task_attributes, #re_run, run, #run, #status, system_prompt

Methods included from Concerns::JsonSchemaDefinition

#schema_for_instance

Methods included from Concerns::LlmResponseParsing

#parse_html_response, #parse_json_response, #parsed_response

Methods included from Concerns::HasRuntimeDuration

#runtime_duration, #runtime_duration_seconds, #runtime_ended_at

Methods included from Concerns::HasAvailableModelTools

#available_model_tools_map

Methods included from Concerns::HasRequestedLanguage

#requested_language_name, #system_prompt_language_preference

Methods included from Concerns::HasLlm

#default_llm_model_key, #llm

Methods inherited from ApplicationRecord

table_name_prefix, where_json_not_blank

Instance Method Details

#build_prompt ⇒ Object 



68
69
70
71
72
73
74
75
76
77
78
 
# File 'app/models/raif/evals/llm_judges/scored.rb', line 68

def build_prompt
  <<~PROMPT.strip
    Scoring rubric:
    #{format_rubric(scoring_rubric)}
    #{additional_context_prompt}
    Evaluate the following content according to the scoring rubric:
    #{content_to_judge}

    Provide your score and detailed reasoning.
  PROMPT
end

#build_system_prompt ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'app/models/raif/evals/llm_judges/scored.rb', line 53

def build_system_prompt
  <<~PROMPT.strip
    You are an expert evaluator providing numerical scores based on a detailed rubric.

    First, provide detailed reasoning/explanation of your evaluation. Then, provide a precise score according to the provided rubric.

    Respond with JSON matching this schema:
    {
      "score": number,
      "reasoning": "detailed explanation",
      "confidence": 0.0-1.0
    }
  PROMPT
end

#judgment_score ⇒ Object 



80
81
82
 
# File 'app/models/raif/evals/llm_judges/scored.rb', line 80

def judgment_score
  parsed_response["score"] if completed?
end