Class: Raif::Evals::LlmJudges::Scored

Inherits:
Raif::Evals::LlmJudge show all
Defined in:
lib/raif/evals/llm_judges/scored.rb

Constant Summary

Constants included from Concerns::LlmResponseParsing

Concerns::LlmResponseParsing::ASCII_CONTROL_CHARS

Instance Attribute Summary

Attributes inherited from Task

#files, #images

Instance Method Summary collapse

Methods inherited from Raif::Evals::LlmJudge

#default_llm_model_key, #judgment_confidence, #judgment_reasoning, #low_confidence?

Methods inherited from Task

json_response_schema, prompt, #re_run, run, #run, #status, system_prompt

Methods included from Concerns::LlmResponseParsing

#parse_html_response, #parse_json_response, #parsed_response

Methods included from Concerns::HasAvailableModelTools

#available_model_tools_map

Methods included from Concerns::HasRequestedLanguage

#requested_language_name, #system_prompt_language_preference

Methods included from Concerns::HasLlm

#default_llm_model_key, #llm

Methods inherited from ApplicationRecord

table_name_prefix

Instance Method Details

#build_prompt ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/raif/evals/llm_judges/scored.rb', line 30

def build_prompt
  <<~PROMPT.strip
    Scoring rubric:
    #{format_rubric(scoring_rubric)}
    #{additional_context_prompt}
    Evaluate the following content according to the scoring rubric:
    #{content_to_judge}

    Provide your score and detailed reasoning.
  PROMPT
end

#build_system_prompt ⇒ Object 



15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/raif/evals/llm_judges/scored.rb', line 15

def build_system_prompt
  <<~PROMPT.strip
    You are an expert evaluator providing numerical scores based on a detailed rubric.

    First, provide detailed reasoning/explanation of your evaluation. Then, provide a precise score according to the provided rubric.

    Respond with JSON matching this schema:
    {
      "score": number,
      "reasoning": "detailed explanation",
      "confidence": 0.0-1.0
    }
  PROMPT
end

#judgment_score ⇒ Object 



42
43
44
 
# File 'lib/raif/evals/llm_judges/scored.rb', line 42

def judgment_score
  parsed_response["score"] if completed?
end