Module: Raif::Evals::EvalSets::Expectations

Included in:
Raif::Evals::EvalSet
Defined in:
lib/raif/evals/eval_sets/expectations.rb

Instance Method Summary collapse

Instance Method Details

#expect(description, result_metadata: nil, &block) ⇒ Object 



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/raif/evals/eval_sets/expectations.rb', line 8

def expect(description, result_metadata: nil, &block)
  result = begin
    if block.call
      output.puts Raif::Utils::Colors.green("  ✓ #{description}")
      output.puts Raif::Utils::Colors.green("    ⎿ #{result_metadata.inspect}") if result_metadata && Raif.config.evals_verbose_output
      ExpectationResult.new(description: description, status: :passed, metadata: result_metadata)
    else
      output.puts Raif::Utils::Colors.red("  ✗ #{description}")
      output.puts Raif::Utils::Colors.red("    ⎿ #{result_metadata.inspect}") if result_metadata && Raif.config.evals_verbose_output
      ExpectationResult.new(description: description, status: :failed, metadata: result_metadata)
    end
  rescue => e
    output.puts Raif::Utils::Colors.red("  ✗ #{description} (Error: #{e.message})")
    ExpectationResult.new(description: description, status: :error, error: e, metadata: result_metadata)
  end

  current_eval_result.add_expectation_result(result)
  result
end

#expect_no_tool_invocation(tool_invoker, tool_name) ⇒ Object 



65
66
67
68
69
 
# File 'lib/raif/evals/eval_sets/expectations.rb', line 65

def expect_no_tool_invocation(tool_invoker, tool_name)
  expect "does not invoke #{tool_name}" do
    tool_invoker.raif_model_tool_invocations.none? { |inv| inv.tool_name == tool_name }
  end
end

#expect_tool_invocation(tool_invoker, tool_type, with: {}) ⇒ Object 



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/raif/evals/eval_sets/expectations.rb', line 49

def expect_tool_invocation(tool_invoker, tool_type, with: {})
  invocations = tool_invoker.raif_model_tool_invocations.select { |inv| inv.tool_type == tool_type }
  invoked_tools = tool_invoker.raif_model_tool_invocations.map{|inv| [inv.tool_type, inv.tool_arguments] }.to_h

  if with.any?
    invocations = invocations.select do |invocation|
      with.all? { |key, value| invocation.tool_arguments[key.to_s] == value }
    end
  end

  result_metadata = { invoked_tools: invoked_tools }
  expect "invokes #{tool_type}#{with.any? ? " with #{with.to_json}" : ""}", result_metadata: result_metadata do
    invocations.any?
  end
end

#score(name, value, scale: nil, min: nil, max: nil, higher_is_better: true) ⇒ Object

Records a number on the eval result, for the quality differences a pass/fail expectation cannot see - once two models clear every bar, their results are identical. Passing min: and/or max: also gates the eval on the value, so a score can replace an expect block rather than sitting alongside one.



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/raif/evals/eval_sets/expectations.rb', line 32

def score(name, value, scale: nil, min: nil, max: nil, higher_is_better: true)
  score_result = ScoreResult.new(
    name: name,
    value: value,
    scale: scale,
    min: min,
    max: max,
    higher_is_better: higher_is_better
  )
  current_eval_result.add_score(score_result)

  output.puts "    #{score_result.name}: #{score_result.formatted_value}"
  expect(score_result.gate_description) { score_result.passed? } if score_result.gated?

  score_result
end