Class: Raif::Evals::EvalSet

Inherits:
Object
  • Object
show all
Includes:
Raif::Evals::EvalSets::Expectations, Raif::Evals::EvalSets::LlmJudgeExpectations
Defined in:
lib/raif/evals/eval_set.rb

Class Attribute Summary collapse

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Raif::Evals::EvalSets::LlmJudgeExpectations

#expect_llm_judge_passes, #expect_llm_judge_prefers, #expect_llm_judge_score

Methods included from Raif::Evals::EvalSets::Expectations

#expect, #expect_no_tool_invocation, #expect_tool_invocation

Constructor Details

#initialize(output: $stdout) ⇒ EvalSet

Returns a new instance of EvalSet.



14
15
16
 
# File 'lib/raif/evals/eval_set.rb', line 14

def initialize(output: $stdout)
  @output = output
end

Class Attribute Details

.setup_block ⇒ Object (readonly)

Returns the value of attribute setup_block.



19
20
21
 
# File 'lib/raif/evals/eval_set.rb', line 19

def setup_block
  @setup_block
end

.teardown_block ⇒ Object (readonly)

Returns the value of attribute teardown_block.



20
21
22
 
# File 'lib/raif/evals/eval_set.rb', line 20

def teardown_block
  @teardown_block
end

Instance Attribute Details

#current_eval ⇒ Object (readonly)

Returns the value of attribute current_eval.



12
13
14
 
# File 'lib/raif/evals/eval_set.rb', line 12

def current_eval
  @current_eval
end

#output ⇒ Object (readonly)

Returns the value of attribute output.



12
13
14
 
# File 'lib/raif/evals/eval_set.rb', line 12

def output
  @output
end

#results ⇒ Object (readonly)

Returns the value of attribute results.



12
13
14
 
# File 'lib/raif/evals/eval_set.rb', line 12

def results
  @results
end

Class Method Details

.eval(description, &block) ⇒ Object 



31
32
33
 
# File 'lib/raif/evals/eval_set.rb', line 31

def eval(description, &block)
  evals << { description: description, block: block, definition_line_number: caller_locations(1, 1).first.lineno }
end

.evals ⇒ Object 



27
28
29
 
# File 'lib/raif/evals/eval_set.rb', line 27

def evals
  @evals ||= []
end

.inherited(subclass) ⇒ Object 



22
23
24
25
 
# File 'lib/raif/evals/eval_set.rb', line 22

def inherited(subclass)
  subclass.instance_variable_set(:@evals, [])
  super
end

.run(output: $stdout, repeats: 1) ⇒ Object 



43
44
45
 
# File 'lib/raif/evals/eval_set.rb', line 43

def run(output: $stdout, repeats: 1)
  new(output: output).run(repeats: repeats)
end

.setup(&block) ⇒ Object 



35
36
37
 
# File 'lib/raif/evals/eval_set.rb', line 35

def setup(&block)
  @setup_block = block
end

.teardown(&block) ⇒ Object 



39
40
41
 
# File 'lib/raif/evals/eval_set.rb', line 39

def teardown(&block)
  @teardown_block = block
end

Instance Method Details

#file(filename) ⇒ Object

Raises:

  • (ArgumentError) 


108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/raif/evals/eval_set.rb', line 108

def file(filename)
  # Validate filename to prevent directory traversal
  raise ArgumentError, "Invalid filename: cannot be empty" if filename.nil? || filename.empty?
  raise ArgumentError, "Invalid filename: cannot contain '..' or absolute paths" if filename.include?("..") || filename.start_with?("/")

  # Ensure we're only accessing files within the raif_evals/files directory
  base_path = Rails.root.join("raif_evals", "files")
  full_path = base_path.join(filename)

  # Verify the resolved path is within the expected directory
  unless full_path.to_s.start_with?(base_path.to_s)
    raise ArgumentError, "Invalid filename: path traversal detected"
  end

  if full_path.exist?
    full_path.read
  else
    raise ArgumentError, "File #{filename} does not exist in raif_evals/files/"
  end
end

#run(repeats: 1) ⇒ Object

Each repeat re-runs setup and the eval block from scratch, so the repeats are independent samples of a non-deterministic model rather than a re-scoring of one response. Comparing models needs the pass rate across them, not a single draw.



51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/raif/evals/eval_set.rb', line 51

def run(repeats: 1)
  @results = []

  self.class.evals.each_with_index do |eval_definition, eval_index|
    repeats.times do |i|
      @results << run_eval(eval_definition, run_index: (i + 1 if repeats > 1), eval_index: eval_index)
    end
  end

  @results
end

#run_eval(eval_definition, run_index: nil, eval_index: nil) ⇒ Object 



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/raif/evals/eval_set.rb', line 63

def run_eval(eval_definition, run_index: nil, eval_index: nil)
  @current_eval = Eval.new(
    description: eval_definition[:description],
    run_index: run_index,
    eval_index: eval_index || self.class.evals.index(eval_definition)
  )

  output.puts "Running: #{eval_definition[:description]}#{" (run #{run_index})" if run_index}"

  ActiveRecord::Base.transaction do
    instance_eval(&self.class.setup_block) if self.class.setup_block

    # Baseline id captured after setup so we only record the LLM calls the eval
    # itself makes. Model completions are persisted during the eval but rolled back
    # below, so they must be captured before the rollback.
    model_completions_start_id = Raif::ModelCompletion.maximum(:id) || 0

    begin
      instance_eval(&eval_definition[:block])
    rescue => e
      output.puts Raif::Utils::Colors.red("  Error in eval block: #{e.message}")
      output.puts Raif::Utils::Colors.red("  #{e.backtrace.join("\n  ")}")
      @current_eval.add_expectation_result(
        ExpectationResult.new(
          description: "Eval block execution",
          status: :error,
          error: e
        )
      )
    ensure
      # Capture completions before teardown: sources like Raif::Agent and
      # Raif::ConversationEntry declare `has_many :raif_model_completions,
      # dependent: :destroy`, so a teardown that destroys them would delete the
      # rows we need before we can read them.
      capture_model_completions(model_completions_start_id)

      instance_eval(&self.class.teardown_block) if self.class.teardown_block
    end

    raise ActiveRecord::Rollback
  end

  @current_eval
end