Class: Raif::Evals::EvalSet

Inherits:
Object
  • Object
show all
Includes:
Raif::Evals::EvalSets::Expectations, Raif::Evals::EvalSets::LlmJudgeExpectations
Defined in:
lib/raif/evals/eval_set.rb

Class Attribute Summary collapse

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Raif::Evals::EvalSets::LlmJudgeExpectations

#expect_llm_judge_passes, #expect_llm_judge_prefers, #expect_llm_judge_score

Methods included from Raif::Evals::EvalSets::Expectations

#expect, #expect_no_tool_invocation, #expect_tool_invocation

Constructor Details

#initialize(output: $stdout) ⇒ EvalSet

Returns a new instance of EvalSet.



14
15
16
 
# File 'lib/raif/evals/eval_set.rb', line 14

def initialize(output: $stdout)
  @output = output
end

Class Attribute Details

.setup_block ⇒ Object (readonly)

Returns the value of attribute setup_block.



19
20
21
 
# File 'lib/raif/evals/eval_set.rb', line 19

def setup_block
  @setup_block
end

.teardown_block ⇒ Object (readonly)

Returns the value of attribute teardown_block.



20
21
22
 
# File 'lib/raif/evals/eval_set.rb', line 20

def teardown_block
  @teardown_block
end

Instance Attribute Details

#current_eval ⇒ Object (readonly)

Returns the value of attribute current_eval.



12
13
14
 
# File 'lib/raif/evals/eval_set.rb', line 12

def current_eval
  @current_eval
end

#output ⇒ Object (readonly)

Returns the value of attribute output.



12
13
14
 
# File 'lib/raif/evals/eval_set.rb', line 12

def output
  @output
end

#results ⇒ Object (readonly)

Returns the value of attribute results.



12
13
14
 
# File 'lib/raif/evals/eval_set.rb', line 12

def results
  @results
end

Class Method Details

.eval(description, &block) ⇒ Object 



31
32
33
 
# File 'lib/raif/evals/eval_set.rb', line 31

def eval(description, &block)
  evals << { description: description, block: block, definition_line_number: caller_locations(1, 1).first.lineno }
end

.evals ⇒ Object 



27
28
29
 
# File 'lib/raif/evals/eval_set.rb', line 27

def evals
  @evals ||= []
end

.inherited(subclass) ⇒ Object 



22
23
24
25
 
# File 'lib/raif/evals/eval_set.rb', line 22

def inherited(subclass)
  subclass.instance_variable_set(:@evals, [])
  super
end

.run(output: $stdout) ⇒ Object 



43
44
45
 
# File 'lib/raif/evals/eval_set.rb', line 43

def run(output: $stdout)
  new(output: output).run
end

.setup(&block) ⇒ Object 



35
36
37
 
# File 'lib/raif/evals/eval_set.rb', line 35

def setup(&block)
  @setup_block = block
end

.teardown(&block) ⇒ Object 



39
40
41
 
# File 'lib/raif/evals/eval_set.rb', line 39

def teardown(&block)
  @teardown_block = block
end

Instance Method Details

#file(filename) ⇒ Object

Raises:

  • (ArgumentError) 


88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/raif/evals/eval_set.rb', line 88

def file(filename)
  # Validate filename to prevent directory traversal
  raise ArgumentError, "Invalid filename: cannot be empty" if filename.nil? || filename.empty?
  raise ArgumentError, "Invalid filename: cannot contain '..' or absolute paths" if filename.include?("..") || filename.start_with?("/")

  # Ensure we're only accessing files within the raif_evals/files directory
  base_path = Rails.root.join("raif_evals", "files")
  full_path = base_path.join(filename)

  # Verify the resolved path is within the expected directory
  unless full_path.to_s.start_with?(base_path.to_s)
    raise ArgumentError, "Invalid filename: path traversal detected"
  end

  if full_path.exist?
    full_path.read
  else
    raise ArgumentError, "File #{filename} does not exist in raif_evals/files/"
  end
end

#run ⇒ Object 



48
49
50
51
52
53
54
55
56
 
# File 'lib/raif/evals/eval_set.rb', line 48

def run
  @results = []

  self.class.evals.each do |eval_definition|
    @results << run_eval(eval_definition)
  end

  @results
end

#run_eval(eval_definition) ⇒ Object 



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/raif/evals/eval_set.rb', line 58

def run_eval(eval_definition)
  @current_eval = Eval.new(description: eval_definition[:description])

  output.puts "Running: #{eval_definition[:description]}"

  ActiveRecord::Base.transaction do
    instance_eval(&self.class.setup_block) if self.class.setup_block

    begin
      instance_eval(&eval_definition[:block])
    rescue => e
      output.puts Raif::Utils::Colors.red("  Error in eval block: #{e.message}")
      output.puts Raif::Utils::Colors.red("  #{e.backtrace.join("\n  ")}")
      @current_eval.add_expectation_result(
        ExpectationResult.new(
          description: "Eval block execution",
          status: :error,
          error: e
        )
      )
    ensure
      instance_eval(&self.class.teardown_block) if self.class.teardown_block
    end

    raise ActiveRecord::Rollback
  end

  @current_eval
end