Class: Raif::Evals::EvalSet

Inherits:
Object
  • Object
show all
Includes:
Raif::Evals::EvalSets::Expectations, Raif::Evals::EvalSets::LlmJudgeExpectations, Raif::Evals::EvalSets::Matchers
Defined in:
lib/raif/evals/eval_set.rb

Overview

A host app's eval sets subclass this: the class body is the DSL (eval, dataset, setup, teardown) and an instance is the context those blocks are evaluated against.

An instance is single-use: #run_eval writes the case and result onto self so #expect and #score can reach them. Deciding what to run and dispatching it is EvalSetCoordinator's.

Constant Summary

Constants included from Raif::Evals::EvalSets::Matchers

Raif::Evals::EvalSets::Matchers::MAX_METADATA_LENGTH

Class Attribute Summary collapse

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Raif::Evals::EvalSets::LlmJudgeExpectations

#expect_llm_judge_passes, #expect_llm_judge_prefers, #expect_llm_judge_score

Methods included from Raif::Evals::EvalSets::Matchers

#expect_exact_match, #expect_includes, #expect_matches, #expect_within

Methods included from Raif::Evals::EvalSets::Expectations

#expect, #expect_no_tool_invocation, #expect_tool_invocation, #score

Constructor Details

#initialize(output: $stdout) ⇒ EvalSet

Returns a new instance of EvalSet.



21
22
23
24
 
# File 'lib/raif/evals/eval_set.rb', line 21

def initialize(output: $stdout)
  @output = output
  @console_output = output
end

Class Attribute Details

.setup_block ⇒ Object (readonly)

Returns the value of attribute setup_block.



27
28
29
 
# File 'lib/raif/evals/eval_set.rb', line 27

def setup_block
  @setup_block
end

.teardown_block ⇒ Object (readonly)

Returns the value of attribute teardown_block.



28
29
30
 
# File 'lib/raif/evals/eval_set.rb', line 28

def teardown_block
  @teardown_block
end

Instance Attribute Details

#current_case ⇒ Object (readonly)

Returns the value of attribute current_case.



19
20
21
 
# File 'lib/raif/evals/eval_set.rb', line 19

def current_case
  @current_case
end

#current_eval_result ⇒ Object (readonly)

Returns the value of attribute current_eval_result.



19
20
21
 
# File 'lib/raif/evals/eval_set.rb', line 19

def current_eval_result
  @current_eval_result
end

#output ⇒ Object (readonly)

Returns the value of attribute output.



19
20
21
 
# File 'lib/raif/evals/eval_set.rb', line 19

def output
  @output
end

Class Method Details

.dataset(name, &block) ⇒ Object

Registers a named collection of eval cases. The block is not run here: it is resolved at run time in instance context so it can use file/files/json/jsonl.

Raises:

  • (ArgumentError) 


46
47
48
49
50
 
# File 'lib/raif/evals/eval_set.rb', line 46

def dataset(name, &block)
  raise ArgumentError, "dataset #{name.inspect} requires a block" unless block

  datasets[name.to_sym] = block
end

.datasets ⇒ Object 



40
41
42
 
# File 'lib/raif/evals/eval_set.rb', line 40

def datasets
  @datasets ||= {}
end

.eval(description, dataset: nil, id: nil, &block) ⇒ Object

This shadows Kernel#eval inside the class body. Without the block check, a block-less call would register an eval that fails only once it runs, after setup has spent money.

Parameters:

  • id (String, Symbol, nil) (defaults to: nil) —

    Overrides the description-derived half of the eval's id - see Raif::Evals::EvalDefinition#id. Pass one when you expect to reword the description without wanting the reworded eval treated as a new one.

Raises:

  • (ArgumentError) 


58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
 
# File 'lib/raif/evals/eval_set.rb', line 58

def eval(description, dataset: nil, id: nil, &block)
  raise ArgumentError, "eval #{description.inspect} requires a block" unless block

  # A registered name and nothing else: an inline array or Proc would be a dataset
  # with no name, and the name is what a case id is reported against.
  if dataset && !dataset.is_a?(Symbol) && !dataset.is_a?(String)
    raise ArgumentError, "eval #{description.inspect} passed #{dataset.class} to dataset:, which takes the name of a " \
      "dataset declared with the `dataset` macro."
  end

  if dataset && !datasets.key?(dataset.to_sym)
    raise ArgumentError, "eval #{description.inspect} names dataset #{dataset.inspect}, which has not been declared. " \
      "Declare it above the evals that use it#{" (declared: #{datasets.keys.inspect})" if datasets.any?}."
  end

  declared_id = validated_eval_id(id, description)
  check_eval_identity!(description, declared_id)

  definition_location = caller_locations(1, 1).first

  # Position assigned here rather than derived later: evals is append-only, so the
  # index a definition registers at is the one it keeps.
  evals << EvalDefinition.new(
    description: description,
    block: block,
    dataset: dataset&.to_sym,
    index: evals.length,
    eval_set_class: self,
    declared_id: declared_id,
    file: definition_location.path,
    line_number: definition_location.lineno
  )
end

.evals ⇒ Object 



36
37
38
 
# File 'lib/raif/evals/eval_set.rb', line 36

def evals
  @evals ||= []
end

.inherited(subclass) ⇒ Object 



30
31
32
33
34
 
# File 'lib/raif/evals/eval_set.rb', line 30

def inherited(subclass)
  subclass.instance_variable_set(:@evals, [])
  subclass.instance_variable_set(:@datasets, {})
  super
end

.run(output: $stdout, repeats: 1, cases: nil, sample: nil, seed: nil, run_log: nil) ⇒ Object

Runs the whole set and returns one EvalResult per execution. The entry point for a host app running an eval set on its own; Raif::Evals::Run drives the coordinator directly so it can interleave several sets.



103
104
105
106
107
 
# File 'lib/raif/evals/eval_set.rb', line 103

def run(output: $stdout, repeats: 1, cases: nil, sample: nil, seed: nil, run_log: nil)
  EvalSetCoordinator
    .new(eval_set_class: self, output: output, run_log: run_log, cases: cases, sample: sample, seed: seed)
    .run(repeats: repeats)
end

.setup(&block) ⇒ Object 



92
93
94
 
# File 'lib/raif/evals/eval_set.rb', line 92

def setup(&block)
  @setup_block = block
end

.teardown(&block) ⇒ Object 



96
97
98
 
# File 'lib/raif/evals/eval_set.rb', line 96

def teardown(&block)
  @teardown_block = block
end

Instance Method Details

#file(filename) ⇒ Object

Raises:

  • (ArgumentError) 


217
218
219
220
221
222
223
 
# File 'lib/raif/evals/eval_set.rb', line 217

def file(filename)
  path = evals_path("files", filename)

  raise ArgumentError, "File #{filename} does not exist in raif_evals/files/" unless path.exist?

  path.read
end

#files(glob) ⇒ Object

Returns paths relative to raif_evals/files, so they compose with #file rather than having to be turned back into relative paths by the caller.



227
228
229
230
231
232
233
 
# File 'lib/raif/evals/eval_set.rb', line 227

def files(glob)
  base_path = evals_dir("files")

  Dir.glob(evals_path("files", glob).to_s).sort.select { |path| File.file?(path) }.map do |path|
    Pathname.new(path).relative_path_from(base_path).to_s
  end
end

#json(filename) ⇒ Object

A JSON array of case objects, from raif_evals/datasets.



241
242
243
 
# File 'lib/raif/evals/eval_set.rb', line 241

def json(filename)
  JSON.parse(dataset_file(filename))
end

#jsonl(filename) ⇒ Object

One JSON case object per line, from raif_evals/datasets.



236
237
238
 
# File 'lib/raif/evals/eval_set.rb', line 236

def jsonl(filename)
  dataset_file(filename).each_line.reject { |line| line.strip.empty? }.map { |line| JSON.parse(line) }
end

#judge_task_attributes ⇒ Object

Attributes every LLM judge this eval set runs needs. Override when the host app has extended Raif::Task with a column the judge task cannot be inserted without. Called per eval, after setup, so it can reference what setup created.



150
151
152
 
# File 'lib/raif/evals/eval_set.rb', line 150

def judge_task_attributes
  {}
end

#resolve_dataset(name) ⇒ Object

Evaluates one of the set's dataset blocks. Here rather than on the coordinator because a dataset block is user DSL - file/files/json/jsonl only exist on an eval set instance.



156
157
158
 
# File 'lib/raif/evals/eval_set.rb', line 156

def resolve_dataset(name)
  instance_eval(&self.class.datasets.fetch(name))
end

#run_eval(eval_definition, eval_case: nil, run_index: nil, case_id_width: nil) ⇒ Object

Runs one execution - one eval block, against one case, on one repeat. Writes the current case and result onto this instance, which is why each execution gets its own.



162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
 
# File 'lib/raif/evals/eval_set.rb', line 162

def run_eval(eval_definition, eval_case: nil, run_index: nil, case_id_width: nil)
  @current_case = eval_case
  @case_id_width = case_id_width
  @current_eval_result = EvalResult.new(
    description: eval_definition.description,
    eval_id: eval_definition.id,
    run_index: run_index,
    eval_index: eval_definition.index,
    case_id: eval_case&.id
  )

  compact = compact_output?
  if compact
    # The compact line reports counts that only exist once the case has run, so
    # per-expectation output is discarded. Errors still go to console_output.
    @output = StringIO.new
  else
    output.puts "Running: #{eval_definition.description}#{" [#{eval_case.id}]" if eval_case}#{" (run #{run_index})" if run_index}"
  end

  begin
    ActiveRecord::Base.transaction do
      # Opened before setup and closed after teardown, so nothing an execution spends escapes
      # the run's cost total. The eval's own share comes from offsets around the eval block.
      completions = ModelCompletionSink.open
      eval_completions_range = nil

      begin
        # A setup that raised leaves the eval block nothing to run against, so it is
        # skipped rather than run against a half-built fixture and billed for it.
        if setup_succeeded?(eval_case)
          eval_completions_start = completions.length
          run_stage("Eval block") { run_block(eval_definition.block, eval_case) }
          eval_completions_range = eval_completions_start...completions.length
        end
      ensure
        run_stage("Teardown") { run_block(self.class.teardown_block, eval_case) } if self.class.teardown_block

        # Closed first, so a capture failure cannot leave the sink open. Capturing after
        # teardown is safe because the sink holds the records rather than re-querying them.
        ModelCompletionSink.close
        capture_model_completions(completions, eval_completions_range)
      end

      raise ActiveRecord::Rollback
    end
  ensure
    @output = console_output
  end

  print_case_summary if compact

  @current_eval_result
end