Class: Raif::Evals::Run

Inherits:
Object
  • Object
show all
Defined in:
lib/raif/evals/run.rb

Defined Under Namespace

Classes: Unit

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(file_paths: nil, output: $stdout, repeats: 1, cases: nil, sample: nil, seed: nil, resume_path: nil, concurrency: nil) ⇒ Run

Returns a new instance of Run.



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/raif/evals/run.rb', line 20

def initialize(file_paths: nil, output: $stdout, repeats: 1, cases: nil, sample: nil, seed: nil, resume_path: nil,
  concurrency: nil)
  @output = output
  @results = {}
  @repeats = [repeats.to_i, 1].max
  @cases = cases.presence
  @resume_path = resume_path.presence
  @sample = sample&.to_i
  @seed = resolve_seed(seed)
  @concurrency = resolve_concurrency(concurrency)
  @reported_eval_sets = Set.new
  @result_order = {}
  @summary_mutex = Mutex.new

  # Before the eval sets, not after: loading an eval set evaluates its class body, so
  # anything setup.rb defines for it (helper modules, rubrics) must exist by then.
  load_setup_file

  @eval_sets = if file_paths&.any?
    load_eval_sets_from_files(file_paths)
  else
    discover_eval_sets
  end
end

Instance Attribute Details

#cases ⇒ Object (readonly)

Returns the value of attribute cases.



18
19
20
 
# File 'lib/raif/evals/run.rb', line 18

def cases
  @cases
end

#concurrency ⇒ Object (readonly)

Returns the value of attribute concurrency.



18
19
20
 
# File 'lib/raif/evals/run.rb', line 18

def concurrency
  @concurrency
end

#eval_sets ⇒ Object (readonly)

Returns the value of attribute eval_sets.



18
19
20
 
# File 'lib/raif/evals/run.rb', line 18

def eval_sets
  @eval_sets
end

#output ⇒ Object (readonly)

Returns the value of attribute output.



18
19
20
 
# File 'lib/raif/evals/run.rb', line 18

def output
  @output
end

#repeats ⇒ Object (readonly)

Returns the value of attribute repeats.



18
19
20
 
# File 'lib/raif/evals/run.rb', line 18

def repeats
  @repeats
end

#results ⇒ Object (readonly)

Returns the value of attribute results.



18
19
20
 
# File 'lib/raif/evals/run.rb', line 18

def results
  @results
end

#resume_path ⇒ Object (readonly)

Returns the value of attribute resume_path.



18
19
20
 
# File 'lib/raif/evals/run.rb', line 18

def resume_path
  @resume_path
end

#sample ⇒ Object (readonly)

Returns the value of attribute sample.



18
19
20
 
# File 'lib/raif/evals/run.rb', line 18

def sample
  @sample
end

#seed ⇒ Object (readonly)

Returns the value of attribute seed.



18
19
20
 
# File 'lib/raif/evals/run.rb', line 18

def seed
  @seed
end

Instance Method Details

#execute ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/raif/evals/run.rb', line 45

def execute
  output.puts "\nStarting Raif Eval Run"
  output.puts ""
  output.puts "Raif.config.default_llm_model_key: #{Raif.config.default_llm_model_key}"
  output.puts "Raif.config.evals_default_llm_judge_model_key: #{configured_judge_description}"
  output.puts "Repeats per eval: #{repeats}"
  output.puts "Concurrency: #{concurrency}" if concurrency > 1
  output.puts "Cases: #{cases.join(", ")}" if cases
  output.puts "Sample per dataset: #{sample}#{" (seed #{seed})" if seed}" if sample

  if resume_path
    output.puts "Resuming: #{run_log.display_path} (#{run_log.results_count} results already recorded)"
  else
    output.puts "Run log: #{run_log.display_path}"
  end

  print_self_judging_warning if judge_is_model_under_test?

  output.puts ""
  output.puts "=" * 50

  run_eval_sets

  # The results file describes a whole run, so it is only written once the whole run has been
  # done - which is a question for the log's plan, not for this invocation's work list. See
  # Raif::Evals::RunPlan.
  complete = run_log.complete?
  export_results if complete
  print_summary

  # --cases filters every dataset in the run, so an id matching nothing anywhere is a typo.
  # Checked on the case ids that actually ran, not the result count, since the non-dataset
  # evals in the same set run regardless. Checked after exporting, since those evals already
  # spent inference money.
  if cases && dataset_evals_present? && @results.values.flatten.none? { |e| e[:case_id] }
    output.puts Raif::Utils::Colors.red("\nNo eval cases matched --cases #{cases.join(",")}")
    exit 1
  end

  unless complete
    print_incomplete_run_notice
    exit 1
  end
end