Module: Raif

Defined in:
lib/raif/default_models.rb,
lib/raif.rb,
lib/raif/cli.rb,
lib/raif/evals.rb,
lib/raif/engine.rb,
lib/raif/version.rb,
lib/raif/cli/base.rb,
lib/raif/messages.rb,
lib/raif/cli/evals.rb,
lib/raif/evals/run.rb,
lib/raif/languages.rb,
lib/raif/llm_registry.rb,
lib/raif/utils/colors.rb,
lib/raif/configuration.rb,
lib/raif/evals/dataset.rb,
lib/raif/evals/run_log.rb,
lib/raif/evals/eval_set.rb,
lib/raif/evals/run_plan.rb,
lib/raif/model_manifest.rb,
lib/raif/cli/evals_setup.rb,
lib/raif/evals/eval_case.rb,
lib/raif/evals/execution.rb,
lib/raif/token_estimator.rb,
lib/raif/cli/evals_report.rb,
lib/raif/evals/comparison.rb,
lib/raif/evals/run_report.rb,
lib/raif/evals/statistics.rb,
lib/raif/cli/evals_compare.rb,
lib/raif/evals/eval_result.rb,
lib/raif/evals/worker_pool.rb,
lib/raif/migration_checker.rb,
lib/raif/evals/console_line.rb,
lib/raif/evals/score_result.rb,
lib/raif/model_manifest/dsl.rb,
lib/raif/json_schema_builder.rb,
lib/raif/evals/console_writer.rb,
lib/raif/evals/eval_definition.rb,
lib/raif/errors/streaming_error.rb,
lib/raif/evals/comparison_report.rb,
lib/raif/embedding_model_registry.rb,
lib/raif/evals/eval_sets/matchers.rb,
lib/raif/evals/expectation_result.rb,
lib/generators/raif/base_generator.rb,
lib/raif/errors/archive_busy_error.rb,
lib/generators/raif/views_generator.rb,
lib/raif/errors/invalid_batch_error.rb,
lib/raif/evals/eval_set_coordinator.rb,
lib/raif/archive_storage/file_system.rb,
lib/raif/errors/blank_response_error.rb,
lib/raif/errors/invalid_config_error.rb,
lib/raif/evals/model_completion_sink.rb,
lib/raif/errors/archive_storage_error.rb,
lib/raif/errors/prompt_template_error.rb,
lib/raif/evals/eval_sets/expectations.rb,
lib/raif/model_manifest/registry_data.rb,
lib/generators/raif/task/task_generator.rb,
lib/generators/raif/agent/agent_generator.rb,
lib/raif/errors/open_ai/json_schema_error.rb,
lib/raif/errors/unsupported_feature_error.rb,
lib/raif/prompt_studio_comparison_builder.rb,
lib/raif/model_manifest/smoke_observations.rb,
lib/raif/errors/action_not_authorized_error.rb,
lib/raif/errors/invalid_json_response_error.rb,
lib/raif/errors/invalid_user_tool_type_error.rb,
lib/generators/raif/install/install_generator.rb,
lib/raif/errors/invalid_model_file_input_error.rb,
lib/generators/raif/eval_set/eval_set_generator.rb,
lib/generators/raif/evals/setup/setup_generator.rb,
lib/raif/errors/instance_dependent_schema_error.rb,
lib/raif/errors/invalid_conversation_type_error.rb,
lib/raif/errors/invalid_model_image_input_error.rb,
lib/raif/evals/eval_sets/llm_judge_expectations.rb,
lib/generators/raif/model_tool/model_tool_generator.rb,
lib/raif/errors/inference_cost_events_backfill_error.rb,
lib/raif/errors/model_completion_authorization_error.rb,
lib/generators/raif/conversation/conversation_generator.rb,
app/models/raif/llm.rb,
app/models/raif/task.rb,
app/models/raif/agent.rb,
app/jobs/raif/archive_job.rb,
app/jobs/raif/application_job.rb,
app/jobs/raif/archive_tasks_job.rb,
app/models/raif/admin/task_stat.rb,
app/models/raif/evals/llm_judge.rb,
app/helpers/raif/application_helper.rb,
app/jobs/raif/conversation_entry_job.rb,
app/models/raif/evals/scoring_rubric.rb,
app/models/raif/model_completion_batch.rb,
app/models/raif/evals/llm_judges/binary.rb,
app/models/raif/evals/llm_judges/scored.rb,
app/models/raif/prompt_studio_batch_run.rb,
app/jobs/raif/prompt_studio_task_run_job.rb,
app/jobs/raif/prompt_studio_batch_run_job.rb,
app/controllers/raif/admin/llms_controller.rb,
app/controllers/raif/admin/stats_controller.rb,
app/controllers/raif/admin/tasks_controller.rb,
app/controllers/raif/application_controller.rb,
app/jobs/raif/archive_model_completions_job.rb,
app/controllers/raif/admin/agents_controller.rb,
app/helpers/raif/shared/conversations_helper.rb,
app/models/raif/evals/llm_judges/comparative.rb,
app/models/raif/prompt_studio_batch_run_item.rb,
app/controllers/raif/admin/configs_controller.rb,
app/jobs/raif/poll_model_completion_batch_job.rb,
app/models/raif/concerns/has_prompt_templates.rb,
app/models/raif/model_completion_batches/x_ai.rb,
app/models/raif/task_batch_completion_handler.rb,
app/controllers/raif/admin/archives_controller.rb,
app/jobs/raif/prompt_studio_batch_run_item_job.rb,
app/jobs/raif/repair_inference_cost_events_job.rb,
app/models/raif/evals/llm_judges/summarization.rb,
app/models/raif/concerns/json_schema_definition.rb,
app/models/raif/model_completion_batches/google.rb,
app/models/raif/agents/native_tool_calling_agent.rb,
app/models/raif/model_completion_batches/open_ai.rb,
app/controllers/raif/admin/application_controller.rb,
app/controllers/raif/admin/stats/tasks_controller.rb,
app/models/raif/model_completion_batches/anthropic.rb,
app/controllers/raif/admin/conversations_controller.rb,
app/controllers/raif/admin/model_completions_controller.rb,
app/jobs/raif/expire_stuck_model_completion_batches_job.rb,
app/controllers/raif/admin/prompt_studio/base_controller.rb,
app/controllers/raif/admin/prompt_studio/tasks_controller.rb,
app/controllers/raif/admin/prompt_studio/agents_controller.rb,
app/controllers/raif/admin/model_tool_invocations_controller.rb,
app/jobs/raif/resume_stalled_model_completion_batch_polls_job.rb,
app/controllers/raif/admin/model_completion_batches_controller.rb,
app/controllers/raif/admin/prompt_studio/batch_runs_controller.rb,
app/controllers/raif/admin/prompt_studio/conversations_controller.rb,
app/controllers/raif/admin/stats/model_tool_invocations_controller.rb

Overview

Schema Information

Table name: raif_model_completion_batches

id :bigint not null, primary key completion_handler_class_name :string creator_type :string ended_at :datetime failed_at :datetime failure_error :string failure_reason :text handler_dispatched_at :datetime llm_model_key :string not null metadata :jsonb model_api_name :string not null next_poll_at :datetime output_token_cost :decimal(10, 6) prompt_token_cost :decimal(10, 6) provider_response :jsonb request_counts :jsonb results_fetched_at :datetime started_at :datetime status :string default("pending"), not null submitted_at :datetime total_cost :decimal(10, 6) type :string not null created_at :datetime not null updated_at :datetime not null creator_id :bigint provider_batch_id :string

Indexes

index_raif_model_completion_batches_on_creator (creator_type,creator_id) index_raif_model_completion_batches_on_next_poll_at (next_poll_at) index_raif_model_completion_batches_on_provider_batch_id (provider_batch_id) index_raif_model_completion_batches_on_status (status) index_raif_model_completion_batches_on_submitted_at (submitted_at) index_raif_model_completion_batches_on_type (type)

Defined Under Namespace

Modules: Admin, Agents, ApplicationHelper, ArchiveAdvisoryLock, ArchiveStorage, CLI, Concerns, Errors, Evals, Generators, Messages, MigrationChecker, ModelCompletionBatches, ModelManifest, Shared, Utils Classes: Agent, ApplicationController, ApplicationJob, ApplicationRecord, Archive, ArchiveJob, ArchiveModelCompletionsJob, ArchivePartition, ArchiveSerializer, ArchiveTasksJob, BaseGenerator, Configuration, Conversation, ConversationEntriesController, ConversationEntry, ConversationEntryJob, ConversationsController, EmbeddingModel, Engine, ExpireStuckModelCompletionBatchesJob, InferenceCostEvent, JsonSchemaBuilder, Llm, ModelCompletion, ModelCompletionBatch, ModelFileInput, ModelImageInput, ModelTool, ModelToolInvocation, PollModelCompletionBatchJob, PromptStudioBatchRun, PromptStudioBatchRunItem, PromptStudioBatchRunItemJob, PromptStudioBatchRunJob, PromptStudioComparisonBuilder, PromptStudioTaskRunJob, RepairInferenceCostEventsJob, ResumeStalledModelCompletionBatchPollsJob, Task, TaskBatchCompletionHandler, TokenEstimator, UserToolInvocation, ViewsGenerator

Constant Summary collapse

VERSION = 
"1.6.0.pre"
SUPPORTED_LANGUAGES = 
[
  "ar",
  "da",
  "de",
  "en",
  "es",
  "fi",
  "fr",
  "he",
  "hi",
  "it",
  "ja",
  "ko",
  "nl",
  "no",
  "pl",
  "pt",
  "ru",
  "sv",
  "th",
  "tr",
  "uk",
  "vi",
  "zh",
].freeze

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.configuration ⇒ Object

Returns the value of attribute configuration.



32
33
34
 
# File 'lib/raif.rb', line 32

def configuration
  @configuration
end

.embedding_model_registry ⇒ Object

Returns the value of attribute embedding_model_registry.



5
6
7
 
# File 'lib/raif/embedding_model_registry.rb', line 5

def embedding_model_registry
  @embedding_model_registry
end

.logger ⇒ Object 



45
46
47
 
# File 'lib/raif.rb', line 45

def self.logger
  @logger ||= Rails.logger
end

Class Method Details

.available_embedding_model_keys ⇒ Object 



44
45
46
 
# File 'lib/raif/embedding_model_registry.rb', line 44

def self.available_embedding_model_keys
  embedding_model_registry.keys
end

.available_embedding_models ⇒ Object 



40
41
42
 
# File 'lib/raif/embedding_model_registry.rb', line 40

def self.available_embedding_models
  embedding_model_registry.values
end

.available_llm_keys ⇒ Object 



51
52
53
 
# File 'lib/raif/llm_registry.rb', line 51

def self.available_llm_keys
  llm_registry.keys
end

.available_llms ⇒ Object 



47
48
49
 
# File 'lib/raif/llm_registry.rb', line 47

def self.available_llms
  llm_registry.values
end

.config ⇒ Object 



37
38
39
 
# File 'lib/raif.rb', line 37

def self.config
  @configuration ||= Raif::Configuration.new
end

.configure {|config| ... } ⇒ Object

Yields:



41
42
43
 
# File 'lib/raif.rb', line 41

def self.configure
  yield(config)
end

.default_embedding_model_key ⇒ Object 



13
14
15
 
# File 'lib/raif/embedding_model_registry.rb', line 13

def self.default_embedding_model_key
  Rails.env.test? ? :raif_test_embedding_model : Raif.config.default_embedding_model_key
end

.default_embedding_models ⇒ Object 



20
21
22
 
# File 'lib/raif/default_models.rb', line 20

def self.default_embedding_models
  @default_embedding_models ||= Raif::ModelManifest::RegistryData.embedding_configs(model_manifest).transform_keys(&:constantize)
end

.default_llms ⇒ Object 



16
17
18
 
# File 'lib/raif/default_models.rb', line 16

def self.default_llms
  @default_llms ||= Raif::ModelManifest::RegistryData.llm_configs(model_manifest).transform_keys(&:constantize)
end

.default_request_options ⇒ Object 



53
54
55
56
57
58
59
 
# File 'lib/raif.rb', line 53

def self.default_request_options
  {
    open_timeout: config.request_open_timeout,
    read_timeout: config.request_read_timeout,
    write_timeout: config.request_write_timeout
  }.compact
end

.default_streaming_unsupported_model_keys ⇒ Object

Read while Raif::Configuration initializes, which can happen inside a host app's initializer before autoloading is ready, so this must not constantize anything.



27
28
29
 
# File 'lib/raif/default_models.rb', line 27

def self.default_streaming_unsupported_model_keys
  @default_streaming_unsupported_model_keys ||= Raif::ModelManifest::RegistryData.streaming_unsupported_keys(model_manifest).freeze
end

.embedding_model(model_key) ⇒ Object 



29
30
31
32
33
34
35
36
37
38
 
# File 'lib/raif/embedding_model_registry.rb', line 29

def self.embedding_model(model_key)
  embedding_model_config = embedding_model_registry[model_key&.to_sym]

  if embedding_model_config.nil?
    raise ArgumentError, "No embedding model found for model key: #{model_key}. Available models: #{available_embedding_model_keys.join(", ")}"
  end

  embedding_model_class = embedding_model_config[:embedding_model_class]
  embedding_model_class.new(**embedding_model_config.except(:embedding_model_class))
end

.embedding_model_config(model_key) ⇒ Object 



48
49
50
 
# File 'lib/raif/embedding_model_registry.rb', line 48

def self.embedding_model_config(model_key)
  embedding_model_registry[model_key&.to_sym]
end

.generate_embedding!(input, dimensions: nil) ⇒ Object 



8
9
10
11
 
# File 'lib/raif/embedding_model_registry.rb', line 8

def self.generate_embedding!(input, dimensions: nil)
  embedding_model = embedding_model(default_embedding_model_key.to_sym)
  embedding_model.generate_embedding!(input, dimensions:)
end

.llm(model_key) ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/raif/llm_registry.rb', line 20

def self.llm(model_key)
  llm_config = llm_registry[model_key&.to_sym]

  if llm_config.nil?
    raise ArgumentError, "No LLM found for model key: #{model_key}. Available models: #{available_llm_keys.join(", ")}"
  end

  llm_class = llm_config[:llm_class]
  llm = llm_class.new(**llm_config.except(:llm_class))
  warn_if_deprecated(llm)
  llm
end

.llm_config(model_key) ⇒ Object 



55
56
57
 
# File 'lib/raif/llm_registry.rb', line 55

def self.llm_config(model_key)
  llm_registry[model_key&.to_sym]
end

.llm_registry ⇒ Object 



4
5
6
 
# File 'lib/raif/llm_registry.rb', line 4

def self.llm_registry
  @llm_registry ||= {}
end

.model_manifest ⇒ Object 



12
13
14
 
# File 'lib/raif/default_models.rb', line 12

def self.model_manifest
  @model_manifest ||= Raif::ModelManifest.load
end

.register_embedding_model(embedding_model_class, embedding_model_config) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/raif/embedding_model_registry.rb', line 17

def self.register_embedding_model(embedding_model_class, embedding_model_config)
  embedding_model = embedding_model_class.new(**embedding_model_config)

  unless embedding_model.valid?
    raise ArgumentError, "The embedding model you tried to register is invalid: #{embedding_model.errors.full_messages.join(", ")}"
  end

  key = embedding_model.key.to_sym
  @embedding_model_registry ||= {}
  @embedding_model_registry[key] = embedding_model_config.merge(embedding_model_class: embedding_model_class, key: key)
end

.register_llm(llm_class, llm_config) ⇒ Object 



8
9
10
11
12
13
14
15
16
17
18
 
# File 'lib/raif/llm_registry.rb', line 8

def self.register_llm(llm_class, llm_config)
  llm = llm_class.new(**llm_config)

  unless llm.valid?
    raise ArgumentError, "The LLM you tried to register is invalid: #{llm.errors.full_messages.join(", ")}"
  end

  key = llm.key.to_sym
  @llm_registry ||= {}
  @llm_registry[key] = llm_config.merge(llm_class: llm_class, key: key)
end

.reset_deprecation_warnings! ⇒ Object 



43
44
45
 
# File 'lib/raif/llm_registry.rb', line 43

def self.reset_deprecation_warnings!
  @deprecation_warnings_issued = {}
end

.running_evals? ⇒ Boolean

Returns:

  • (Boolean) 


49
50
51
 
# File 'lib/raif.rb', line 49

def self.running_evals?
  ENV["RAIF_RUNNING_EVALS"] == "true"
end

.supported_languages ⇒ Object 



30
31
32
 
# File 'lib/raif/languages.rb', line 30

def self.supported_languages
  SUPPORTED_LANGUAGES
end

.warn_if_deprecated(llm) ⇒ Object 



33
34
35
36
37
38
39
40
41
 
# File 'lib/raif/llm_registry.rb', line 33

def self.warn_if_deprecated(llm)
  return unless llm.deprecated?

  @deprecation_warnings_issued ||= {}
  return if @deprecation_warnings_issued[llm.key]

  @deprecation_warnings_issued[llm.key] = true
  Raif.logger.warn(llm.deprecation_message)
end