diff --git a/docs/fdl.md b/docs/fdl.md index c442a7f3..ab8e87f4 100644 --- a/docs/fdl.md +++ b/docs/fdl.md @@ -209,8 +209,8 @@ storage_providers: | Field | Description | |------------------------------| --------------------------------------------| -| `model_format`
*string* | Model format expected by KServe for `inference` services. Required when `type` is `inference`. Typical values include: `onnx`, `sklearn`, `xgboost`, `pytorch`, `tensorflow`, `triton`, `huggingface`. | -| `runtime`
*string* | Explicit KServe ServingRuntime name to use for `inference` services. Optional. | +| `model_format`
*string* | Model format expected by KServe for `inference` services. Required when `type` is `inference`. Typical values include: `onnx`, `sklearn`, `xgboost`, `pytorch`, `tensorflow`, `triton`, `huggingface`. Every model format has its own runtime and the available runtimes may vary depending on the cluster configuration | +| `runtime_image`
*string* | Explicit KServe ServingRuntime name to use for `inference` services. Use if the model format are not enough. Optional. | | `api_version`
*string* | Protocol version used by KServe predictors. Allowed values: `v1`, `v2`. Optional. (default: `v1`) | ## KServeLLMInferenceSettings diff --git a/pkg/types/kserve.go b/pkg/types/kserve.go index 2e68e052..48f9e169 100644 --- a/pkg/types/kserve.go +++ b/pkg/types/kserve.go @@ -137,7 +137,7 @@ func (k Kserve) ValidateUpdate(old Kserve) error { if old.Inference == nil || k.Inference == nil { return fmt.Errorf("inference configuration cannot be nil for KServe service") } - if old.Inference.Runtime != k.Inference.Runtime { + if old.Inference.RuntimeImage != k.Inference.RuntimeImage { return fmt.Errorf("cannot update runtime for KServe") } if old.Inference.ModelFormat != k.Inference.ModelFormat { @@ -186,13 +186,15 @@ func (k Kserve) Equal(other Kserve) bool { } type KserveInference struct { - // ModelFormat the model format to use for KServe InferenceService + // ModelFormat the model format to use for KServe InferenceService. + // Every model format has its own runtime and the available runtimes may vary depending on the cluster configuration. // ("onnx", "sklearn", "xgboost", "pytorch", "tensorflow", "triton", "huggingface"). ModelFormat string `json:"model_format,omitempty"` - // Runtime the KServe runtime to use + // RuntimeImage the KServe runtime to use. Its a custom runtime if + // the available runtimes in the model format are not enough // Ref: https://kserve.github.io/website/docs/concepts/resources/servingruntime // Optional. - Runtime string `json:"runtime,omitempty"` + RuntimeImage string `json:"runtime_image,omitempty"` // Can be used to specify the protocol version for KServe (e.g., "v1", "v2"). // Optional. (default: "v1") APIVersion string `json:"api_version,omitempty" default:"v1"` @@ -234,7 +236,7 @@ func (k KserveInference) Validate() error { func (k KserveInference) Equal(other KserveInference) bool { return k.ModelFormat == other.ModelFormat && - k.Runtime == other.Runtime && + k.RuntimeImage == other.RuntimeImage && k.APIVersion == other.APIVersion } diff --git a/pkg/utils/kserve.go b/pkg/utils/kserve.go index 43547d74..d28c01c4 100644 --- a/pkg/utils/kserve.go +++ b/pkg/utils/kserve.go @@ -372,8 +372,8 @@ func newKserveInferenceServiceSpec(service *types.Service, owner *KserveServiceO "protocolVersion": service.Kserve.Inference.APIVersion, } - if service.Kserve.Inference.Runtime != "" { - modelSpec["runtime"] = service.Kserve.Inference.Runtime + if service.Kserve.Inference.RuntimeImage != "" { + modelSpec["runtime"] = service.Kserve.Inference.RuntimeImage } // TO DO: consider if we want to inject root path for LLM services as well, and if so, how to handle the case when the framework is vllm that expects the prefix to be preserved for routing //injectRootPath(service) @@ -436,8 +436,8 @@ func updateKserveInferenceServiceSpec(service *types.Service, oldIsvc *unstructu "storageUri": service.Kserve.StorageUri, "protocolVersion": service.Kserve.Inference.APIVersion, } - if service.Kserve.Inference.Runtime != "" { - modelSpec["runtime"] = service.Kserve.Inference.Runtime + if service.Kserve.Inference.RuntimeImage != "" { + modelSpec["runtime"] = service.Kserve.Inference.RuntimeImage } modelSpec["resources"] = resources diff --git a/pkg/utils/kserve_test.go b/pkg/utils/kserve_test.go index 313ac512..7880d270 100644 --- a/pkg/utils/kserve_test.go +++ b/pkg/utils/kserve_test.go @@ -1018,8 +1018,8 @@ func TestCheckKserveUpdate(t *testing.T) { { name: "cannot change runtime", mutate: func(oldSvc, newSvc *oscarType.Service) { - oldSvc.Kserve.Inference.Runtime = "kserve-runtime-a" - newSvc.Kserve.Inference.Runtime = "kserve-runtime-b" + oldSvc.Kserve.Inference.RuntimeImage = "kserve-runtime-a" + newSvc.Kserve.Inference.RuntimeImage = "kserve-runtime-b" }, wantErr: true, },