Skip to content

feat: rename KServe Inference configuration to use RuntimeImage instead of Runtime #621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
RK181 wants to merge 1 commit into
base: devel
Choose a base branch
from
Open

feat: rename KServe Inference configuration to use RuntimeImage instead of Runtime #621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/fdl.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,8 +209,8 @@ storage_providers:

| Field | Description |
|------------------------------| --------------------------------------------|
| `model_format` </br> *string* | Model format expected by KServe for `inference` services. Required when `type` is `inference`. Typical values include: `onnx`, `sklearn`, `xgboost`, `pytorch`, `tensorflow`, `triton`, `huggingface`. |
| `runtime` </br> *string* | Explicit KServe ServingRuntime name to use for `inference` services. Optional. |
| `model_format` </br> *string* | Model format expected by KServe for `inference` services. Required when `type` is `inference`. Typical values include: `onnx`, `sklearn`, `xgboost`, `pytorch`, `tensorflow`, `triton`, `huggingface`. Every model format has its own runtime and the available runtimes may vary depending on the cluster configuration |
| `runtime_image` </br> *string* | Explicit KServe ServingRuntime name to use for `inference` services. Use if the model format are not enough. Optional. |
| `api_version` </br> *string* | Protocol version used by KServe predictors. Allowed values: `v1`, `v2`. Optional. (default: `v1`) |

## KServeLLMInferenceSettings
Expand Down
12 changes: 7 additions & 5 deletions pkg/types/kserve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func (k Kserve) ValidateUpdate(old Kserve) error {
if old.Inference == nil || k.Inference == nil {
return fmt.Errorf("inference configuration cannot be nil for KServe service")
}
if old.Inference.Runtime != k.Inference.Runtime {
if old.Inference.RuntimeImage != k.Inference.RuntimeImage {
return fmt.Errorf("cannot update runtime for KServe")
}
if old.Inference.ModelFormat != k.Inference.ModelFormat {
Expand Down Expand Up @@ -186,13 +186,15 @@ func (k Kserve) Equal(other Kserve) bool {
}

type KserveInference struct {
// ModelFormat the model format to use for KServe InferenceService
// ModelFormat the model format to use for KServe InferenceService.
// Every model format has its own runtime and the available runtimes may vary depending on the cluster configuration.
// ("onnx", "sklearn", "xgboost", "pytorch", "tensorflow", "triton", "huggingface").
ModelFormat string `json:"model_format,omitempty"`
// Runtime the KServe runtime to use
// RuntimeImage the KServe runtime to use. Its a custom runtime if
// the available runtimes in the model format are not enough
// Ref: https://kserve.github.io/website/docs/concepts/resources/servingruntime
// Optional.
Runtime string `json:"runtime,omitempty"`
RuntimeImage string `json:"runtime_image,omitempty"`
// Can be used to specify the protocol version for KServe (e.g., "v1", "v2").
// Optional. (default: "v1")
APIVersion string `json:"api_version,omitempty" default:"v1"`
Expand Down Expand Up @@ -234,7 +236,7 @@ func (k KserveInference) Validate() error {

func (k KserveInference) Equal(other KserveInference) bool {
return k.ModelFormat == other.ModelFormat &&
k.Runtime == other.Runtime &&
k.RuntimeImage == other.RuntimeImage &&
k.APIVersion == other.APIVersion
}

Expand Down
8 changes: 4 additions & 4 deletions pkg/utils/kserve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,8 +372,8 @@ func newKserveInferenceServiceSpec(service *types.Service, owner *KserveServiceO
"protocolVersion": service.Kserve.Inference.APIVersion,
}

if service.Kserve.Inference.Runtime != "" {
modelSpec["runtime"] = service.Kserve.Inference.Runtime
if service.Kserve.Inference.RuntimeImage != "" {
modelSpec["runtime"] = service.Kserve.Inference.RuntimeImage
}
// TO DO: consider if we want to inject root path for LLM services as well, and if so, how to handle the case when the framework is vllm that expects the prefix to be preserved for routing
//injectRootPath(service)
Expand Down Expand Up @@ -436,8 +436,8 @@ func updateKserveInferenceServiceSpec(service *types.Service, oldIsvc *unstructu
"storageUri": service.Kserve.StorageUri,
"protocolVersion": service.Kserve.Inference.APIVersion,
}
if service.Kserve.Inference.Runtime != "" {
modelSpec["runtime"] = service.Kserve.Inference.Runtime
if service.Kserve.Inference.RuntimeImage != "" {
modelSpec["runtime"] = service.Kserve.Inference.RuntimeImage
}

modelSpec["resources"] = resources
Expand Down
4 changes: 2 additions & 2 deletions pkg/utils/kserve_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1018,8 +1018,8 @@ func TestCheckKserveUpdate(t *testing.T) {
{
name: "cannot change runtime",
mutate: func(oldSvc, newSvc *oscarType.Service) {
oldSvc.Kserve.Inference.Runtime = "kserve-runtime-a"
newSvc.Kserve.Inference.Runtime = "kserve-runtime-b"
oldSvc.Kserve.Inference.RuntimeImage = "kserve-runtime-a"
newSvc.Kserve.Inference.RuntimeImage = "kserve-runtime-b"
},
wantErr: true,
},
Expand Down
Loading