Open Inference Protocol Using Python SDK

Use the following code sample to interact with a model endpoint using the Open Inference Protocol.

Run the following command to customize the ENDPOINT_NAME, DOMAIN, and MODEL_NAME placeholders. The inference request payload shape must match what your model is expecting, which you can determine with the read_model_metadata() function.
#!pip install open-inference-openapi

from open_inference.openapi.client import OpenInferenceClient, InferenceRequest
import httpx
import json
import os

#
# inspired by https://pypi.org/project/open-inference-openapi/
#

CDP_TOKEN = os.environ['CDP_TOKEN']
BASE_URL = 'https://[***DOMAIN***]/namespaces/serving-default/endpoints/[***ENDPOINT_NAME***]'
MODEL_NAME = '[***MODEL-NAME***]'
headers = {'Authorization': 'Bearer ' + CDP_TOKEN,
           'Content-Type': 'application/json'}

httpx_client = httpx.Client(headers=headers)
client = OpenInferenceClient(base_url=BASE_URL, httpx_client=httpx_client)

# Check that the server is live, and it has the model loaded
client.check_server_readiness()
metadata = client.read_model_metadata(MODEL_NAME)
metadata_str = json.dumps(json.loads(metadata.json()), indent=2)
print(metadata_str)

# Make an inference request
pred = client.model_infer(
    MODEL_NAME,
    request=InferenceRequest(
        inputs=[
            {
            "name": "float_input",
            "shape": [1, 11],
            "datatype": "FP32",
            "data": [7.4] * 11  # This is a shorter way to write the same data
           }
        ]
    ),
)

json_resp_str = json.dumps(json.loads(pred.json()), indent=2)
print(json_resp_str)