Open Inference Protocol Using Python SDK
Use the following code sample to interact with a model endpoint using the Open Inference Protocol.
Run the following command to customize the
ENDPOINT_NAME,
DOMAIN, and MODEL_NAME placeholders. The inference
request payload shape must match what your model is expecting, which you can determine
with the read_model_metadata() function.
#!pip install open-inference-openapi from open_inference.openapi.client import OpenInferenceClient, InferenceRequest import httpx import json import os # # inspired by https://pypi.org/project/open-inference-openapi/ # CDP_TOKEN = os.environ['CDP_TOKEN'] BASE_URL = 'https://[***DOMAIN***]/namespaces/serving-default/endpoints/[***ENDPOINT_NAME***]' MODEL_NAME = '[***MODEL-NAME***]' headers = {'Authorization': 'Bearer ' + CDP_TOKEN, 'Content-Type': 'application/json'} httpx_client = httpx.Client(headers=headers) client = OpenInferenceClient(base_url=BASE_URL, httpx_client=httpx_client) # Check that the server is live, and it has the model loaded client.check_server_readiness() metadata = client.read_model_metadata(MODEL_NAME) metadata_str = json.dumps(json.loads(metadata.json()), indent=2) print(metadata_str) # Make an inference request pred = client.model_infer( MODEL_NAME, request=InferenceRequest( inputs=[ { "name": "float_input", "shape": [1, 11], "datatype": "FP32", "data": [7.4] * 11 # This is a shorter way to write the same data } ] ), ) json_resp_str = json.dumps(json.loads(pred.json()), indent=2) print(json_resp_str)
