I would like to deploy an ML endpoint (first locally to see the error messages) from custom environment. But I am getting the following error message. Can anybody clarify what does this stands for?
The environment deployment is fine, I can pull and play with the environment image, the the ML model also works as it expected, but in the endpoint development I fail.
from azureml.core.model import InferenceConfig, Model
from azureml.core.webservice import AciWebservice, LocalWebservice
ocr_model = Model(name='OCR_model', workspace=workspace)
quality_model = Model(name='Quality_model', workspace=workspace)
inference_config = InferenceConfig(entry_script="ml_scores.py", environment=python_env)
deployment_config = LocalWebservice.deploy_configuration(port=6789)
service = Model.deploy(workspace=workspace,
name="ml-service-v2",
overwrite=True,
models=[ocr_model, quality_model],
inference_config=inference_config,
deployment_config=deployment_config)
service.wait_for_deployment(show_output=True)
The error message:
Warning, custom base image or base dockerfile detected without a specified `inferencing_stack_version`. Please set environment.inferencing_stack_version='latest'
Warning, custom base image or base dockerfile detected without a specified `inferencing_stack_version`. Please set environment.inferencing_stack_version='latest'
Docker container start has failed:
400 Client Error for http+docker://localhost/v1.41/containers/f4ee0f109af60a21a11683dd3d69db87b6371b2c3b038ea0227a8cc8422c4200/start: Bad Request ("failed to create shim: OCI runtime create failed: container_linux.go:380: starting container process caused: exec: "runsvdir": executable file not found in $PATH: unknown")
Downloading model OCR_model:42 to /tmp/azureml_25emn1le/OCR_model/42
Downloading model Quality_model:40 to /tmp/azureml_25emn1le/Quality_model/40
Generating Docker build context.
Package creation Succeeded
Logging into Docker registry 088e52be0d7942c4a6e11258fa37a140.azurecr.io
Logging into Docker registry 088e52be0d7942c4a6e11258fa37a140.azurecr.io
Building Docker image from Dockerfile...
Step 1/5 : FROM 088e52be0d7942c4a6e11258fa37a140.azurecr.io/azureml/azureml_152ffef9bfdf00715d63c8352ff2cc66
---> 31b97aac1e18
Step 2/5 : COPY azureml-app /var/azureml-app
---> a31c2d80deb2
Step 3/5 : RUN mkdir -p '/var/azureml-app' && echo eyJhY2NvdW50Q29udGV4dCI6eyJzdWJzY3JpcHRpb25JZCI6IjIwMjQ0MjYwLWQyY2ItNDNkMy05NjZlLWQwZjcyMDE3ZWJhMCIsInJlc291cmNlR3JvdXBOYW1lIjoibWxfYmV0YV9yZyIsImFjY291bnROYW1lIjoibWFjaGluZS1sZWFybmluZy1iZXRhLXdzIiwid29ya3NwYWNlSWQiOiIwODhlNTJiZS0wZDc5LTQyYzQtYTZlMS0xMjU4ZmEzN2ExNDAifSwibW9kZWxzIjp7fSwibW9kZWxzSW5mbyI6e319 | base64 --decode > /var/azureml-app/model_config_map.json
---> Running in 783f521ea037
---> 8900434882b2
Step 4/5 : RUN mv '/var/azureml-app/tmpdswbb3xk.py' /var/azureml-app/main.py
---> Running in 8b61d3f35a90
---> 05890493cdd1
Step 5/5 : CMD ["runsvdir","/var/runit"]
---> Running in 052ae96b5d75
---> b15b8dfb9deb
Successfully built b15b8dfb9deb
Successfully tagged ml-service-v2:latest
Container (name:xenodochial_goldwasser, id:2757f295fa1836dd83d2dcb162ccbb566c04cba1f3fb0c35943f5d4d101d5479) cannot be killed.
Container has been successfully cleaned up.
Image sha256:ed2b4e50c397597d857883a76e5d53a5c27ce700894d3b2f7167d477f7b82be2 successfully removed.
Starting Docker container...
---------------------------------------------------------------------------
HTTPError Traceback (most recent call last)
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/docker/api/client.py:268, in APIClient._raise_for_status(self, response)
267 try:
--> 268 response.raise_for_status()
269 except requests.exceptions.HTTPError as e:
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/requests/models.py:1021, in Response.raise_for_status(self)
1020 if http_error_msg:
-> 1021 raise HTTPError(http_error_msg, response=self)
HTTPError: 400 Client Error: Bad Request for url: http+docker://localhost/v1.41/containers/f4ee0f109af60a21a11683dd3d69db87b6371b2c3b038ea0227a8cc8422c4200/start
During handling of the above exception, another exception occurred:
APIError Traceback (most recent call last)
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/_model_management/_util.py:495, in start_docker_container(container)
493 print("Starting Docker container...")
--> 495 container.start()
497 print('Docker container running.')
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/docker/models/containers.py:404, in Container.start(self, **kwargs)
396 """
397 Start this container. Similar to the ``docker start`` command, but
398 doesn't support attach options.
(...)
402 If the server returns an error.
403 """
--> 404 return self.client.api.start(self.id, **kwargs)
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/docker/utils/decorators.py:19, in check_resource.<locals>.decorator.<locals>.wrapped(self, resource_id, *args, **kwargs)
16 raise errors.NullResource(
17 'Resource ID was not provided'
18 )
---> 19 return f(self, resource_id, *args, **kwargs)
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/docker/api/container.py:1109, in ContainerApiMixin.start(self, container, *args, **kwargs)
1108 res = self._post(url)
-> 1109 self._raise_for_status(res)
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/docker/api/client.py:270, in APIClient._raise_for_status(self, response)
269 except requests.exceptions.HTTPError as e:
--> 270 raise create_api_error_from_http_exception(e)
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/docker/errors.py:31, in create_api_error_from_http_exception(e)
30 cls = NotFound
---> 31 raise cls(e, response=response, explanation=explanation)
APIError: 400 Client Error for http+docker://localhost/v1.41/containers/f4ee0f109af60a21a11683dd3d69db87b6371b2c3b038ea0227a8cc8422c4200/start: Bad Request ("failed to create shim: OCI runtime create failed: container_linux.go:380: starting container process caused: exec: "runsvdir": executable file not found in $PATH: unknown")
During handling of the above exception, another exception occurred:
WebserviceException Traceback (most recent call last)
Input In [20], in <cell line: 12>()
9 # deployment_config = AciWebservice.deploy_configuration(cpu_cores=2, memory_gb=4)
10 deployment_config = LocalWebservice.deploy_configuration(port=6789)
---> 12 service = Model.deploy(workspace=workspace,
13 name="ml-service-v2",
14 overwrite=True,
15 models=[ocr_model, quality_model],
16 inference_config=inference_config,
17 deployment_config=deployment_config)
19 service.wait_for_deployment(show_output=True)
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/core/model.py:1649, in Model.deploy(workspace, name, models, inference_config, deployment_config, deployment_target, overwrite, show_output)
1647 # Local webservice.
1648 if deployment_config and isinstance(deployment_config, LocalWebserviceDeploymentConfiguration):
-> 1649 return deployment_config._webservice_type._deploy(workspace, name, models,
1650 inference_config=inference_config,
1651 deployment_config=deployment_config)
1653 # IotWebservice does not support environment-style deployment,
1654 # so make sure we don't deploy IotWebservice with environment;
1655 # We only support ACI, AKS, AKS endpoint, and MIR for now.
1656 from azureml._model_management._constants import IOT_WEBSERVICE_TYPE
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/core/webservice/local.py:739, in LocalWebservice._deploy(workspace, name, models, image_config, deployment_config, wait, inference_config)
735 raise WebserviceException('Error, provided inference configuration must be of type InferenceConfig '
736 'in order to deploy a local service.', logger=module_logger)
738 service = LocalWebservice(workspace, name, must_exist=False)
--> 739 service.update(models=models,
740 image_config=image_config,
741 inference_config=inference_config,
742 deployment_config=deployment_config,
743 wait=wait)
744 return service
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/core/webservice/local.py:72, in _in_state.<locals>.decorator.<locals>.decorated(self, *args, **kwargs)
69 if self.state not in states:
70 raise WebserviceException('Cannot call {}() when service is {}.'.format(func.__name__, self.state),
71 logger=module_logger)
---> 72 return func(self, *args, **kwargs)
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/core/webservice/local.py:549, in LocalWebservice.update(self, models, image_config, deployment_config, wait, inference_config)
547 self._generate_docker_context()
548 self._build_image()
--> 549 self._run_container(wait=wait)
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/core/webservice/local.py:1021, in LocalWebservice._run_container(self, wait)
1018 LocalWebservice._copy_local_asset(container, self._inference_config.entry_script)
1020 # Engage!
-> 1021 start_docker_container(container)
1023 # Record the state of the deployment.
1024 self._container = container
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/_model_management/_util.py:503, in start_docker_container(container)
500 raise WebserviceException('Docker container start has failed, the port you are attempting to use '
501 'is already in use:\n{}'.format(e), logger=module_logger)
502 else:
--> 503 raise WebserviceException('Docker container start has failed:\n{}'.format(e), logger=module_logger)
WebserviceException: WebserviceException:
Message: Docker container start has failed:
400 Client Error for http+docker://localhost/v1.41/containers/f4ee0f109af60a21a11683dd3d69db87b6371b2c3b038ea0227a8cc8422c4200/start: Bad Request ("failed to create shim: OCI runtime create failed: container_linux.go:380: starting container process caused: exec: "runsvdir": executable file not found in $PATH: unknown")
InnerException None
ErrorResponse
{
"error": {
"message": "Docker container start has failed:\n400 Client Error for http+docker://localhost/v1.41/containers/f4ee0f109af60a21a11683dd3d69db87b6371b2c3b038ea0227a8cc8422c4200/start: Bad Request (\"failed to create shim: OCI runtime create failed: container_linux.go:380: starting container process caused: exec: \"runsvdir\": executable file not found in $PATH: unknown\")"
}
}
Thanks for any kind of guidance!
