HL-SMI Version: hl-1.16.2-rc-fw-50.1.2.0
Driver Version: 1.16.2-f195ec4
Traceback (most recent call last):
File “/usr/local/bin/python-text-embeddings-server”, line 8, in
sys.exit(app())
File “/usr/local/lib/python3.10/dist-packages/typer/main.py”, line 311, in call
return get_command(self)(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/click/core.py”, line 1157, in call
return self.main(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/typer/core.py”, line 716, in main
return _main(
File “/usr/local/lib/python3.10/dist-packages/typer/core.py”, line 216, in _main
rv = self.invoke(ctx)
File “/usr/local/lib/python3.10/dist-packages/click/core.py”, line 1434, in invoke
return ctx.invoke(self.callback, **ctx.params)
File “/usr/local/lib/python3.10/dist-packages/click/core.py”, line 783, in invoke
return __callback(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/typer/main.py”, line 683, in wrapper
return callback(**use_params) # type: ignore
File “/usr/src/backends/python/server/text_embeddings_server/cli.py”, line 50, in serve
server.serve(model_path, dtype, uds_path)
File “/usr/src/backends/python/server/text_embeddings_server/server.py”, line 79, in serve
asyncio.run(serve_inner(model_path, dtype))
File “/usr/lib/python3.10/asyncio/runners.py”, line 44, in run
return loop.run_until_complete(main)
File “/usr/lib/python3.10/asyncio/base_events.py”, line 636, in run_until_complete
self.run_forever()
File “/usr/lib/python3.10/asyncio/base_events.py”, line 603, in run_forever
self._run_once()
File “/usr/lib/python3.10/asyncio/base_events.py”, line 1909, in _run_once
handle._run()
File “/usr/lib/python3.10/asyncio/events.py”, line 80, in _run
self._context.run(self._callback, *self._args)
File “/usr/src/backends/python/server/text_embeddings_server/server.py”, line 48, in serve_inner
model = get_model(model_path, dtype)
File “/usr/src/backends/python/server/text_embeddings_server/models/init.py”, line 66, in get_model
return DefaultModel(model_path, device, dtype)
File “/usr/src/backends/python/server/text_embeddings_server/models/default_model.py”, line 23, in init
model = AutoModel.from_pretrained(model_path).to(dtype).to(device)
File “/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py”, line 2556, in to
return super().to(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/habana_frameworks/torch/core/weight_sharing.py”, line 173, in wrapped_to
result = self.original_to(*args, **kwargs)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1155, in to
return self._apply(convert)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 802, in _apply
module._apply(fn)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 802, in _apply
module._apply(fn)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 825, in _apply
param_applied = fn(param)
File “/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py”, line 1153, in convert
return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking)
File “/usr/local/lib/python3.10/dist-packages/habana_frameworks/torch/core/weight_sharing.py”, line 53, in torch_function
return super().torch_function(func, types, new_args, kwargs)
RuntimeError: synStatus=26 [Generic failure] Device acquire failed.