'Tensor' object has no attribute 'bnb_quant_state'
#2
by
b0yan9
- opened
Hi,
I tried to run this model by vllm serve "unsloth/Qwen2.5-VL-32B-Instruct-bnb-4bit" --dtype bfloat16 --load_format bitsandbytes --quantization bitsandbytes --host 0.0.0.0 --port 8000 --api-key mykey --cpu-offload-gb 10, the following exceptions occured:
ERROR 03-26 15:39:49 [engine.py:448] 'Tensor' object has no attribute 'bnb_quant_state'
ERROR 03-26 15:39:49 [engine.py:448] Traceback (most recent call last):
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 436, in run_mp_engine
ERROR 03-26 15:39:49 [engine.py:448] engine = MQLLMEngine.from_vllm_config(
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 128, in from_vllm_config
ERROR 03-26 15:39:49 [engine.py:448] return cls(
ERROR 03-26 15:39:49 [engine.py:448] ^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 82, in __init__
ERROR 03-26 15:39:49 [engine.py:448] self.engine = LLMEngine(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/llm_engine.py", line 283, in __init__
ERROR 03-26 15:39:49 [engine.py:448] self._initialize_kv_caches()
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/llm_engine.py", line 432, in _initialize_kv_caches
ERROR 03-26 15:39:49 [engine.py:448] self.model_executor.determine_num_available_blocks())
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/executor/executor_base.py", line 102, in determine_num_available_blocks
ERROR 03-26 15:39:49 [engine.py:448] results = self.collective_rpc("determine_num_available_blocks")
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/executor/uniproc_executor.py", line 56, in collective_rpc
ERROR 03-26 15:39:49 [engine.py:448] answer = run_method(self.driver_worker, method, args, kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/utils.py", line 2255, in run_method
ERROR 03-26 15:39:49 [engine.py:448] return func(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
ERROR 03-26 15:39:49 [engine.py:448] return func(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/worker.py", line 229, in determine_num_available_blocks
ERROR 03-26 15:39:49 [engine.py:448] self.model_runner.profile_run()
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
ERROR 03-26 15:39:49 [engine.py:448] return func(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/model_runner.py", line 1243, in profile_run
ERROR 03-26 15:39:49 [engine.py:448] self._dummy_run(max_num_batched_tokens, max_num_seqs)
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/model_runner.py", line 1354, in _dummy_run
ERROR 03-26 15:39:49 [engine.py:448] self.execute_model(model_input, kv_caches, intermediate_tensors)
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
ERROR 03-26 15:39:49 [engine.py:448] return func(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/model_runner.py", line 1742, in execute_model
ERROR 03-26 15:39:49 [engine.py:448] hidden_or_intermediate_states = model_executable(
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
ERROR 03-26 15:39:49 [engine.py:448] return self._call_impl(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
ERROR 03-26 15:39:49 [engine.py:448] return forward_call(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2_5_vl.py", line 1086, in forward
ERROR 03-26 15:39:49 [engine.py:448] hidden_states = self.language_model.model(
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/compilation/decorators.py", line 172, in __call__
ERROR 03-26 15:39:49 [engine.py:448] return self.forward(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2.py", line 338, in forward
ERROR 03-26 15:39:49 [engine.py:448] hidden_states, residual = layer(
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
ERROR 03-26 15:39:49 [engine.py:448] return self._call_impl(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
ERROR 03-26 15:39:49 [engine.py:448] return forward_call(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 541, in forward
ERROR 03-26 15:39:49 [engine.py:448] output = functional_call(module,
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/_functorch/functional_call.py", line 148, in functional_call
ERROR 03-26 15:39:49 [engine.py:448] return nn.utils.stateless._functional_call(
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/utils/stateless.py", line 300, in _functional_call
ERROR 03-26 15:39:49 [engine.py:448] return module(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
ERROR 03-26 15:39:49 [engine.py:448] return self._call_impl(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
ERROR 03-26 15:39:49 [engine.py:448] return forward_call(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2.py", line 243, in forward
ERROR 03-26 15:39:49 [engine.py:448] hidden_states = self.self_attn(
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
ERROR 03-26 15:39:49 [engine.py:448] return self._call_impl(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
ERROR 03-26 15:39:49 [engine.py:448] return forward_call(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2.py", line 174, in forward
ERROR 03-26 15:39:49 [engine.py:448] qkv, _ = self.qkv_proj(hidden_states)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
ERROR 03-26 15:39:49 [engine.py:448] return self._call_impl(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
ERROR 03-26 15:39:49 [engine.py:448] return forward_call(*args, **kwargs)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/layers/linear.py", line 474, in forward
ERROR 03-26 15:39:49 [engine.py:448] output_parallel = self.quant_method.apply(self, input_, bias)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/layers/quantization/bitsandbytes.py", line 237, in apply
ERROR 03-26 15:39:49 [engine.py:448] return self._apply_4bit_weight(layer, x, bias)
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/layers/quantization/bitsandbytes.py", line 336, in _apply_4bit_weight
ERROR 03-26 15:39:49 [engine.py:448] quant_states = qweight.bnb_quant_state
ERROR 03-26 15:39:49 [engine.py:448] ^^^^^^^^^^^^^^^^^^^^^^^
ERROR 03-26 15:39:49 [engine.py:448] AttributeError: 'Tensor' object has no attribute 'bnb_quant_state'
Process SpawnProcess-1:
Traceback (most recent call last):
File "/home/bo/.conda/envs/vllm/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/home/bo/.conda/envs/vllm/lib/python3.12/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 450, in run_mp_engine
raise e
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 436, in run_mp_engine
engine = MQLLMEngine.from_vllm_config(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 128, in from_vllm_config
return cls(
^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 82, in __init__
self.engine = LLMEngine(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/llm_engine.py", line 283, in __init__
self._initialize_kv_caches()
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/llm_engine.py", line 432, in _initialize_kv_caches
self.model_executor.determine_num_available_blocks())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/executor/executor_base.py", line 102, in determine_num_available_blocks
results = self.collective_rpc("determine_num_available_blocks")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/executor/uniproc_executor.py", line 56, in collective_rpc
answer = run_method(self.driver_worker, method, args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/utils.py", line 2255, in run_method
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/worker.py", line 229, in determine_num_available_blocks
self.model_runner.profile_run()
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/model_runner.py", line 1243, in profile_run
self._dummy_run(max_num_batched_tokens, max_num_seqs)
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/model_runner.py", line 1354, in _dummy_run
self.execute_model(model_input, kv_caches, intermediate_tensors)
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/model_runner.py", line 1742, in execute_model
hidden_or_intermediate_states = model_executable(
^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2_5_vl.py", line 1086, in forward
hidden_states = self.language_model.model(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/compilation/decorators.py", line 172, in __call__
return self.forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2.py", line 338, in forward
hidden_states, residual = layer(
^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 541, in forward
output = functional_call(module,
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/_functorch/functional_call.py", line 148, in functional_call
return nn.utils.stateless._functional_call(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/utils/stateless.py", line 300, in _functional_call
return module(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2.py", line 243, in forward
hidden_states = self.self_attn(
^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2.py", line 174, in forward
qkv, _ = self.qkv_proj(hidden_states)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/layers/linear.py", line 474, in forward
output_parallel = self.quant_method.apply(self, input_, bias)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/layers/quantization/bitsandbytes.py", line 237, in apply
return self._apply_4bit_weight(layer, x, bias)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/layers/quantization/bitsandbytes.py", line 336, in _apply_4bit_weight
quant_states = qweight.bnb_quant_state
^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: 'Tensor' object has no attribute 'bnb_quant_state'
[rank0]:[W326 15:39:49.723718046 ProcessGroupNCCL.cpp:1496] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
Traceback (most recent call last):
File "/home/bo/.conda/envs/vllm/bin/vllm", line 8, in <module>
sys.exit(main())
^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/entrypoints/cli/main.py", line 75, in main
args.dispatch_function(args)
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/entrypoints/cli/serve.py", line 33, in cmd
uvloop.run(run_server(args))
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/uvloop/__init__.py", line 109, in run
return __asyncio.run(
^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/uvloop/__init__.py", line 61, in wrapper
return await main
^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1016, in run_server
async with build_async_engine_client(args) as engine_client:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/contextlib.py", line 210, in __aenter__
return await anext(self.gen)
^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 141, in build_async_engine_client
async with build_async_engine_client_from_engine_args(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/contextlib.py", line 210, in __aenter__
return await anext(self.gen)
^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 264, in build_async_engine_client_from_engine_args
raise RuntimeError(
RuntimeError: Engine process failed to start. See stack trace for the root cause.
b0yan9
changed discussion status to
closed