Update README.md
Browse files
README.md
CHANGED
@@ -45,16 +45,28 @@ KeyError: 'qwen3'
|
|
45 |
```
|
46 |
|
47 |
The following contains a code snippet illustrating how to use the model generate content based on given inputs.
|
|
|
48 |
```python
|
49 |
from mlx_lm import load, generate
|
|
|
50 |
model, tokenizer = load("Qwen/Qwen3-0.6B-MLX-4bit")
|
51 |
-
prompt = "hello, Introduce yourself, and what can you do
|
|
|
52 |
if tokenizer.chat_template is not None:
|
53 |
messages = [{"role": "user", "content": prompt}]
|
54 |
prompt = tokenizer.apply_chat_template(
|
55 |
-
messages,
|
|
|
56 |
)
|
57 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
58 |
```
|
59 |
|
60 |
## Switching Between Thinking and Non-Thinking Mode
|
@@ -108,6 +120,8 @@ Here is an example of a multi-turn conversation:
|
|
108 |
|
109 |
```python
|
110 |
from mlx_lm import load, generate
|
|
|
|
|
111 |
class QwenChatbot:
|
112 |
def __init__(self, model_name="Qwen/Qwen3-0.6B-MLX-4bit"):
|
113 |
self.model, self.tokenizer = load(model_name)
|
@@ -122,13 +136,20 @@ class QwenChatbot:
|
|
122 |
add_generation_prompt=True
|
123 |
)
|
124 |
|
125 |
-
response = generate(
|
|
|
|
|
|
|
|
|
|
|
|
|
126 |
# Update history
|
127 |
self.history.append({"role": "user", "content": user_input})
|
128 |
self.history.append({"role": "assistant", "content": response})
|
129 |
|
130 |
return response
|
131 |
|
|
|
132 |
# Example Usage
|
133 |
if __name__ == "__main__":
|
134 |
chatbot = QwenChatbot()
|
@@ -144,7 +165,7 @@ if __name__ == "__main__":
|
|
144 |
user_input_2 = "Then, how many r's in blueberries? /no_think"
|
145 |
print(f"User: {user_input_2}")
|
146 |
response_2 = chatbot.generate_response(user_input_2)
|
147 |
-
print(f"Bot: {response_2}")
|
148 |
print("----------------------")
|
149 |
|
150 |
# Third input with /think
|
@@ -163,35 +184,37 @@ if __name__ == "__main__":
|
|
163 |
Qwen3 excels in tool calling capabilities. We recommend using [Qwen-Agent](https://github.com/QwenLM/Qwen-Agent) to make the best use of agentic ability of Qwen3. Qwen-Agent encapsulates tool-calling templates and tool-calling parsers internally, greatly reducing coding complexity.
|
164 |
|
165 |
To define the available tools, you can use the MCP configuration file, use the integrated tool of Qwen-Agent, or integrate other tools by yourself.
|
|
|
166 |
```python
|
167 |
from qwen_agent.agents import Assistant
|
168 |
|
169 |
# Define LLM
|
170 |
llm_cfg = {
|
171 |
-
|
172 |
|
173 |
# Use the endpoint provided by Alibaba Model Studio:
|
174 |
-
#
|
175 |
-
#
|
176 |
|
177 |
# Use a custom endpoint compatible with OpenAI API:
|
178 |
-
|
179 |
-
|
180 |
|
181 |
# Other parameters:
|
182 |
-
#
|
183 |
-
#
|
184 |
-
#
|
185 |
-
#
|
186 |
-
#
|
187 |
}
|
188 |
|
189 |
# Define Tools
|
190 |
tools = [
|
191 |
-
{
|
192 |
-
|
193 |
-
|
194 |
-
|
|
|
195 |
},
|
196 |
"fetch": {
|
197 |
"command": "uvx",
|
@@ -199,16 +222,23 @@ tools = [
|
|
199 |
}
|
200 |
}
|
201 |
},
|
202 |
-
|
203 |
]
|
204 |
|
205 |
# Define Agent
|
206 |
bot = Assistant(llm=llm_cfg, function_list=tools)
|
207 |
|
208 |
# Streaming generation
|
209 |
-
messages = [
|
|
|
|
|
|
|
|
|
|
|
|
|
210 |
for responses in bot.run(messages=messages):
|
211 |
pass
|
|
|
212 |
print(responses)
|
213 |
```
|
214 |
|
|
|
45 |
```
|
46 |
|
47 |
The following contains a code snippet illustrating how to use the model generate content based on given inputs.
|
48 |
+
|
49 |
```python
|
50 |
from mlx_lm import load, generate
|
51 |
+
|
52 |
model, tokenizer = load("Qwen/Qwen3-0.6B-MLX-4bit")
|
53 |
+
prompt = "hello, Introduce yourself, and what can you do?"
|
54 |
+
|
55 |
if tokenizer.chat_template is not None:
|
56 |
messages = [{"role": "user", "content": prompt}]
|
57 |
prompt = tokenizer.apply_chat_template(
|
58 |
+
messages,
|
59 |
+
add_generation_prompt=True
|
60 |
)
|
61 |
+
|
62 |
+
response = generate(
|
63 |
+
model,
|
64 |
+
tokenizer,
|
65 |
+
prompt=prompt,
|
66 |
+
verbose=True,
|
67 |
+
max_tokens=1024
|
68 |
+
)
|
69 |
+
print(response)
|
70 |
```
|
71 |
|
72 |
## Switching Between Thinking and Non-Thinking Mode
|
|
|
120 |
|
121 |
```python
|
122 |
from mlx_lm import load, generate
|
123 |
+
|
124 |
+
|
125 |
class QwenChatbot:
|
126 |
def __init__(self, model_name="Qwen/Qwen3-0.6B-MLX-4bit"):
|
127 |
self.model, self.tokenizer = load(model_name)
|
|
|
136 |
add_generation_prompt=True
|
137 |
)
|
138 |
|
139 |
+
response = generate(
|
140 |
+
self.model,
|
141 |
+
self.tokenizer,
|
142 |
+
prompt=text,
|
143 |
+
verbose=True,
|
144 |
+
max_tokens=32768
|
145 |
+
)
|
146 |
# Update history
|
147 |
self.history.append({"role": "user", "content": user_input})
|
148 |
self.history.append({"role": "assistant", "content": response})
|
149 |
|
150 |
return response
|
151 |
|
152 |
+
|
153 |
# Example Usage
|
154 |
if __name__ == "__main__":
|
155 |
chatbot = QwenChatbot()
|
|
|
165 |
user_input_2 = "Then, how many r's in blueberries? /no_think"
|
166 |
print(f"User: {user_input_2}")
|
167 |
response_2 = chatbot.generate_response(user_input_2)
|
168 |
+
print(f"Bot: {response_2}")
|
169 |
print("----------------------")
|
170 |
|
171 |
# Third input with /think
|
|
|
184 |
Qwen3 excels in tool calling capabilities. We recommend using [Qwen-Agent](https://github.com/QwenLM/Qwen-Agent) to make the best use of agentic ability of Qwen3. Qwen-Agent encapsulates tool-calling templates and tool-calling parsers internally, greatly reducing coding complexity.
|
185 |
|
186 |
To define the available tools, you can use the MCP configuration file, use the integrated tool of Qwen-Agent, or integrate other tools by yourself.
|
187 |
+
|
188 |
```python
|
189 |
from qwen_agent.agents import Assistant
|
190 |
|
191 |
# Define LLM
|
192 |
llm_cfg = {
|
193 |
+
"model": "Qwen3-0.6B-MLX-4bit",
|
194 |
|
195 |
# Use the endpoint provided by Alibaba Model Studio:
|
196 |
+
# "model_type": "qwen_dashscope",
|
197 |
+
# "api_key": os.getenv("DASHSCOPE_API_KEY"),
|
198 |
|
199 |
# Use a custom endpoint compatible with OpenAI API:
|
200 |
+
"model_server": "http://localhost:8000/v1", # api_base
|
201 |
+
"api_key": "EMPTY",
|
202 |
|
203 |
# Other parameters:
|
204 |
+
# "generate_cfg": {
|
205 |
+
# # Add: When the response content is `<think>this is the thought</think>this is the answer;
|
206 |
+
# # Do not add: When the response has been separated by reasoning_content and content.
|
207 |
+
# "thought_in_content": True,
|
208 |
+
# },
|
209 |
}
|
210 |
|
211 |
# Define Tools
|
212 |
tools = [
|
213 |
+
{
|
214 |
+
"mcpServers": { # You can specify the MCP configuration file
|
215 |
+
"time": {
|
216 |
+
"command": "uvx",
|
217 |
+
"args": ["mcp-server-time", "--local-timezone=Asia/Shanghai"]
|
218 |
},
|
219 |
"fetch": {
|
220 |
"command": "uvx",
|
|
|
222 |
}
|
223 |
}
|
224 |
},
|
225 |
+
"code_interpreter", # Built-in tools
|
226 |
]
|
227 |
|
228 |
# Define Agent
|
229 |
bot = Assistant(llm=llm_cfg, function_list=tools)
|
230 |
|
231 |
# Streaming generation
|
232 |
+
messages = [
|
233 |
+
{
|
234 |
+
"role": "user",
|
235 |
+
"content": "https://qwenlm.github.io/blog/ Introduce the latest developments of Qwen"
|
236 |
+
}
|
237 |
+
]
|
238 |
+
|
239 |
for responses in bot.run(messages=messages):
|
240 |
pass
|
241 |
+
|
242 |
print(responses)
|
243 |
```
|
244 |
|