JustinLin610 commited on
Commit
8d7d06f
·
verified ·
1 Parent(s): 5fd5718

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +51 -21
README.md CHANGED
@@ -45,16 +45,28 @@ KeyError: 'qwen3'
45
  ```
46
 
47
  The following contains a code snippet illustrating how to use the model generate content based on given inputs.
 
48
  ```python
49
  from mlx_lm import load, generate
 
50
  model, tokenizer = load("Qwen/Qwen3-0.6B-MLX-4bit")
51
- prompt = "hello, Introduce yourself, and what can you do ?"
 
52
  if tokenizer.chat_template is not None:
53
  messages = [{"role": "user", "content": prompt}]
54
  prompt = tokenizer.apply_chat_template(
55
- messages, add_generation_prompt=True
 
56
  )
57
- response = generate(model, tokenizer, prompt=prompt, verbose=True, max_tokens=1024)
 
 
 
 
 
 
 
 
58
  ```
59
 
60
  ## Switching Between Thinking and Non-Thinking Mode
@@ -108,6 +120,8 @@ Here is an example of a multi-turn conversation:
108
 
109
  ```python
110
  from mlx_lm import load, generate
 
 
111
  class QwenChatbot:
112
  def __init__(self, model_name="Qwen/Qwen3-0.6B-MLX-4bit"):
113
  self.model, self.tokenizer = load(model_name)
@@ -122,13 +136,20 @@ class QwenChatbot:
122
  add_generation_prompt=True
123
  )
124
 
125
- response = generate(self.model, self.tokenizer, prompt=text, verbose=True, max_tokens=32768)
 
 
 
 
 
 
126
  # Update history
127
  self.history.append({"role": "user", "content": user_input})
128
  self.history.append({"role": "assistant", "content": response})
129
 
130
  return response
131
 
 
132
  # Example Usage
133
  if __name__ == "__main__":
134
  chatbot = QwenChatbot()
@@ -144,7 +165,7 @@ if __name__ == "__main__":
144
  user_input_2 = "Then, how many r's in blueberries? /no_think"
145
  print(f"User: {user_input_2}")
146
  response_2 = chatbot.generate_response(user_input_2)
147
- print(f"Bot: {response_2}")
148
  print("----------------------")
149
 
150
  # Third input with /think
@@ -163,35 +184,37 @@ if __name__ == "__main__":
163
  Qwen3 excels in tool calling capabilities. We recommend using [Qwen-Agent](https://github.com/QwenLM/Qwen-Agent) to make the best use of agentic ability of Qwen3. Qwen-Agent encapsulates tool-calling templates and tool-calling parsers internally, greatly reducing coding complexity.
164
 
165
  To define the available tools, you can use the MCP configuration file, use the integrated tool of Qwen-Agent, or integrate other tools by yourself.
 
166
  ```python
167
  from qwen_agent.agents import Assistant
168
 
169
  # Define LLM
170
  llm_cfg = {
171
- 'model': 'Qwen3-0.6B-MLX-4bit',
172
 
173
  # Use the endpoint provided by Alibaba Model Studio:
174
- # 'model_type': 'qwen_dashscope',
175
- # 'api_key': os.getenv('DASHSCOPE_API_KEY'),
176
 
177
  # Use a custom endpoint compatible with OpenAI API:
178
- 'model_server': 'http://localhost:8000/v1', # api_base
179
- 'api_key': 'EMPTY',
180
 
181
  # Other parameters:
182
- # 'generate_cfg': {
183
- # # Add: When the response content is `<think>this is the thought</think>this is the answer;
184
- # # Do not add: When the response has been separated by reasoning_content and content.
185
- # 'thought_in_content': True,
186
- # },
187
  }
188
 
189
  # Define Tools
190
  tools = [
191
- {'mcpServers': { # You can specify the MCP configuration file
192
- 'time': {
193
- 'command': 'uvx',
194
- 'args': ['mcp-server-time', '--local-timezone=Asia/Shanghai']
 
195
  },
196
  "fetch": {
197
  "command": "uvx",
@@ -199,16 +222,23 @@ tools = [
199
  }
200
  }
201
  },
202
- 'code_interpreter', # Built-in tools
203
  ]
204
 
205
  # Define Agent
206
  bot = Assistant(llm=llm_cfg, function_list=tools)
207
 
208
  # Streaming generation
209
- messages = [{'role': 'user', 'content': 'https://qwenlm.github.io/blog/ Introduce the latest developments of Qwen'}]
 
 
 
 
 
 
210
  for responses in bot.run(messages=messages):
211
  pass
 
212
  print(responses)
213
  ```
214
 
 
45
  ```
46
 
47
  The following contains a code snippet illustrating how to use the model generate content based on given inputs.
48
+
49
  ```python
50
  from mlx_lm import load, generate
51
+
52
  model, tokenizer = load("Qwen/Qwen3-0.6B-MLX-4bit")
53
+ prompt = "hello, Introduce yourself, and what can you do?"
54
+
55
  if tokenizer.chat_template is not None:
56
  messages = [{"role": "user", "content": prompt}]
57
  prompt = tokenizer.apply_chat_template(
58
+ messages,
59
+ add_generation_prompt=True
60
  )
61
+
62
+ response = generate(
63
+ model,
64
+ tokenizer,
65
+ prompt=prompt,
66
+ verbose=True,
67
+ max_tokens=1024
68
+ )
69
+ print(response)
70
  ```
71
 
72
  ## Switching Between Thinking and Non-Thinking Mode
 
120
 
121
  ```python
122
  from mlx_lm import load, generate
123
+
124
+
125
  class QwenChatbot:
126
  def __init__(self, model_name="Qwen/Qwen3-0.6B-MLX-4bit"):
127
  self.model, self.tokenizer = load(model_name)
 
136
  add_generation_prompt=True
137
  )
138
 
139
+ response = generate(
140
+ self.model,
141
+ self.tokenizer,
142
+ prompt=text,
143
+ verbose=True,
144
+ max_tokens=32768
145
+ )
146
  # Update history
147
  self.history.append({"role": "user", "content": user_input})
148
  self.history.append({"role": "assistant", "content": response})
149
 
150
  return response
151
 
152
+
153
  # Example Usage
154
  if __name__ == "__main__":
155
  chatbot = QwenChatbot()
 
165
  user_input_2 = "Then, how many r's in blueberries? /no_think"
166
  print(f"User: {user_input_2}")
167
  response_2 = chatbot.generate_response(user_input_2)
168
+ print(f"Bot: {response_2}")
169
  print("----------------------")
170
 
171
  # Third input with /think
 
184
  Qwen3 excels in tool calling capabilities. We recommend using [Qwen-Agent](https://github.com/QwenLM/Qwen-Agent) to make the best use of agentic ability of Qwen3. Qwen-Agent encapsulates tool-calling templates and tool-calling parsers internally, greatly reducing coding complexity.
185
 
186
  To define the available tools, you can use the MCP configuration file, use the integrated tool of Qwen-Agent, or integrate other tools by yourself.
187
+
188
  ```python
189
  from qwen_agent.agents import Assistant
190
 
191
  # Define LLM
192
  llm_cfg = {
193
+ "model": "Qwen3-0.6B-MLX-4bit",
194
 
195
  # Use the endpoint provided by Alibaba Model Studio:
196
+ # "model_type": "qwen_dashscope",
197
+ # "api_key": os.getenv("DASHSCOPE_API_KEY"),
198
 
199
  # Use a custom endpoint compatible with OpenAI API:
200
+ "model_server": "http://localhost:8000/v1", # api_base
201
+ "api_key": "EMPTY",
202
 
203
  # Other parameters:
204
+ # "generate_cfg": {
205
+ # # Add: When the response content is `<think>this is the thought</think>this is the answer;
206
+ # # Do not add: When the response has been separated by reasoning_content and content.
207
+ # "thought_in_content": True,
208
+ # },
209
  }
210
 
211
  # Define Tools
212
  tools = [
213
+ {
214
+ "mcpServers": { # You can specify the MCP configuration file
215
+ "time": {
216
+ "command": "uvx",
217
+ "args": ["mcp-server-time", "--local-timezone=Asia/Shanghai"]
218
  },
219
  "fetch": {
220
  "command": "uvx",
 
222
  }
223
  }
224
  },
225
+ "code_interpreter", # Built-in tools
226
  ]
227
 
228
  # Define Agent
229
  bot = Assistant(llm=llm_cfg, function_list=tools)
230
 
231
  # Streaming generation
232
+ messages = [
233
+ {
234
+ "role": "user",
235
+ "content": "https://qwenlm.github.io/blog/ Introduce the latest developments of Qwen"
236
+ }
237
+ ]
238
+
239
  for responses in bot.run(messages=messages):
240
  pass
241
+
242
  print(responses)
243
  ```
244