Add a response template
Browse filesSame as https://huggingface.co/allenai/AstaBrief_8B/discussions/1 (the tokenizer config is identical).
This PR adds a `response_template` to `tokenizer_config.json`, so `tokenizer.parse_response(...)` turns a generation back into a message with `reasoning_content`, `content` and `tool_calls` (transformers >= 5.13). It's the Qwen3 one, since the chat template is the Qwen3 one.
- tokenizer_config.json +28 -1
tokenizer_config.json
CHANGED
|
@@ -235,5 +235,32 @@
|
|
| 235 |
"pad_token": "<|endoftext|>",
|
| 236 |
"split_special_tokens": false,
|
| 237 |
"tokenizer_class": "Qwen2Tokenizer",
|
| 238 |
-
"unk_token": null
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 239 |
}
|
|
|
|
| 235 |
"pad_token": "<|endoftext|>",
|
| 236 |
"split_special_tokens": false,
|
| 237 |
"tokenizer_class": "Qwen2Tokenizer",
|
| 238 |
+
"unk_token": null,
|
| 239 |
+
"response_template": {
|
| 240 |
+
"defaults": {
|
| 241 |
+
"role": "assistant"
|
| 242 |
+
},
|
| 243 |
+
"start_anchor": "<|im_start|>assistant\n",
|
| 244 |
+
"fields": {
|
| 245 |
+
"reasoning_content": {
|
| 246 |
+
"open": "<think>",
|
| 247 |
+
"close_pattern": "</think>\\s*",
|
| 248 |
+
"content": "text"
|
| 249 |
+
},
|
| 250 |
+
"tool_calls": {
|
| 251 |
+
"open": "<tool_call>",
|
| 252 |
+
"close_pattern": "</tool_call>\\s*",
|
| 253 |
+
"repeats": true,
|
| 254 |
+
"content": "json",
|
| 255 |
+
"transform": {
|
| 256 |
+
"type": "function",
|
| 257 |
+
"function": "{content}"
|
| 258 |
+
}
|
| 259 |
+
},
|
| 260 |
+
"content": {
|
| 261 |
+
"close_pattern": "<\\|im_end\\|>\\s*",
|
| 262 |
+
"content": "text"
|
| 263 |
+
}
|
| 264 |
+
}
|
| 265 |
+
}
|
| 266 |
}
|