vladislavbro commited on
Commit
2ab2c69
·
verified ·
1 Parent(s): d446d53

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -11
tokenizer.json CHANGED
@@ -447,17 +447,7 @@
447
  ],
448
  "normalizer": null,
449
  "pre_tokenizer": {
450
- "type": "Sequence",
451
- "pre_tokenizers": [
452
- {
453
- "type": "Replace",
454
- "pattern": " ",
455
- "content": "[SPACE]"
456
- },
457
- {
458
- "type": "Whitespace"
459
- }
460
- ]
461
  },
462
  "post_processor": {
463
  "type": "TemplateProcessing",
 
447
  ],
448
  "normalizer": null,
449
  "pre_tokenizer": {
450
+ "type": "Whitespace"
 
 
 
 
 
 
 
 
 
 
451
  },
452
  "post_processor": {
453
  "type": "TemplateProcessing",