Enhanced DARVO detector v2 - 84% accuracy, improved accountability detection
Browse files- README.md +171 -165
- config.json +17 -20
- model.safetensors +2 -2
- model_info.json +14 -0
- special_tokens_map.json +5 -49
- tokenizer.json +0 -0
- tokenizer_config.json +23 -32
- vocab.txt +0 -0
README.md
CHANGED
@@ -1,199 +1,205 @@
|
|
1 |
---
|
|
|
|
|
2 |
library_name: transformers
|
3 |
-
tags:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
4 |
---
|
5 |
|
6 |
-
#
|
7 |
|
8 |
-
|
9 |
|
|
|
10 |
|
|
|
|
|
|
|
11 |
|
12 |
-
##
|
13 |
|
14 |
-
|
|
|
|
|
|
|
15 |
|
16 |
-
|
17 |
|
18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
19 |
|
20 |
-
|
21 |
-
- **Funded by [optional]:** [More Information Needed]
|
22 |
-
- **Shared by [optional]:** [More Information Needed]
|
23 |
-
- **Model type:** [More Information Needed]
|
24 |
-
- **Language(s) (NLP):** [More Information Needed]
|
25 |
-
- **License:** [More Information Needed]
|
26 |
-
- **Finetuned from model [optional]:** [More Information Needed]
|
27 |
|
28 |
-
|
|
|
|
|
29 |
|
30 |
-
|
|
|
|
|
31 |
|
32 |
-
|
33 |
-
|
34 |
-
- **Demo [optional]:** [More Information Needed]
|
35 |
|
36 |
-
|
|
|
|
|
|
|
|
|
37 |
|
38 |
-
|
|
|
39 |
|
40 |
-
|
41 |
|
42 |
-
|
|
|
|
|
|
|
43 |
|
44 |
-
|
45 |
|
46 |
-
|
|
|
|
|
|
|
|
|
47 |
|
48 |
-
|
49 |
|
50 |
-
|
|
|
|
|
|
|
|
|
51 |
|
52 |
-
|
53 |
|
54 |
-
|
|
|
|
|
|
|
55 |
|
56 |
-
|
|
|
|
|
|
|
57 |
|
58 |
-
|
|
|
|
|
|
|
59 |
|
60 |
-
|
61 |
|
62 |
-
|
63 |
|
64 |
-
|
|
|
|
|
|
|
65 |
|
66 |
-
|
67 |
|
68 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
69 |
|
70 |
-
|
71 |
-
|
72 |
-
Use the code below to get started with the model.
|
73 |
-
|
74 |
-
[More Information Needed]
|
75 |
-
|
76 |
-
## Training Details
|
77 |
-
|
78 |
-
### Training Data
|
79 |
-
|
80 |
-
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
81 |
-
|
82 |
-
[More Information Needed]
|
83 |
-
|
84 |
-
### Training Procedure
|
85 |
-
|
86 |
-
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
87 |
-
|
88 |
-
#### Preprocessing [optional]
|
89 |
-
|
90 |
-
[More Information Needed]
|
91 |
-
|
92 |
-
|
93 |
-
#### Training Hyperparameters
|
94 |
-
|
95 |
-
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
96 |
-
|
97 |
-
#### Speeds, Sizes, Times [optional]
|
98 |
-
|
99 |
-
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
100 |
-
|
101 |
-
[More Information Needed]
|
102 |
-
|
103 |
-
## Evaluation
|
104 |
-
|
105 |
-
<!-- This section describes the evaluation protocols and provides the results. -->
|
106 |
-
|
107 |
-
### Testing Data, Factors & Metrics
|
108 |
-
|
109 |
-
#### Testing Data
|
110 |
-
|
111 |
-
<!-- This should link to a Dataset Card if possible. -->
|
112 |
-
|
113 |
-
[More Information Needed]
|
114 |
-
|
115 |
-
#### Factors
|
116 |
-
|
117 |
-
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
118 |
-
|
119 |
-
[More Information Needed]
|
120 |
-
|
121 |
-
#### Metrics
|
122 |
-
|
123 |
-
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
124 |
-
|
125 |
-
[More Information Needed]
|
126 |
-
|
127 |
-
### Results
|
128 |
-
|
129 |
-
[More Information Needed]
|
130 |
-
|
131 |
-
#### Summary
|
132 |
-
|
133 |
-
|
134 |
-
|
135 |
-
## Model Examination [optional]
|
136 |
-
|
137 |
-
<!-- Relevant interpretability work for the model goes here -->
|
138 |
-
|
139 |
-
[More Information Needed]
|
140 |
-
|
141 |
-
## Environmental Impact
|
142 |
-
|
143 |
-
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
144 |
-
|
145 |
-
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
146 |
-
|
147 |
-
- **Hardware Type:** [More Information Needed]
|
148 |
-
- **Hours used:** [More Information Needed]
|
149 |
-
- **Cloud Provider:** [More Information Needed]
|
150 |
-
- **Compute Region:** [More Information Needed]
|
151 |
-
- **Carbon Emitted:** [More Information Needed]
|
152 |
-
|
153 |
-
## Technical Specifications [optional]
|
154 |
-
|
155 |
-
### Model Architecture and Objective
|
156 |
-
|
157 |
-
[More Information Needed]
|
158 |
-
|
159 |
-
### Compute Infrastructure
|
160 |
-
|
161 |
-
[More Information Needed]
|
162 |
-
|
163 |
-
#### Hardware
|
164 |
-
|
165 |
-
[More Information Needed]
|
166 |
-
|
167 |
-
#### Software
|
168 |
-
|
169 |
-
[More Information Needed]
|
170 |
-
|
171 |
-
## Citation [optional]
|
172 |
-
|
173 |
-
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
174 |
-
|
175 |
-
**BibTeX:**
|
176 |
-
|
177 |
-
[More Information Needed]
|
178 |
-
|
179 |
-
**APA:**
|
180 |
-
|
181 |
-
[More Information Needed]
|
182 |
-
|
183 |
-
## Glossary [optional]
|
184 |
-
|
185 |
-
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
186 |
-
|
187 |
-
[More Information Needed]
|
188 |
-
|
189 |
-
## More Information [optional]
|
190 |
-
|
191 |
-
[More Information Needed]
|
192 |
-
|
193 |
-
## Model Card Authors [optional]
|
194 |
-
|
195 |
-
[More Information Needed]
|
196 |
-
|
197 |
-
## Model Card Contact
|
198 |
|
199 |
-
|
|
|
1 |
---
|
2 |
+
language: en
|
3 |
+
license: mit
|
4 |
library_name: transformers
|
5 |
+
tags:
|
6 |
+
- text-classification
|
7 |
+
- psychology
|
8 |
+
- abuse-detection
|
9 |
+
- darvo
|
10 |
+
- manipulation-detection
|
11 |
+
- mental-health
|
12 |
+
- relationship-analysis
|
13 |
+
- tether-pro
|
14 |
+
datasets:
|
15 |
+
- custom
|
16 |
+
metrics:
|
17 |
+
- mse
|
18 |
+
- mae
|
19 |
+
- accuracy
|
20 |
+
- auc
|
21 |
+
model-index:
|
22 |
+
- name: tether-darvo-regressor-v1
|
23 |
+
results:
|
24 |
+
- task:
|
25 |
+
type: text-classification
|
26 |
+
name: DARVO Detection
|
27 |
+
metrics:
|
28 |
+
- type: mse
|
29 |
+
value: 0.043
|
30 |
+
- type: mae
|
31 |
+
value: 0.171
|
32 |
+
- type: accuracy
|
33 |
+
value: 0.842
|
34 |
+
- type: auc
|
35 |
+
value: 0.881
|
36 |
---
|
37 |
|
38 |
+
# Tether Pro DARVO Regressor v2
|
39 |
|
40 |
+
## Model Description
|
41 |
|
42 |
+
This model detects DARVO (Deny, Attack, Reverse Victim & Offender) manipulation tactics in text communication. DARVO is a psychological manipulation strategy where an abuser:
|
43 |
|
44 |
+
1. **Denies** the abuse ever happened
|
45 |
+
2. **Attacks** the victim for bringing it up
|
46 |
+
3. **Reverses** the roles to claim they are the victim
|
47 |
|
48 |
+
## Key Features
|
49 |
|
50 |
+
🎯 **Role-Aware Detection**: Distinguishes between genuine accountability and manipulation tactics
|
51 |
+
🔬 **Research-Grade Accuracy**: 84% accuracy with 0.88 AUC
|
52 |
+
⚡ **Real-Time Analysis**: Optimized for fast inference
|
53 |
+
🛡️ **Professional Use**: Designed for therapists, legal professionals, and safety applications
|
54 |
|
55 |
+
## Performance Metrics
|
56 |
|
57 |
+
| Metric | Score |
|
58 |
+
|--------|-------|
|
59 |
+
| **R²** | 0.665 |
|
60 |
+
| **MAE** | 0.171 |
|
61 |
+
| **MSE** | 0.043 |
|
62 |
+
| **Accuracy** | 84.2% |
|
63 |
+
| **AUC** | 88.1% |
|
64 |
|
65 |
+
## Usage
|
|
|
|
|
|
|
|
|
|
|
|
|
66 |
|
67 |
+
```python
|
68 |
+
from transformers import AutoTokenizer, AutoModelForSequenceClassification
|
69 |
+
import torch
|
70 |
|
71 |
+
# Load model and tokenizer
|
72 |
+
tokenizer = AutoTokenizer.from_pretrained("SamanthaStorm/tether-darvo-regressor-v1")
|
73 |
+
model = AutoModelForSequenceClassification.from_pretrained("SamanthaStorm/tether-darvo-regressor-v1")
|
74 |
|
75 |
+
# Example usage
|
76 |
+
text = "You're the one being abusive to me right now"
|
|
|
77 |
|
78 |
+
# Tokenize and predict
|
79 |
+
inputs = tokenizer(text, return_tensors="pt", truncation=True, padding=True)
|
80 |
+
with torch.no_grad():
|
81 |
+
outputs = model(**inputs)
|
82 |
+
darvo_score = outputs.logits.item()
|
83 |
|
84 |
+
print(f"DARVO Score: {darvo_score:.3f}") # Higher scores = more DARVO tactics
|
85 |
+
```
|
86 |
|
87 |
+
## Score Interpretation
|
88 |
|
89 |
+
- **0.0 - 0.3**: Genuine accountability, healthy communication
|
90 |
+
- **0.3 - 0.6**: Some defensive patterns, mild deflection
|
91 |
+
- **0.6 - 0.8**: Moderate DARVO tactics, concerning patterns
|
92 |
+
- **0.8 - 1.0**: Strong DARVO tactics, victim reversal
|
93 |
|
94 |
+
## Example Predictions
|
95 |
|
96 |
+
| Text | DARVO Score | Interpretation |
|
97 |
+
|------|-------------|----------------|
|
98 |
+
| "You're the one being abusive to me right now" | 0.870 | High DARVO - victim reversal |
|
99 |
+
| "I don't remember saying that" | 0.224 | Low DARVO - simple denial |
|
100 |
+
| "I take full responsibility for my actions" | 0.205 | Very low DARVO - accountability |
|
101 |
|
102 |
+
## Training Data
|
103 |
|
104 |
+
Trained on 285 carefully curated examples including:
|
105 |
+
- **High DARVO**: Explicit victim reversal tactics
|
106 |
+
- **Medium DARVO**: Deflection and minimization patterns
|
107 |
+
- **Low DARVO**: Genuine accountability and healthy communication
|
108 |
+
- **Contrast Examples**: Non-apologies vs real apologies
|
109 |
|
110 |
+
## Applications
|
111 |
|
112 |
+
### 🏥 Clinical Therapy
|
113 |
+
- Help therapists identify manipulation patterns in client relationships
|
114 |
+
- Assist in couples counseling to recognize unhealthy dynamics
|
115 |
+
- Support trauma therapy by validating victim experiences
|
116 |
|
117 |
+
### ⚖️ Legal Documentation
|
118 |
+
- Analyze communication patterns in domestic violence cases
|
119 |
+
- Provide objective evidence of psychological manipulation
|
120 |
+
- Support legal professionals in building abuse cases
|
121 |
|
122 |
+
### 🏢 Workplace Safety
|
123 |
+
- Identify harassment patterns in workplace communications
|
124 |
+
- Support HR investigations with objective analysis
|
125 |
+
- Create safer work environments through pattern recognition
|
126 |
|
127 |
+
## Ethical Considerations
|
128 |
|
129 |
+
⚠️ **Important**: This model is designed to assist professionals and should not be used as the sole basis for serious decisions about relationships or safety.
|
130 |
|
131 |
+
- **Professional Use**: Best used by trained therapists, counselors, and legal professionals
|
132 |
+
- **Context Matters**: Consider cultural, situational, and individual factors
|
133 |
+
- **Not Diagnostic**: Does not diagnose psychological conditions
|
134 |
+
- **Privacy**: Ensure consent when analyzing personal communications
|
135 |
|
136 |
+
## Technical Details
|
137 |
|
138 |
+
- **Base Model**: DistilBERT (distilbert-base-uncased)
|
139 |
+
- **Architecture**: Custom regression head with 4-layer neural network
|
140 |
+
- **Training**: 8 epochs with cosine learning rate scheduling
|
141 |
+
- **Optimization**: Mixed precision training (FP16)
|
142 |
+
- **Max Length**: 256 tokens for efficiency
|
143 |
+
|
144 |
+
## Model Architecture
|
145 |
+
|
146 |
+
```
|
147 |
+
DistilBERT Base
|
148 |
+
↓
|
149 |
+
Linear(768 → 768) + GELU + Dropout
|
150 |
+
↓
|
151 |
+
Linear(768 → 384) + GELU + Dropout
|
152 |
+
↓
|
153 |
+
Linear(384 → 192) + GELU + Dropout
|
154 |
+
↓
|
155 |
+
Linear(192 → 1) + Sigmoid
|
156 |
+
↓
|
157 |
+
DARVO Score (0.0 - 1.0)
|
158 |
+
```
|
159 |
+
|
160 |
+
## Version History
|
161 |
+
|
162 |
+
### v2 (Current)
|
163 |
+
- ✅ Enhanced training dataset (285 examples)
|
164 |
+
- ✅ Improved architecture with deeper regression head
|
165 |
+
- ✅ Better score calibration for accountability detection
|
166 |
+
- ✅ Added contrast examples (fake vs real apologies)
|
167 |
+
- ✅ 84% accuracy (up from 40%)
|
168 |
+
|
169 |
+
### v1 (Previous)
|
170 |
+
- Basic DARVO detection capability
|
171 |
+
- Limited training data
|
172 |
+
- Lower accuracy performance
|
173 |
+
|
174 |
+
## Citation
|
175 |
+
|
176 |
+
If you use this model in research or professional practice, please cite:
|
177 |
+
|
178 |
+
```bibtex
|
179 |
+
@misc{tether-darvo-regressor-v1,
|
180 |
+
title={Tether Pro DARVO Regressor: Role-Aware Detection of Manipulation Tactics},
|
181 |
+
author={SamanthaStorm},
|
182 |
+
year={2024},
|
183 |
+
howpublished={\url{https://huggingface.co/SamanthaStorm/tether-darvo-regressor-v1}},
|
184 |
+
}
|
185 |
+
```
|
186 |
+
|
187 |
+
## Contact & Support
|
188 |
+
|
189 |
+
For questions about integration, licensing, or professional applications:
|
190 |
+
- 📧 Enterprise: [email protected]
|
191 |
+
- 🌐 Documentation: docs.tether.ai
|
192 |
+
- 📅 Consultation: calendly.com/tether-pro
|
193 |
+
|
194 |
+
## Related Models
|
195 |
+
|
196 |
+
Part of the **Tether Pro AI Suite**:
|
197 |
+
- 🛡️ **Boundary Health Detector**: `SamanthaStorm/healthy-boundary-predictor`
|
198 |
+
- 🎯 **Abuse Pattern Detector**: `SamanthaStorm/tether-multilabel-v6`
|
199 |
+
- 🎭 **Sentiment Analyzer**: `SamanthaStorm/tether-sentiment-v3`
|
200 |
+
- 🧩 **Fallacy Detector**: `SamanthaStorm/fallacy-detector` (coming soon)
|
201 |
+
- 🎯 **Intent Classifier**: `SamanthaStorm/intent-detector` (coming soon)
|
202 |
|
203 |
+
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
204 |
|
205 |
+
*Built with ❤️ for safer communication analysis*
|
config.json
CHANGED
@@ -1,33 +1,30 @@
|
|
1 |
{
|
|
|
2 |
"architectures": [
|
3 |
-
"
|
4 |
],
|
5 |
-
"
|
6 |
-
"
|
7 |
-
"
|
8 |
-
"
|
9 |
-
"hidden_act": "gelu",
|
10 |
-
"hidden_dropout_prob": 0.1,
|
11 |
-
"hidden_size": 768,
|
12 |
"id2label": {
|
13 |
"0": "LABEL_0"
|
14 |
},
|
15 |
"initializer_range": 0.02,
|
16 |
-
"intermediate_size": 3072,
|
17 |
"label2id": {
|
18 |
"LABEL_0": 0
|
19 |
},
|
20 |
-
"
|
21 |
-
"
|
22 |
-
"
|
23 |
-
"
|
24 |
-
"
|
25 |
-
"pad_token_id": 1,
|
26 |
-
"position_embedding_type": "absolute",
|
27 |
"problem_type": "regression",
|
|
|
|
|
|
|
|
|
28 |
"torch_dtype": "float32",
|
29 |
-
"transformers_version": "4.
|
30 |
-
"
|
31 |
-
"use_cache": true,
|
32 |
-
"vocab_size": 50265
|
33 |
}
|
|
|
1 |
{
|
2 |
+
"activation": "gelu",
|
3 |
"architectures": [
|
4 |
+
"DistilBertForSequenceClassification"
|
5 |
],
|
6 |
+
"attention_dropout": 0.1,
|
7 |
+
"dim": 768,
|
8 |
+
"dropout": 0.1,
|
9 |
+
"hidden_dim": 3072,
|
|
|
|
|
|
|
10 |
"id2label": {
|
11 |
"0": "LABEL_0"
|
12 |
},
|
13 |
"initializer_range": 0.02,
|
|
|
14 |
"label2id": {
|
15 |
"LABEL_0": 0
|
16 |
},
|
17 |
+
"max_position_embeddings": 512,
|
18 |
+
"model_type": "distilbert",
|
19 |
+
"n_heads": 12,
|
20 |
+
"n_layers": 6,
|
21 |
+
"pad_token_id": 0,
|
|
|
|
|
22 |
"problem_type": "regression",
|
23 |
+
"qa_dropout": 0.1,
|
24 |
+
"seq_classif_dropout": 0.2,
|
25 |
+
"sinusoidal_pos_embds": false,
|
26 |
+
"tie_weights_": true,
|
27 |
"torch_dtype": "float32",
|
28 |
+
"transformers_version": "4.53.0",
|
29 |
+
"vocab_size": 30522
|
|
|
|
|
30 |
}
|
model.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:6a4e929f41e35442855d7ded918ee23f9d68a7e0fb3c7675df0728d5421c0e86
|
3 |
+
size 267829484
|
model_info.json
ADDED
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"model_type": "regression",
|
3 |
+
"task": "darvo-detection",
|
4 |
+
"version": "2.0",
|
5 |
+
"performance": {
|
6 |
+
"mse": 0.043,
|
7 |
+
"mae": 0.171,
|
8 |
+
"accuracy": 0.842,
|
9 |
+
"auc": 0.881,
|
10 |
+
"r_squared": 0.665
|
11 |
+
},
|
12 |
+
"training_examples": 285,
|
13 |
+
"architecture": "distilbert-base-uncased + custom regression head"
|
14 |
+
}
|
special_tokens_map.json
CHANGED
@@ -1,51 +1,7 @@
|
|
1 |
{
|
2 |
-
"
|
3 |
-
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
"single_word": false
|
8 |
-
},
|
9 |
-
"cls_token": {
|
10 |
-
"content": "<s>",
|
11 |
-
"lstrip": false,
|
12 |
-
"normalized": true,
|
13 |
-
"rstrip": false,
|
14 |
-
"single_word": false
|
15 |
-
},
|
16 |
-
"eos_token": {
|
17 |
-
"content": "</s>",
|
18 |
-
"lstrip": false,
|
19 |
-
"normalized": true,
|
20 |
-
"rstrip": false,
|
21 |
-
"single_word": false
|
22 |
-
},
|
23 |
-
"mask_token": {
|
24 |
-
"content": "<mask>",
|
25 |
-
"lstrip": true,
|
26 |
-
"normalized": false,
|
27 |
-
"rstrip": false,
|
28 |
-
"single_word": false
|
29 |
-
},
|
30 |
-
"pad_token": {
|
31 |
-
"content": "<pad>",
|
32 |
-
"lstrip": false,
|
33 |
-
"normalized": true,
|
34 |
-
"rstrip": false,
|
35 |
-
"single_word": false
|
36 |
-
},
|
37 |
-
"sep_token": {
|
38 |
-
"content": "</s>",
|
39 |
-
"lstrip": false,
|
40 |
-
"normalized": true,
|
41 |
-
"rstrip": false,
|
42 |
-
"single_word": false
|
43 |
-
},
|
44 |
-
"unk_token": {
|
45 |
-
"content": "<unk>",
|
46 |
-
"lstrip": false,
|
47 |
-
"normalized": true,
|
48 |
-
"rstrip": false,
|
49 |
-
"single_word": false
|
50 |
-
}
|
51 |
}
|
|
|
1 |
{
|
2 |
+
"cls_token": "[CLS]",
|
3 |
+
"mask_token": "[MASK]",
|
4 |
+
"pad_token": "[PAD]",
|
5 |
+
"sep_token": "[SEP]",
|
6 |
+
"unk_token": "[UNK]"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
7 |
}
|
tokenizer.json
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_config.json
CHANGED
@@ -1,65 +1,56 @@
|
|
1 |
{
|
2 |
-
"add_prefix_space": false,
|
3 |
"added_tokens_decoder": {
|
4 |
"0": {
|
5 |
-
"content": "
|
6 |
"lstrip": false,
|
7 |
-
"normalized":
|
8 |
"rstrip": false,
|
9 |
"single_word": false,
|
10 |
"special": true
|
11 |
},
|
12 |
-
"
|
13 |
-
"content": "
|
14 |
"lstrip": false,
|
15 |
-
"normalized":
|
16 |
"rstrip": false,
|
17 |
"single_word": false,
|
18 |
"special": true
|
19 |
},
|
20 |
-
"
|
21 |
-
"content": "
|
22 |
"lstrip": false,
|
23 |
-
"normalized":
|
24 |
"rstrip": false,
|
25 |
"single_word": false,
|
26 |
"special": true
|
27 |
},
|
28 |
-
"
|
29 |
-
"content": "
|
30 |
"lstrip": false,
|
31 |
-
"normalized":
|
32 |
"rstrip": false,
|
33 |
"single_word": false,
|
34 |
"special": true
|
35 |
},
|
36 |
-
"
|
37 |
-
"content": "
|
38 |
-
"lstrip":
|
39 |
"normalized": false,
|
40 |
"rstrip": false,
|
41 |
"single_word": false,
|
42 |
"special": true
|
43 |
}
|
44 |
},
|
45 |
-
"bos_token": "<s>",
|
46 |
"clean_up_tokenization_spaces": false,
|
47 |
-
"cls_token": "
|
48 |
-
"
|
49 |
-
"errors": "replace",
|
50 |
"extra_special_tokens": {},
|
51 |
-
"mask_token": "
|
52 |
-
"max_length": 256,
|
53 |
"model_max_length": 512,
|
54 |
-
"
|
55 |
-
"
|
56 |
-
"
|
57 |
-
"
|
58 |
-
"
|
59 |
-
"
|
60 |
-
"tokenizer_class": "RobertaTokenizer",
|
61 |
-
"trim_offsets": true,
|
62 |
-
"truncation_side": "right",
|
63 |
-
"truncation_strategy": "longest_first",
|
64 |
-
"unk_token": "<unk>"
|
65 |
}
|
|
|
1 |
{
|
|
|
2 |
"added_tokens_decoder": {
|
3 |
"0": {
|
4 |
+
"content": "[PAD]",
|
5 |
"lstrip": false,
|
6 |
+
"normalized": false,
|
7 |
"rstrip": false,
|
8 |
"single_word": false,
|
9 |
"special": true
|
10 |
},
|
11 |
+
"100": {
|
12 |
+
"content": "[UNK]",
|
13 |
"lstrip": false,
|
14 |
+
"normalized": false,
|
15 |
"rstrip": false,
|
16 |
"single_word": false,
|
17 |
"special": true
|
18 |
},
|
19 |
+
"101": {
|
20 |
+
"content": "[CLS]",
|
21 |
"lstrip": false,
|
22 |
+
"normalized": false,
|
23 |
"rstrip": false,
|
24 |
"single_word": false,
|
25 |
"special": true
|
26 |
},
|
27 |
+
"102": {
|
28 |
+
"content": "[SEP]",
|
29 |
"lstrip": false,
|
30 |
+
"normalized": false,
|
31 |
"rstrip": false,
|
32 |
"single_word": false,
|
33 |
"special": true
|
34 |
},
|
35 |
+
"103": {
|
36 |
+
"content": "[MASK]",
|
37 |
+
"lstrip": false,
|
38 |
"normalized": false,
|
39 |
"rstrip": false,
|
40 |
"single_word": false,
|
41 |
"special": true
|
42 |
}
|
43 |
},
|
|
|
44 |
"clean_up_tokenization_spaces": false,
|
45 |
+
"cls_token": "[CLS]",
|
46 |
+
"do_lower_case": true,
|
|
|
47 |
"extra_special_tokens": {},
|
48 |
+
"mask_token": "[MASK]",
|
|
|
49 |
"model_max_length": 512,
|
50 |
+
"pad_token": "[PAD]",
|
51 |
+
"sep_token": "[SEP]",
|
52 |
+
"strip_accents": null,
|
53 |
+
"tokenize_chinese_chars": true,
|
54 |
+
"tokenizer_class": "DistilBertTokenizer",
|
55 |
+
"unk_token": "[UNK]"
|
|
|
|
|
|
|
|
|
|
|
56 |
}
|
vocab.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|