v1.0.3 (#7)
Browse files- v1.0.3 (1cd199cf50b8d3b1d42cbd1a06ede32deecfe05f)
- config.json +2 -6
- configuration_internlm.py +7 -8
- generation_config.json +1 -1
- modeling_internlm.py +45 -116
- pytorch_model-00001-of-00008.bin +2 -2
- pytorch_model-00002-of-00008.bin +2 -2
- pytorch_model-00003-of-00008.bin +2 -2
- pytorch_model-00004-of-00008.bin +2 -2
- pytorch_model-00005-of-00008.bin +2 -2
- pytorch_model-00006-of-00008.bin +2 -2
- pytorch_model-00007-of-00008.bin +2 -2
- pytorch_model-00008-of-00008.bin +1 -1
config.json
CHANGED
@@ -22,11 +22,7 @@
|
|
22 |
"rms_norm_eps": 1e-06,
|
23 |
"tie_word_embeddings": false,
|
24 |
"torch_dtype": "float16",
|
25 |
-
"transformers_version": "4.
|
26 |
"use_cache": true,
|
27 |
-
"vocab_size": 103168
|
28 |
-
"rotary": {
|
29 |
-
"base": 10000,
|
30 |
-
"type": "dynamic"
|
31 |
-
}
|
32 |
}
|
|
|
22 |
"rms_norm_eps": 1e-06,
|
23 |
"tie_word_embeddings": false,
|
24 |
"torch_dtype": "float16",
|
25 |
+
"transformers_version": "4.33.2",
|
26 |
"use_cache": true,
|
27 |
+
"vocab_size": 103168
|
|
|
|
|
|
|
|
|
28 |
}
|
configuration_internlm.py
CHANGED
@@ -19,8 +19,9 @@
|
|
19 |
# limitations under the License.
|
20 |
""" InternLM model configuration"""
|
21 |
|
22 |
-
from transformers.configuration_utils import PretrainedConfig
|
23 |
from transformers.utils import logging
|
|
|
|
|
24 |
|
25 |
logger = logging.get_logger(__name__)
|
26 |
|
@@ -29,9 +30,9 @@ INTERNLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
|
|
29 |
|
30 |
class InternLMConfig(PretrainedConfig):
|
31 |
r"""
|
32 |
-
This is the configuration class to store the configuration of a [`InternLMModel`]. It is used to instantiate
|
33 |
-
|
34 |
-
|
35 |
|
36 |
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
37 |
documentation from [`PretrainedConfig`] for more information.
|
@@ -80,7 +81,7 @@ class InternLMConfig(PretrainedConfig):
|
|
80 |
model_type = "internlm"
|
81 |
_auto_class = "AutoConfig"
|
82 |
|
83 |
-
def __init__(
|
84 |
self,
|
85 |
vocab_size=103168,
|
86 |
hidden_size=4096,
|
@@ -97,7 +98,6 @@ class InternLMConfig(PretrainedConfig):
|
|
97 |
eos_token_id=2,
|
98 |
tie_word_embeddings=False,
|
99 |
bias=True,
|
100 |
-
rotary={"base": 10000, "type": "dynamic"}, # pylint: disable=W0102
|
101 |
**kwargs,
|
102 |
):
|
103 |
self.vocab_size = vocab_size
|
@@ -111,11 +111,10 @@ class InternLMConfig(PretrainedConfig):
|
|
111 |
self.rms_norm_eps = rms_norm_eps
|
112 |
self.use_cache = use_cache
|
113 |
self.bias = bias
|
114 |
-
self.rotary = rotary
|
115 |
super().__init__(
|
116 |
pad_token_id=pad_token_id,
|
117 |
bos_token_id=bos_token_id,
|
118 |
eos_token_id=eos_token_id,
|
119 |
tie_word_embeddings=tie_word_embeddings,
|
120 |
**kwargs,
|
121 |
-
)
|
|
|
19 |
# limitations under the License.
|
20 |
""" InternLM model configuration"""
|
21 |
|
|
|
22 |
from transformers.utils import logging
|
23 |
+
from transformers.configuration_utils import PretrainedConfig
|
24 |
+
|
25 |
|
26 |
logger = logging.get_logger(__name__)
|
27 |
|
|
|
30 |
|
31 |
class InternLMConfig(PretrainedConfig):
|
32 |
r"""
|
33 |
+
This is the configuration class to store the configuration of a [`InternLMModel`]. It is used to instantiate an InternLM
|
34 |
+
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
|
35 |
+
defaults will yield a similar configuration to that of the InternLM-7B.
|
36 |
|
37 |
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
38 |
documentation from [`PretrainedConfig`] for more information.
|
|
|
81 |
model_type = "internlm"
|
82 |
_auto_class = "AutoConfig"
|
83 |
|
84 |
+
def __init__(
|
85 |
self,
|
86 |
vocab_size=103168,
|
87 |
hidden_size=4096,
|
|
|
98 |
eos_token_id=2,
|
99 |
tie_word_embeddings=False,
|
100 |
bias=True,
|
|
|
101 |
**kwargs,
|
102 |
):
|
103 |
self.vocab_size = vocab_size
|
|
|
111 |
self.rms_norm_eps = rms_norm_eps
|
112 |
self.use_cache = use_cache
|
113 |
self.bias = bias
|
|
|
114 |
super().__init__(
|
115 |
pad_token_id=pad_token_id,
|
116 |
bos_token_id=bos_token_id,
|
117 |
eos_token_id=eos_token_id,
|
118 |
tie_word_embeddings=tie_word_embeddings,
|
119 |
**kwargs,
|
120 |
+
)
|
generation_config.json
CHANGED
@@ -3,5 +3,5 @@
|
|
3 |
"bos_token_id": 1,
|
4 |
"eos_token_id": 2,
|
5 |
"pad_token_id": 2,
|
6 |
-
"transformers_version": "4.
|
7 |
}
|
|
|
3 |
"bos_token_id": 1,
|
4 |
"eos_token_id": 2,
|
5 |
"pad_token_id": 2,
|
6 |
+
"transformers_version": "4.33.2"
|
7 |
}
|
modeling_internlm.py
CHANGED
@@ -19,31 +19,31 @@
|
|
19 |
# limitations under the License.
|
20 |
""" PyTorch InternLM model."""
|
21 |
import math
|
22 |
-
import queue
|
23 |
-
import threading
|
24 |
from typing import List, Optional, Tuple, Union
|
|
|
25 |
|
26 |
import torch
|
27 |
import torch.utils.checkpoint
|
28 |
from torch import nn
|
29 |
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
|
|
|
30 |
from transformers.activations import ACT2FN
|
31 |
-
from transformers.generation.streamers import BaseStreamer
|
32 |
from transformers.modeling_outputs import (
|
33 |
BaseModelOutputWithPast,
|
34 |
CausalLMOutputWithPast,
|
35 |
SequenceClassifierOutputWithPast,
|
36 |
)
|
37 |
from transformers.modeling_utils import PreTrainedModel
|
|
|
38 |
from transformers.utils import (
|
39 |
add_start_docstrings,
|
40 |
add_start_docstrings_to_model_forward,
|
41 |
logging,
|
42 |
replace_return_docstrings,
|
43 |
)
|
44 |
-
|
45 |
from .configuration_internlm import InternLMConfig
|
46 |
|
|
|
47 |
logger = logging.get_logger(__name__)
|
48 |
|
49 |
_CONFIG_FOR_DOC = "InternLMConfig"
|
@@ -83,8 +83,6 @@ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int]
|
|
83 |
|
84 |
|
85 |
class InternLMRMSNorm(nn.Module):
|
86 |
-
"""RMSNorm implemention."""
|
87 |
-
|
88 |
def __init__(self, hidden_size, eps=1e-6):
|
89 |
"""
|
90 |
InternLMRMSNorm is equivalent to T5LayerNorm
|
@@ -105,18 +103,10 @@ class InternLMRMSNorm(nn.Module):
|
|
105 |
|
106 |
|
107 |
class InternLMRotaryEmbedding(torch.nn.Module):
|
108 |
-
"""Implement InternLM's rotary embedding.
|
109 |
-
|
110 |
-
Args:
|
111 |
-
dim (int): Characteristic dimension of each self-attentional head.
|
112 |
-
max_position_embeddings (int, optional): Model's training length. Defaults to 2048.
|
113 |
-
base (int, optional): The rotation position encodes the rotation Angle base number. Defaults to 10000.
|
114 |
-
device (Any, optional): Running device. Defaults to None.
|
115 |
-
"""
|
116 |
def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
|
117 |
super().__init__()
|
118 |
inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim))
|
119 |
-
self.register_buffer("inv_freq", inv_freq
|
120 |
|
121 |
# Build here to make `torch.jit.trace` work.
|
122 |
self.max_seq_len_cached = max_position_embeddings
|
@@ -144,66 +134,6 @@ class InternLMRotaryEmbedding(torch.nn.Module):
|
|
144 |
)
|
145 |
|
146 |
|
147 |
-
class InternLMDynamicNTKScalingRotaryEmbedding(torch.nn.Module):
|
148 |
-
"""Implement InternLM's DyanmicNTK extrapolation method, thereby broadening the model support context to 16K.
|
149 |
-
|
150 |
-
Args:
|
151 |
-
dim (int): Characteristic dimension of each self-attentional head.
|
152 |
-
max_position_embeddings (int, optional): Model's training length. Defaults to 2048.
|
153 |
-
base (int, optional): The rotation position encodes the rotation Angle base number. Defaults to 10000.
|
154 |
-
device (Any, optional): Running device. Defaults to None.
|
155 |
-
scaling_factor (float, optional): NTK method extrapolation coefficient. Defaults to 1.0.
|
156 |
-
"""
|
157 |
-
|
158 |
-
def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
|
159 |
-
super().__init__()
|
160 |
-
inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim))
|
161 |
-
self.register_buffer("inv_freq", inv_freq)
|
162 |
-
self.dim = dim
|
163 |
-
self.base = base
|
164 |
-
self.scaling_factor = scaling_factor
|
165 |
-
|
166 |
-
# Build here to make `torch.jit.trace` work.
|
167 |
-
self.max_position_embeddings = max_position_embeddings
|
168 |
-
self.max_seq_len_cached = max_position_embeddings
|
169 |
-
t = torch.arange(self.max_seq_len_cached, device=self.inv_freq.device, dtype=self.inv_freq.dtype)
|
170 |
-
freqs = torch.einsum("i,j->ij", t, self.inv_freq)
|
171 |
-
# Different from paper, but it uses a different permutation in order to obtain the same calculation
|
172 |
-
emb = torch.cat((freqs, freqs), dim=-1)
|
173 |
-
self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False)
|
174 |
-
self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False)
|
175 |
-
|
176 |
-
def _update_cached(self, x, seq_len=None):
|
177 |
-
self.max_seq_len_cached = max(seq_len, self.max_position_embeddings)
|
178 |
-
if seq_len > self.max_position_embeddings:
|
179 |
-
base = self.base * (
|
180 |
-
(self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)
|
181 |
-
) ** (self.dim / (self.dim - 2))
|
182 |
-
inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(x.device) / self.dim))
|
183 |
-
else:
|
184 |
-
inv_freq = self.inv_freq
|
185 |
-
t = torch.arange(self.max_seq_len_cached, device=inv_freq.device, dtype=inv_freq.dtype)
|
186 |
-
freqs = torch.einsum("i,j->ij", t, inv_freq)
|
187 |
-
emb = torch.cat((freqs, freqs), dim=-1)
|
188 |
-
self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False)
|
189 |
-
self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False)
|
190 |
-
|
191 |
-
def forward(self, x, seq_len=None):
|
192 |
-
# x: [bs, num_attention_heads, seq_len, head_size]
|
193 |
-
# This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case.
|
194 |
-
if seq_len <= self.max_position_embeddings:
|
195 |
-
# Reset the tables if the sequence length has changed,
|
196 |
-
if self.max_seq_len_cached > self.max_position_embeddings:
|
197 |
-
self._update_cached(x, seq_len)
|
198 |
-
else:
|
199 |
-
self._update_cached(x, seq_len)
|
200 |
-
|
201 |
-
return (
|
202 |
-
self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
|
203 |
-
self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
|
204 |
-
)
|
205 |
-
|
206 |
-
|
207 |
def rotate_half(x):
|
208 |
"""Rotates half the hidden dims of the input."""
|
209 |
x1 = x[..., : x.shape[-1] // 2]
|
@@ -215,18 +145,10 @@ def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
|
|
215 |
# The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
|
216 |
cos = cos.squeeze(1).squeeze(0) # [seq_len, dim]
|
217 |
sin = sin.squeeze(1).squeeze(0) # [seq_len, dim]
|
218 |
-
cos = cos.unsqueeze(
|
219 |
-
sin = sin.unsqueeze(
|
220 |
-
|
221 |
-
|
222 |
-
else:
|
223 |
-
q_embed = (q * cos) + (rotate_half(q) * sin)
|
224 |
-
|
225 |
-
if k.size(2) == 1:
|
226 |
-
k_embed = (k * cos[:, :, -1, :]) + (rotate_half(k) * sin[:, :, -1, :])
|
227 |
-
else:
|
228 |
-
k_embed = (k * cos) + (rotate_half(k) * sin)
|
229 |
-
|
230 |
return q_embed, k_embed
|
231 |
|
232 |
|
@@ -267,25 +189,7 @@ class InternLMAttention(nn.Module):
|
|
267 |
self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias)
|
268 |
self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias)
|
269 |
self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias)
|
270 |
-
self.rotary_emb = self.
|
271 |
-
|
272 |
-
def _init_rope(self):
|
273 |
-
if self.config.rotary["type"] == "origin":
|
274 |
-
self.rotary_emb = InternLMRotaryEmbedding(
|
275 |
-
self.head_dim,
|
276 |
-
max_position_embeddings=self.max_position_embeddings,
|
277 |
-
base=self.config.rotary["base"],
|
278 |
-
)
|
279 |
-
elif self.config.rotary["type"] == "dynamic":
|
280 |
-
self.rotary_emb = InternLMDynamicNTKScalingRotaryEmbedding(
|
281 |
-
self.head_dim,
|
282 |
-
max_position_embeddings=self.max_position_embeddings,
|
283 |
-
base=self.config.rotary["base"],
|
284 |
-
scaling_factor=self.config.rotary.get("scaling_factor", 1.0),
|
285 |
-
)
|
286 |
-
else:
|
287 |
-
raise ValueError("Currently we only support rotary embedding's type being one of ('origin', 'dynamic').")
|
288 |
-
return self.rotary_emb
|
289 |
|
290 |
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
|
291 |
return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
|
@@ -305,18 +209,20 @@ class InternLMAttention(nn.Module):
|
|
305 |
key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
306 |
value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
307 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
308 |
if past_key_value is not None:
|
309 |
# reuse k, v, self_attention
|
310 |
key_states = torch.cat([past_key_value[0], key_states], dim=2)
|
311 |
value_states = torch.cat([past_key_value[1], value_states], dim=2)
|
312 |
|
313 |
-
# print(use_cache)
|
314 |
past_key_value = (key_states, value_states) if use_cache else None
|
315 |
|
316 |
-
kv_seq_len = key_states.shape[-2]
|
317 |
-
cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
|
318 |
-
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
|
319 |
-
|
320 |
attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
|
321 |
|
322 |
if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
|
@@ -426,9 +332,11 @@ INTERNLM_START_DOCSTRING = r"""
|
|
426 |
This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
|
427 |
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
|
428 |
etc.)
|
|
|
429 |
This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
|
430 |
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
|
431 |
and behavior.
|
|
|
432 |
Parameters:
|
433 |
config ([`InternLMConfig`]):
|
434 |
Model configuration class with all the parameters of the model. Initializing with a config file does not
|
@@ -469,34 +377,44 @@ INTERNLM_INPUTS_DOCSTRING = r"""
|
|
469 |
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
|
470 |
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
|
471 |
it.
|
|
|
472 |
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
473 |
[`PreTrainedTokenizer.__call__`] for details.
|
|
|
474 |
[What are input IDs?](../glossary#input-ids)
|
475 |
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
|
476 |
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
|
|
477 |
- 1 for tokens that are **not masked**,
|
478 |
- 0 for tokens that are **masked**.
|
|
|
479 |
[What are attention masks?](../glossary#attention-mask)
|
|
|
480 |
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
481 |
[`PreTrainedTokenizer.__call__`] for details.
|
|
|
482 |
If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
|
483 |
`past_key_values`).
|
|
|
484 |
If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
|
485 |
and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
|
486 |
information on the default strategy.
|
|
|
487 |
- 1 indicates the head is **not masked**,
|
488 |
- 0 indicates the head is **masked**.
|
489 |
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
490 |
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
|
491 |
config.n_positions - 1]`.
|
|
|
492 |
[What are position IDs?](../glossary#position-ids)
|
493 |
-
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or
|
494 |
-
when `config.use_cache=True`):
|
495 |
Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
|
496 |
`(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
|
497 |
`(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
|
|
|
498 |
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
|
499 |
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
|
|
|
500 |
If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
|
501 |
don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
|
502 |
`decoder_input_ids` of shape `(batch_size, sequence_length)`.
|
@@ -525,6 +443,7 @@ INTERNLM_INPUTS_DOCSTRING = r"""
|
|
525 |
class InternLMModel(InternLMPreTrainedModel):
|
526 |
"""
|
527 |
Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLMDecoderLayer`]
|
|
|
528 |
Args:
|
529 |
config: InternLMConfig
|
530 |
"""
|
@@ -754,14 +673,20 @@ class InternLMForCausalLM(InternLMPreTrainedModel):
|
|
754 |
Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
|
755 |
config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
|
756 |
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
|
|
|
757 |
Returns:
|
|
|
758 |
Example:
|
|
|
759 |
```python
|
760 |
>>> from transformers import AutoTokenizer, InternLMForCausalLM
|
|
|
761 |
>>> model = InternLMForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
|
762 |
>>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
|
|
|
763 |
>>> prompt = "Hey, are you consciours? Can you talk to me?"
|
764 |
>>> inputs = tokenizer(prompt, return_tensors="pt")
|
|
|
765 |
>>> # Generate
|
766 |
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
|
767 |
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
|
@@ -855,7 +780,9 @@ class InternLMForCausalLM(InternLMPreTrainedModel):
|
|
855 |
def build_inputs(self, tokenizer, query: str, history: List[Tuple[str, str]] = []):
|
856 |
prompt = ""
|
857 |
for record in history:
|
858 |
-
prompt += f"""
|
|
|
|
|
859 |
prompt += f"""<|User|>:{query}<eoh>\n<|Bot|>:"""
|
860 |
return tokenizer([prompt], return_tensors="pt")
|
861 |
|
@@ -959,7 +886,7 @@ class InternLMForCausalLM(InternLMPreTrainedModel):
|
|
959 |
producer.start()
|
960 |
while True:
|
961 |
res = response_queue.get()
|
962 |
-
if res is None:
|
963 |
return
|
964 |
yield res
|
965 |
|
@@ -969,8 +896,10 @@ class InternLMForCausalLM(InternLMPreTrainedModel):
|
|
969 |
@add_start_docstrings(
|
970 |
"""
|
971 |
The InternLM Model transformer with a sequence classification head on top (linear layer).
|
|
|
972 |
[`InternLMForSequenceClassification`] uses the last token in order to do the classification, as other causal models
|
973 |
(e.g. GPT-2) do.
|
|
|
974 |
Since it does classification on the last token, it requires to know the position of the last token. If a
|
975 |
`pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
|
976 |
no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
|
@@ -1083,4 +1012,4 @@ class InternLMForSequenceClassification(InternLMPreTrainedModel):
|
|
1083 |
past_key_values=transformer_outputs.past_key_values,
|
1084 |
hidden_states=transformer_outputs.hidden_states,
|
1085 |
attentions=transformer_outputs.attentions,
|
1086 |
-
)
|
|
|
19 |
# limitations under the License.
|
20 |
""" PyTorch InternLM model."""
|
21 |
import math
|
|
|
|
|
22 |
from typing import List, Optional, Tuple, Union
|
23 |
+
import threading, queue
|
24 |
|
25 |
import torch
|
26 |
import torch.utils.checkpoint
|
27 |
from torch import nn
|
28 |
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
|
29 |
+
|
30 |
from transformers.activations import ACT2FN
|
|
|
31 |
from transformers.modeling_outputs import (
|
32 |
BaseModelOutputWithPast,
|
33 |
CausalLMOutputWithPast,
|
34 |
SequenceClassifierOutputWithPast,
|
35 |
)
|
36 |
from transformers.modeling_utils import PreTrainedModel
|
37 |
+
from transformers.generation.streamers import BaseStreamer
|
38 |
from transformers.utils import (
|
39 |
add_start_docstrings,
|
40 |
add_start_docstrings_to_model_forward,
|
41 |
logging,
|
42 |
replace_return_docstrings,
|
43 |
)
|
|
|
44 |
from .configuration_internlm import InternLMConfig
|
45 |
|
46 |
+
|
47 |
logger = logging.get_logger(__name__)
|
48 |
|
49 |
_CONFIG_FOR_DOC = "InternLMConfig"
|
|
|
83 |
|
84 |
|
85 |
class InternLMRMSNorm(nn.Module):
|
|
|
|
|
86 |
def __init__(self, hidden_size, eps=1e-6):
|
87 |
"""
|
88 |
InternLMRMSNorm is equivalent to T5LayerNorm
|
|
|
103 |
|
104 |
|
105 |
class InternLMRotaryEmbedding(torch.nn.Module):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
106 |
def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
|
107 |
super().__init__()
|
108 |
inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim))
|
109 |
+
self.register_buffer("inv_freq", inv_freq)
|
110 |
|
111 |
# Build here to make `torch.jit.trace` work.
|
112 |
self.max_seq_len_cached = max_position_embeddings
|
|
|
134 |
)
|
135 |
|
136 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
137 |
def rotate_half(x):
|
138 |
"""Rotates half the hidden dims of the input."""
|
139 |
x1 = x[..., : x.shape[-1] // 2]
|
|
|
145 |
# The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
|
146 |
cos = cos.squeeze(1).squeeze(0) # [seq_len, dim]
|
147 |
sin = sin.squeeze(1).squeeze(0) # [seq_len, dim]
|
148 |
+
cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim]
|
149 |
+
sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim]
|
150 |
+
q_embed = (q * cos) + (rotate_half(q) * sin)
|
151 |
+
k_embed = (k * cos) + (rotate_half(k) * sin)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
152 |
return q_embed, k_embed
|
153 |
|
154 |
|
|
|
189 |
self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias)
|
190 |
self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias)
|
191 |
self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias)
|
192 |
+
self.rotary_emb = InternLMRotaryEmbedding(self.head_dim, max_position_embeddings=self.max_position_embeddings)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
193 |
|
194 |
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
|
195 |
return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
|
|
|
209 |
key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
210 |
value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
211 |
|
212 |
+
kv_seq_len = key_states.shape[-2]
|
213 |
+
if past_key_value is not None:
|
214 |
+
kv_seq_len += past_key_value[0].shape[-2]
|
215 |
+
cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
|
216 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
|
217 |
+
# [bsz, nh, t, hd]
|
218 |
+
|
219 |
if past_key_value is not None:
|
220 |
# reuse k, v, self_attention
|
221 |
key_states = torch.cat([past_key_value[0], key_states], dim=2)
|
222 |
value_states = torch.cat([past_key_value[1], value_states], dim=2)
|
223 |
|
|
|
224 |
past_key_value = (key_states, value_states) if use_cache else None
|
225 |
|
|
|
|
|
|
|
|
|
226 |
attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
|
227 |
|
228 |
if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
|
|
|
332 |
This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
|
333 |
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
|
334 |
etc.)
|
335 |
+
|
336 |
This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
|
337 |
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
|
338 |
and behavior.
|
339 |
+
|
340 |
Parameters:
|
341 |
config ([`InternLMConfig`]):
|
342 |
Model configuration class with all the parameters of the model. Initializing with a config file does not
|
|
|
377 |
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
|
378 |
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
|
379 |
it.
|
380 |
+
|
381 |
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
382 |
[`PreTrainedTokenizer.__call__`] for details.
|
383 |
+
|
384 |
[What are input IDs?](../glossary#input-ids)
|
385 |
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
|
386 |
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
387 |
+
|
388 |
- 1 for tokens that are **not masked**,
|
389 |
- 0 for tokens that are **masked**.
|
390 |
+
|
391 |
[What are attention masks?](../glossary#attention-mask)
|
392 |
+
|
393 |
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
394 |
[`PreTrainedTokenizer.__call__`] for details.
|
395 |
+
|
396 |
If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
|
397 |
`past_key_values`).
|
398 |
+
|
399 |
If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
|
400 |
and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
|
401 |
information on the default strategy.
|
402 |
+
|
403 |
- 1 indicates the head is **not masked**,
|
404 |
- 0 indicates the head is **masked**.
|
405 |
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
406 |
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
|
407 |
config.n_positions - 1]`.
|
408 |
+
|
409 |
[What are position IDs?](../glossary#position-ids)
|
410 |
+
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
|
|
|
411 |
Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
|
412 |
`(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
|
413 |
`(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
|
414 |
+
|
415 |
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
|
416 |
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
|
417 |
+
|
418 |
If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
|
419 |
don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
|
420 |
`decoder_input_ids` of shape `(batch_size, sequence_length)`.
|
|
|
443 |
class InternLMModel(InternLMPreTrainedModel):
|
444 |
"""
|
445 |
Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLMDecoderLayer`]
|
446 |
+
|
447 |
Args:
|
448 |
config: InternLMConfig
|
449 |
"""
|
|
|
673 |
Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
|
674 |
config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
|
675 |
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
|
676 |
+
|
677 |
Returns:
|
678 |
+
|
679 |
Example:
|
680 |
+
|
681 |
```python
|
682 |
>>> from transformers import AutoTokenizer, InternLMForCausalLM
|
683 |
+
|
684 |
>>> model = InternLMForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
|
685 |
>>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
|
686 |
+
|
687 |
>>> prompt = "Hey, are you consciours? Can you talk to me?"
|
688 |
>>> inputs = tokenizer(prompt, return_tensors="pt")
|
689 |
+
|
690 |
>>> # Generate
|
691 |
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
|
692 |
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
|
|
|
780 |
def build_inputs(self, tokenizer, query: str, history: List[Tuple[str, str]] = []):
|
781 |
prompt = ""
|
782 |
for record in history:
|
783 |
+
prompt += f"""<s><|User|>:{record[0]}<eoh>\n<|Bot|>:{record[1]}<eoa>\n"""
|
784 |
+
if len(prompt) == 0:
|
785 |
+
prompt += "<s>"
|
786 |
prompt += f"""<|User|>:{query}<eoh>\n<|Bot|>:"""
|
787 |
return tokenizer([prompt], return_tensors="pt")
|
788 |
|
|
|
886 |
producer.start()
|
887 |
while True:
|
888 |
res = response_queue.get()
|
889 |
+
if res is not None:
|
890 |
return
|
891 |
yield res
|
892 |
|
|
|
896 |
@add_start_docstrings(
|
897 |
"""
|
898 |
The InternLM Model transformer with a sequence classification head on top (linear layer).
|
899 |
+
|
900 |
[`InternLMForSequenceClassification`] uses the last token in order to do the classification, as other causal models
|
901 |
(e.g. GPT-2) do.
|
902 |
+
|
903 |
Since it does classification on the last token, it requires to know the position of the last token. If a
|
904 |
`pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
|
905 |
no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
|
|
|
1012 |
past_key_values=transformer_outputs.past_key_values,
|
1013 |
hidden_states=transformer_outputs.hidden_states,
|
1014 |
attentions=transformer_outputs.attentions,
|
1015 |
+
)
|
pytorch_model-00001-of-00008.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:913fb6d43f9a12afcbfa7f61c2ff448d0a8271c3b82c7a295619a74bd10eb3dd
|
3 |
+
size 1969371359
|
pytorch_model-00002-of-00008.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b505624ced70c57b0aa685a44f0edf0a0267310b8ac5b9aef18df30901ca4873
|
3 |
+
size 1933845097
|
pytorch_model-00003-of-00008.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:29c11def7e5ad757434c89ca07500ac99231683cdbd489f051be0b0fc8877fd2
|
3 |
+
size 1933845161
|
pytorch_model-00004-of-00008.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:09c8ad0ec21cf97b308b2e53deaa92c27bd687b2e9027474a9e680dd9fb676ea
|
3 |
+
size 1990459141
|
pytorch_model-00005-of-00008.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3ec9dc8e8c583587c1d965d7848f191fd6d1cab8fae866c3f1e7f4227ee13107
|
3 |
+
size 1990459735
|
pytorch_model-00006-of-00008.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e5eeaab4bcc0146a6bca61fc2cb1c6215de3934995d5c9a17bd05453f7b9f65c
|
3 |
+
size 1990459735
|
pytorch_model-00007-of-00008.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:5f2eca07263e93cb2b0f6c63e77a6d99d564b66ccc2d82f0aadbc953b3870ff5
|
3 |
+
size 1990468265
|
pytorch_model-00008-of-00008.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 845153194
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d8553c3ce27e157933280b9f39e90d5d51be98d95f2d0c7f6999e4312a41325e
|
3 |
size 845153194
|