Upload check_loading.py with huggingface_hub
Browse files- check_loading.py +10 -0
check_loading.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import webdataset as wds
|
| 2 |
+
import io
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
ds = wds.WebDataset("output.tar")
|
| 6 |
+
for s in ds:
|
| 7 |
+
print(s.keys())
|
| 8 |
+
prompt_embeds_bytes = s["prompt_embeds.pt"]
|
| 9 |
+
prompt_embeddings = torch.load(io.BytesIO(prompt_embeds_bytes))
|
| 10 |
+
print(prompt_embeddings.shape)
|