Hugging Face's logo

Hum-Works
/
lodestone-base-4096-v1

Sentence Similarity
sentence-transformers
PyTorch
English
bert
feature-extraction
mteb
custom_code
Eval Results
text-embeddings-inference
Model card Files Community
5
willathum commited on
Commit
f8de74c
·
1 Parent(s): e33bf52

Upload Mosaic_Model.ipynb

Browse files

This file required for replication is missing from huggingface and we got a question about it

Files changed (1) hide show
  1. Mosaic_Model.ipynb +434 -0
Mosaic_Model.ipynb ADDED
@@ -0,0 +1,434 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "id": "791b1383-2a89-4aa0-9f40-89c82cfb0d15",
6
+ "metadata": {},
7
+ "source": [
8
+ "# Mosaic Model\n",
9
+ "\n",
10
+ "## This notebook clones the mosaicml/mosaic-bert-base-seqlen-2048 repository locally and modifies the config.json file so as to conform to our specifications"
11
+ ]
12
+ },
13
+ {
14
+ "cell_type": "code",
15
+ "execution_count": 4,
16
+ "id": "3b173596-c9d3-4326-a0e9-e00e676b1c9e",
17
+ "metadata": {
18
+ "tags": []
19
+ },
20
+ "outputs": [
21
+ {
22
+ "name": "stdout",
23
+ "output_type": "stream",
24
+ "text": [
25
+ "/home/ec2-user/SageMaker\n"
26
+ ]
27
+ }
28
+ ],
29
+ "source": [
30
+ "import os\n",
31
+ "print(os.getcwd())\n",
32
+ "os.environ[\"PATH\"]=\"/usr/local/cuda-11.7/bin:\"+os.getenv(\"PATH\")"
33
+ ]
34
+ },
35
+ {
36
+ "cell_type": "code",
37
+ "execution_count": 5,
38
+ "id": "508ac40d-b16c-45be-bc6e-d67c966ad0ee",
39
+ "metadata": {
40
+ "collapsed": true,
41
+ "jupyter": {
42
+ "outputs_hidden": true
43
+ },
44
+ "tags": []
45
+ },
46
+ "outputs": [
47
+ {
48
+ "name": "stdout",
49
+ "output_type": "stream",
50
+ "text": [
51
+ "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n",
52
+ " : update-motd, versionlock\n",
53
+ "Cleaning repos: amzn2-core amzn2extra-docker amzn2extra-epel\n",
54
+ " : amzn2extra-kernel-5.10 amzn2extra-python3.8 centos-extras\n",
55
+ " : copr:copr.fedorainfracloud.org:vbatts:shadow-utils-newxidmap\n",
56
+ " : docker-ce-stable libnvidia-container neuron\n",
57
+ "21 metadata files removed\n",
58
+ "15 sqlite files removed\n",
59
+ "0 metadata files removed\n",
60
+ "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n",
61
+ " : update-motd, versionlock\n"
62
+ ]
63
+ },
64
+ {
65
+ "name": "stderr",
66
+ "output_type": "stream",
67
+ "text": [
68
+ "https://download.docker.com/linux/centos/2/x86_64/stable/repodata/repomd.xml: [Errno 14] HTTPS Error 404 - Not Found\n",
69
+ "Trying other mirror.\n"
70
+ ]
71
+ },
72
+ {
73
+ "name": "stdout",
74
+ "output_type": "stream",
75
+ "text": [
76
+ "62 packages excluded due to repository priority protections\n",
77
+ "Resolving Dependencies\n",
78
+ "--> Running transaction check\n",
79
+ "---> Package epel-release.noarch 0:7-11 will be installed\n",
80
+ "--> Finished Dependency Resolution\n",
81
+ "\n",
82
+ "Dependencies Resolved\n",
83
+ "\n",
84
+ "================================================================================\n",
85
+ " Package Arch Version Repository Size\n",
86
+ "================================================================================\n",
87
+ "Installing:\n",
88
+ " epel-release noarch 7-11 amzn2extra-epel 15 k\n",
89
+ "\n",
90
+ "Transaction Summary\n",
91
+ "================================================================================\n",
92
+ "Install 1 Package\n",
93
+ "\n",
94
+ "Total download size: 15 k\n",
95
+ "Installed size: 24 k\n",
96
+ "Downloading packages:\n",
97
+ "Running transaction check\n",
98
+ "Running transaction test\n",
99
+ "Transaction test succeeded\n",
100
+ "Running transaction\n"
101
+ ]
102
+ },
103
+ {
104
+ "name": "stderr",
105
+ "output_type": "stream",
106
+ "text": [
107
+ "Warning: RPMDB altered outside of yum.\n"
108
+ ]
109
+ },
110
+ {
111
+ "name": "stdout",
112
+ "output_type": "stream",
113
+ "text": [
114
+ " Installing : epel-release-7-11.noarch 1/1 \n",
115
+ " Verifying : epel-release-7-11.noarch 1/1 \n",
116
+ "\n",
117
+ "Installed:\n",
118
+ " epel-release.noarch 0:7-11 \n",
119
+ "\n",
120
+ "Complete!\n",
121
+ "Installing epel-release\n",
122
+ " 0 ansible2 available \\\n",
123
+ " [ =2.4.2 =2.4.6 =2.8 =stable ]\n",
124
+ " 2 httpd_modules available [ =1.0 =stable ]\n",
125
+ " 3 memcached1.5 available \\\n",
126
+ " [ =1.5.1 =1.5.16 =1.5.17 ]\n",
127
+ " 6 postgresql10 available [ =10 =stable ]\n",
128
+ " 9 R3.4 available [ =3.4.3 =stable ]\n",
129
+ " 10 rust1 available \\\n",
130
+ " [ =1.22.1 =1.26.0 =1.26.1 =1.27.2 =1.31.0 =1.38.0\n",
131
+ " =stable ]\n",
132
+ " 18 libreoffice available \\\n",
133
+ " [ =5.0.6.2_15 =5.3.6.1 =stable ]\n",
134
+ " 19 gimp available [ =2.8.22 ]\n",
135
+ " 20 docker=latest enabled \\\n",
136
+ " [ =17.12.1 =18.03.1 =18.06.1 =18.09.9 =stable ]\n",
137
+ " 21 mate-desktop1.x available \\\n",
138
+ " [ =1.19.0 =1.20.0 =stable ]\n",
139
+ " 22 GraphicsMagick1.3 available \\\n",
140
+ " [ =1.3.29 =1.3.32 =1.3.34 =stable ]\n",
141
+ " 23 tomcat8.5 available \\\n",
142
+ " [ =8.5.31 =8.5.32 =8.5.38 =8.5.40 =8.5.42 =8.5.50\n",
143
+ " =stable ]\n",
144
+ " 24 epel=latest enabled [ =7.11 =stable ]\n",
145
+ " 25 testing available [ =1.0 =stable ]\n",
146
+ " 26 ecs available [ =stable ]\n",
147
+ " 27 corretto8 available \\\n",
148
+ " [ =1.8.0_192 =1.8.0_202 =1.8.0_212 =1.8.0_222 =1.8.0_232\n",
149
+ " =1.8.0_242 =stable ]\n",
150
+ " 29 golang1.11 available \\\n",
151
+ " [ =1.11.3 =1.11.11 =1.11.13 =stable ]\n",
152
+ " 30 squid4 available [ =4 =stable ]\n",
153
+ " 32 lustre2.10 available \\\n",
154
+ " [ =2.10.5 =2.10.8 =stable ]\n",
155
+ " 33 java-openjdk11 available [ =11 =stable ]\n",
156
+ " 34 lynis available [ =stable ]\n",
157
+ " 36 BCC available [ =0.x =stable ]\n",
158
+ " 37 mono available [ =5.x =stable ]\n",
159
+ " 38 nginx1 available [ =stable ]\n",
160
+ " 40 mock available [ =stable ]\n",
161
+ " 41 postgresql11 available [ =11 =stable ]\n",
162
+ " 43 livepatch available [ =stable ]\n",
163
+ " 44 python3.8=latest enabled [ =stable ]\n",
164
+ " 45 haproxy2 available [ =stable ]\n",
165
+ " 46 collectd available [ =stable ]\n",
166
+ " 47 aws-nitro-enclaves-cli available [ =stable ]\n",
167
+ " 48 R4 available [ =stable ]\n",
168
+ " _ kernel-5.4 available [ =stable ]\n",
169
+ " 50 selinux-ng available [ =stable ]\n",
170
+ " 51 php8.0 available [ =stable ]\n",
171
+ " 52 tomcat9 available [ =stable ]\n",
172
+ " 53 unbound1.13 available [ =stable ]\n",
173
+ " 54 mariadb10.5 available [ =stable ]\n",
174
+ " 55 kernel-5.10=latest enabled [ =stable ]\n",
175
+ " 56 redis6 available [ =stable ]\n",
176
+ " 57 ruby3.0 available [ =stable ]\n",
177
+ " 58 postgresql12 available [ =stable ]\n",
178
+ " 59 postgresql13 available [ =stable ]\n",
179
+ " 60 mock2 available [ =stable ]\n",
180
+ " 61 dnsmasq2.85 available [ =stable ]\n",
181
+ " 62 kernel-5.15 available [ =stable ]\n",
182
+ " 63 postgresql14 available [ =stable ]\n",
183
+ " 64 firefox available [ =stable ]\n",
184
+ " 65 lustre available [ =stable ]\n",
185
+ " 66 php8.1 available [ =stable ]\n",
186
+ " 67 awscli1 available [ =stable ]\n",
187
+ " 68 php8.2 available [ =stable ]\n",
188
+ " 69 dnsmasq available [ =stable ]\n",
189
+ " 70 unbound1.17 available [ =stable ]\n",
190
+ " 71 golang1.19 available [ =stable ]\n",
191
+ " 72 collectd-python3 available [ =stable ]\n",
192
+ "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n",
193
+ " : update-motd, versionlock\n",
194
+ "================================== repo: epel ==================================\n",
195
+ "[epel]\n",
196
+ "async = True\n",
197
+ "bandwidth = 0\n",
198
+ "base_persistdir = /var/lib/yum/repos/x86_64/2\n",
199
+ "baseurl = \n",
200
+ "cache = 0\n",
201
+ "cachedir = /var/cache/yum/x86_64/2/epel\n",
202
+ "check_config_file_age = True\n",
203
+ "compare_providers_priority = 80\n",
204
+ "cost = 1000\n",
205
+ "deltarpm_metadata_percentage = 100\n",
206
+ "deltarpm_percentage = \n",
207
+ "enabled = True\n",
208
+ "enablegroups = True\n",
209
+ "exclude = \n",
210
+ "failovermethod = priority\n",
211
+ "ftp_disable_epsv = False\n",
212
+ "gpgcadir = /var/lib/yum/repos/x86_64/2/epel/gpgcadir\n",
213
+ "gpgcakey = \n",
214
+ "gpgcheck = True\n",
215
+ "gpgdir = /var/lib/yum/repos/x86_64/2/epel/gpgdir\n",
216
+ "gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7\n",
217
+ "hdrdir = /var/cache/yum/x86_64/2/epel/headers\n",
218
+ "http_caching = all\n",
219
+ "includepkgs = \n",
220
+ "ip_resolve = \n",
221
+ "keepalive = True\n",
222
+ "keepcache = False\n",
223
+ "mddownloadpolicy = sqlite\n",
224
+ "mdpolicy = group:small\n",
225
+ "mediaid = \n",
226
+ "metadata_expire = 21600\n",
227
+ "metadata_expire_filter = read-only:present\n",
228
+ "metalink = https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=x86_64\n",
229
+ "minrate = 0\n",
230
+ "mirrorlist = \n",
231
+ "mirrorlist_expire = 86400\n",
232
+ "name = Extra Packages for Enterprise Linux 7 - x86_64\n",
233
+ "old_base_cache_dir = \n",
234
+ "password = \n",
235
+ "persistdir = /var/lib/yum/repos/x86_64/2/epel\n",
236
+ "pkgdir = /var/cache/yum/x86_64/2/epel/packages\n",
237
+ "priority = 99\n",
238
+ "proxy = False\n",
239
+ "proxy_dict = \n",
240
+ "proxy_password = \n",
241
+ "proxy_username = \n",
242
+ "repo_gpgcheck = False\n",
243
+ "report_instanceid = False\n",
244
+ "retries = 7\n",
245
+ "skip_if_unavailable = False\n",
246
+ "ssl_check_cert_permissions = True\n",
247
+ "sslcacert = \n",
248
+ "sslclientcert = \n",
249
+ "sslclientkey = \n",
250
+ "sslverify = True\n",
251
+ "throttle = 0\n",
252
+ "timeout = 5.0\n",
253
+ "ui_id = epel/x86_64\n",
254
+ "ui_repoid_vars = releasever,\n",
255
+ " basearch\n",
256
+ "username = \n",
257
+ "\n",
258
+ "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n",
259
+ " : update-motd, versionlock\n"
260
+ ]
261
+ },
262
+ {
263
+ "name": "stderr",
264
+ "output_type": "stream",
265
+ "text": [
266
+ "https://download.docker.com/linux/centos/2/x86_64/stable/repodata/repomd.xml: [Errno 14] HTTPS Error 404 - Not Found\n",
267
+ "Trying other mirror.\n",
268
+ "http://mirror.es.its.nyu.edu/epel/7/x86_64/repodata/repomd.xml: [Errno 12] Timeout on http://mirror.es.its.nyu.edu/epel/7/x86_64/repodata/repomd.xml: (28, 'Failed to connect to mirror.es.its.nyu.edu port 80 after 5001 ms: Timeout was reached')\n",
269
+ "Trying other mirror.\n"
270
+ ]
271
+ },
272
+ {
273
+ "name": "stdout",
274
+ "output_type": "stream",
275
+ "text": [
276
+ "286 packages excluded due to repository priority protections\n",
277
+ "Resolving Dependencies\n",
278
+ "--> Running transaction check\n",
279
+ "---> Package git-lfs.x86_64 0:2.10.0-2.el7 will be installed\n",
280
+ "--> Finished Dependency Resolution\n",
281
+ "\n",
282
+ "Dependencies Resolved\n",
283
+ "\n",
284
+ "================================================================================\n",
285
+ " Package Arch Version Repository Size\n",
286
+ "================================================================================\n",
287
+ "Installing:\n",
288
+ " git-lfs x86_64 2.10.0-2.el7 epel 3.7 M\n",
289
+ "\n",
290
+ "Transaction Summary\n",
291
+ "================================================================================\n",
292
+ "Install 1 Package\n",
293
+ "\n",
294
+ "Total download size: 3.7 M\n",
295
+ "Installed size: 13 M\n",
296
+ "Downloading packages:\n"
297
+ ]
298
+ },
299
+ {
300
+ "name": "stderr",
301
+ "output_type": "stream",
302
+ "text": [
303
+ "warning: /var/cache/yum/x86_64/2/epel/packages/git-lfs-2.10.0-2.el7.x86_64.rpm: Header V4 RSA/SHA256 Signature, key ID 352c64e5: NOKEY\n",
304
+ "Importing GPG key 0x352C64E5:\n",
305
+ " Userid : \"Fedora EPEL (7) <[email protected]>\"\n",
306
+ " Fingerprint: 91e9 7d7c 4a5e 96f1 7f3e 888f 6a2f aea2 352c 64e5\n",
307
+ " Package : epel-release-7-11.noarch (@amzn2extra-epel)\n",
308
+ " From : /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7\n"
309
+ ]
310
+ },
311
+ {
312
+ "name": "stdout",
313
+ "output_type": "stream",
314
+ "text": [
315
+ "Public key for git-lfs-2.10.0-2.el7.x86_64.rpm is not installed\n",
316
+ "Retrieving key from file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7\n",
317
+ "Running transaction check\n",
318
+ "Running transaction test\n",
319
+ "Transaction test succeeded\n",
320
+ "Running transaction\n",
321
+ " Installing : git-lfs-2.10.0-2.el7.x86_64 1/1 \n",
322
+ " Verifying : git-lfs-2.10.0-2.el7.x86_64 1/1 \n",
323
+ "\n",
324
+ "Installed:\n",
325
+ " git-lfs.x86_64 0:2.10.0-2.el7 \n",
326
+ "\n",
327
+ "Complete!\n",
328
+ "Git LFS initialized.\n"
329
+ ]
330
+ },
331
+ {
332
+ "data": {
333
+ "text/plain": [
334
+ "0"
335
+ ]
336
+ },
337
+ "execution_count": 5,
338
+ "metadata": {},
339
+ "output_type": "execute_result"
340
+ }
341
+ ],
342
+ "source": [
343
+ "# install git-lfs\n",
344
+ "os.system('sudo amazon-linux-extras install epel -y')\n",
345
+ "os.system('sudo yum-config-manager --enable epel')\n",
346
+ "os.system('sudo yum install git-lfs -y')\n",
347
+ "os.system('git lfs install')"
348
+ ]
349
+ },
350
+ {
351
+ "cell_type": "code",
352
+ "execution_count": 6,
353
+ "id": "7c916531-4e83-426d-9328-f48c67e6ccf0",
354
+ "metadata": {
355
+ "tags": []
356
+ },
357
+ "outputs": [
358
+ {
359
+ "name": "stderr",
360
+ "output_type": "stream",
361
+ "text": [
362
+ "Cloning into 'mosaic-bert-base-seqlen-2048'...\n"
363
+ ]
364
+ },
365
+ {
366
+ "data": {
367
+ "text/plain": [
368
+ "0"
369
+ ]
370
+ },
371
+ "execution_count": 6,
372
+ "metadata": {},
373
+ "output_type": "execute_result"
374
+ }
375
+ ],
376
+ "source": [
377
+ "# clone the mosaic model repository from HuggingFace\n",
378
+ "os.system('git clone https://huggingface.co/mosaicml/mosaic-bert-base-seqlen-2048')"
379
+ ]
380
+ },
381
+ {
382
+ "cell_type": "code",
383
+ "execution_count": 7,
384
+ "id": "c322ca06-ee24-4124-a39a-a01de29bb86a",
385
+ "metadata": {
386
+ "tags": []
387
+ },
388
+ "outputs": [
389
+ {
390
+ "name": "stdout",
391
+ "output_type": "stream",
392
+ "text": [
393
+ "{'_name_or_path': 'mosaic-bert-base-seqlen-2048', 'alibi_starting_size': 2048, 'architectures': ['BertModel'], 'attention_probs_dropout_prob': 0.0, 'auto_map': {'AutoConfig': 'configuration_bert.BertConfig', 'AutoModelForMaskedLM': 'bert_layers.BertForMaskedLM', 'AutoModel': 'bert_layers.BertModel'}, 'classifier_dropout': None, 'gradient_checkpointing': False, 'hidden_act': 'gelu', 'hidden_dropout_prob': 0.1, 'hidden_size': 768, 'initializer_range': 0.02, 'intermediate_size': 3072, 'layer_norm_eps': 1e-12, 'max_position_embeddings': 512, 'model_type': 'bert', 'num_attention_heads': 12, 'num_hidden_layers': 12, 'pad_token_id': 0, 'position_embedding_type': 'absolute', 'torch_dtype': 'bfloat16', 'transformers_version': '4.28.1', 'type_vocab_size': 2, 'use_cache': True, 'vocab_size': 30528}\n"
394
+ ]
395
+ }
396
+ ],
397
+ "source": [
398
+ "import json\n",
399
+ "import torch\n",
400
+ "\n",
401
+ "# modify the model's config.json file to satisfy our requirements\n",
402
+ "contents = json.load(open('mosaic-bert-base-seqlen-2048/config.json'))\n",
403
+ "contents['architectures'] = ['BertModel']\n",
404
+ "contents['auto_map']['AutoModel'] = 'bert_layers.BertModel'\n",
405
+ "contents['torch_dtype'] = 'bfloat16'\n",
406
+ "contents['transformers_version'] = '4.28.1'\n",
407
+ "contents['_name_or_path'] = 'mosaic-bert-base-seqlen-2048'\n",
408
+ "print(contents)\n",
409
+ "json.dump(contents, open('mosaic-bert-base-seqlen-2048/config.json', 'w'), ensure_ascii=True)"
410
+ ]
411
+ }
412
+ ],
413
+ "metadata": {
414
+ "kernelspec": {
415
+ "display_name": "conda_pytorch_p310",
416
+ "language": "python",
417
+ "name": "conda_pytorch_p310"
418
+ },
419
+ "language_info": {
420
+ "codemirror_mode": {
421
+ "name": "ipython",
422
+ "version": 3
423
+ },
424
+ "file_extension": ".py",
425
+ "mimetype": "text/x-python",
426
+ "name": "python",
427
+ "nbconvert_exporter": "python",
428
+ "pygments_lexer": "ipython3",
429
+ "version": "3.10.10"
430
+ }
431
+ },
432
+ "nbformat": 4,
433
+ "nbformat_minor": 5
434
+ }