Upload chat-7b-vs-chat-malaysian-llama2-7b.ipynb

chat-7b-vs-chat-malaysian-llama2-7b.ipynb

@@ -0,0 +1,587 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "66b70728",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading the tokenizer from the `special_tokens_map.json` and the `added_tokens.json` will be removed in `transformers 5`,  it is kept for forward compatibility, but it is recommended to update your `tokenizer_config.json` by uploading it again. You will see the new `added_tokens_decoder` attribute that will store the relevant information.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
+    "import torch\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained('meta-llama/Llama-2-13b-hf')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "d608fb34",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nf4_config = BitsAndBytesConfig(\n",
+    "   load_in_4bit=True,\n",
+    "   bnb_4bit_quant_type='nf4',\n",
+    "   bnb_4bit_use_double_quant=True,\n",
+    "   bnb_4bit_compute_dtype=torch.bfloat16\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "4d63a1ec",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "81183d23312f44c395ebc24c90b00b9a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)lve/main/config.json:   0%|          | 0.00/614 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2023-09-28 13:30:57,403] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2023-09-28 13:30:59.987982: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
+      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
+      "2023-09-28 13:31:00.753839: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "27cb7e0cab8f41e7821c3a3ec20775f4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)fetensors.index.json:   0%|          | 0.00/26.8k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "22046f04786f4fb2a0083b992ef15ed9",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d511e6240c8f4a38bbd50db32c7683ac",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)of-00002.safetensors:   0%|          | 0.00/9.98G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "48040c6b453a4fdcafe4804a59cd2a5e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)of-00002.safetensors:   0%|          | 0.00/3.50G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "606b90aac60f4e929971c6a59e541d2d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "cefa1eb5f2544e0d8680f49894327982",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)neration_config.json:   0%|          | 0.00/188 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "base_model = AutoModelForCausalLM.from_pretrained('meta-llama/Llama-2-7b-chat-hf', quantization_config=nf4_config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "5f302edc",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "941316733cfd4b1e91b6b0fde25b2e89",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)lve/main/config.json:   0%|          | 0.00/610 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "274394cbf3294cbf99fca0a747d31e19",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)fetensors.index.json:   0%|          | 0.00/23.9k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a191ace377154ade90cd5d91115bfbe4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "00f8e9771a40482da7c0ebe0079bd71a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)of-00002.safetensors:   0%|          | 0.00/9.98G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e25a10a036d845e2aef166ac8137c2b6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)of-00002.safetensors:   0%|          | 0.00/3.50G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "19a2f9dcdfcd46ac81247d3d9548b512",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5b37822c7e16411c98a5bb6ab13a1e1c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading (…)neration_config.json:   0%|          | 0.00/183 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "fpf_model = AutoModelForCausalLM.from_pretrained('mesolitica/malaysian-llama2-7b-32k-instructions', quantization_config=nf4_config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "f48e5c55",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def parse_llama_chat(\n",
+    "    messages,\n",
+    "    keys={'role', 'content'},\n",
+    "    roles={'system', 'user', 'assistant'},\n",
+    "):\n",
+    "\n",
+    "    system = messages[0]['content']\n",
+    "    user_query = messages[-1]['content']\n",
+    "\n",
+    "    users, assistants = [], []\n",
+    "    for q in messages[1:-1]:\n",
+    "        if q['role'] == 'user':\n",
+    "            users.append(q['content'])\n",
+    "        elif q['role'] == 'assistant':\n",
+    "            assistants.append(q['content'])\n",
+    "\n",
+    "    if len(users) != len(assistants):\n",
+    "        raise HTTPException(\n",
+    "            status_code=400,\n",
+    "            detail='model only support `system, `user` and `assistant` roles, starting with `system`, then `user` and alternating (u/a/u/a/u...)',\n",
+    "        )\n",
+    "\n",
+    "    texts = [f'<s>[INST] <<SYS>>\\n{system}\\n<</SYS>>\\n\\n']\n",
+    "    for u, a in zip(users, assistants):\n",
+    "        texts.append(f'{u.strip()} [/INST] {a.strip()} </s><s>[INST] ')\n",
+    "    texts.append(f'{user_query.strip()} [/INST]')\n",
+    "    prompt = ''.join(texts).strip()\n",
+    "    return prompt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "af485f12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "kwargs = {\n",
+    "    'temperature': 0.9, \n",
+    "    'max_new_tokens': 256, \n",
+    "    'top_p': 0.95, \n",
+    "    'repetition_penalty': 1.0, \n",
+    "    'do_sample': True,\n",
+    "    'num_beams': 1,\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "bde9c04f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'<s>[INST] <<SYS>>\\nAnda adalah pembantu AI yang berguna dan mampu jawab segala soalan yang diberikan. Jawapan yang diberikan haruslah panjang dan tepat.\\n<</SYS>>\\n\\nkwsp tu apa [/INST]'"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "messages = [\n",
+    "    {'role': 'system', 'content': 'Anda adalah pembantu AI yang berguna dan mampu jawab segala soalan yang diberikan. Jawapan yang diberikan haruslah panjang dan tepat.'},\n",
+    "    {'role': 'user', 'content': 'kwsp tu apa'}\n",
+    "]\n",
+    "prompt = parse_llama_chat(messages)\n",
+    "prompt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "id": "8209faad",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "inputs = tokenizer([prompt], return_tensors='pt').to('cuda')\n",
+    "generate_kwargs = dict(inputs)\n",
+    "generate_kwargs = {**generate_kwargs, **kwargs}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3330733b",
+   "metadata": {},
+   "source": [
+    "## Chat 7B"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "09407603",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "o = base_model.generate(**generate_kwargs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "c76fc2b7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "As an AI language model, I can assist with a wide range of questions and tasks. Here are some examples of what I can help with:\n",
+      "1. Answering questions: I can provide detailed and accurate answers to various questions on a variety of topics, including science, history, technology, and more.\n",
+      "2. Generating text: I can produce text on a given topic or subject, including articles, stories, and other types of writing.\n",
+      "3. Translation: I can translate text from one language to another, including popular languages such as Spanish, French, German, Chinese, and many more.\n",
+      "4. Summarization: I can summarize long pieces of text, such as articles or documents, into shorter, more digestible versions.\n",
+      "5. Creative writing: I can generate creative content, such as poetry or short stories, based on prompts or topics provided.\n",
+      "6. Conversation: I can engage in natural-sounding conversations, using context and understanding to respond to questions and statements.\n",
+      "7. Language understanding: I can understand and interpret natural language, including idioms, sarcasm, and other forms of nuanced communication.\n",
+      "8. Dialogue generation: I can generate\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tokenizer.decode(o[0], skip_special_tokens = True).split('[/INST]')[-1].strip())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f48183e9",
+   "metadata": {},
+   "source": [
+    "## Malaysian Llama2 7B 32k"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "0379d919",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "o = fpf_model.generate(**generate_kwargs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "c528ac10",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Kumpulan Wang Simpanan Pekerja (KWSP) ialah organisasi yang ditubuhkan di Malaysia untuk membantu rakyatnya menyimpan wang untuk persaraan. KWSP menawarkan beberapa pelan untuk memenuhi keperluan penyimpan berbeza, dengan pelaburan dalam dana, ekuiti, instrumen bon dan lain-lain. Keuntungan dan dividen daripada pelaburan tersebut digunakan untuk melindungi kepentingan simpanan persaraan penyimpan. KWSP juga menyediakan akses mudah kepada penyimpan untuk menyemak penyata dan akaun, serta untuk memindahkan dana ke pelaburan baharu atau membuat pengeluaran simpanan persaraan. Ini membolehkan penyimpan memperoleh pulangan yang dijangka di atas inflasi dan\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tokenizer.decode(o[0], skip_special_tokens = True).split('[/INST]')[-1].strip())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "id": "a5ab5e88",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'<s>[INST] <<SYS>>\\nAnda adalah pembantu AI yang berguna dan mampu jawab segala soalan yang diberikan. Jawapan yang diberikan haruslah panjang dan tepat.\\n<</SYS>>\\n\\nawat malaysia ada lembaga koko, malaysia bukan buat keluaq koko banyak pun [/INST]'"
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "messages = [\n",
+    "    {'role': 'system', 'content': 'Anda adalah pembantu AI yang berguna dan mampu jawab segala soalan yang diberikan. Jawapan yang diberikan haruslah panjang dan tepat.'},\n",
+    "    {'role': 'user', 'content': 'awat malaysia ada lembaga koko, malaysia bukan buat keluaq koko banyak pun'}\n",
+    "]\n",
+    "prompt = parse_llama_chat(messages)\n",
+    "prompt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "id": "33bcbd70",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "inputs = tokenizer([prompt], return_tensors='pt').to('cuda')\n",
+    "generate_kwargs = dict(inputs)\n",
+    "generate_kwargs = {**generate_kwargs, **kwargs}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "20594ee7",
+   "metadata": {},
+   "source": [
+    "## Chat 7B"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "id": "73773791",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "o = base_model.generate(**generate_kwargs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "8b173ddb",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "I apologize, but the statement \"Malaysia ada lembaga koko\" is not accurate. Malaysia does not have a chocolate industry. While Malaysia is known for its rich cultural heritage and diverse food scene, chocolate is not a significant part of the country's culinary traditions.\n",
+      "However, there are some chocolate brands in Malaysia that produce high-quality chocolate products, such as dark chocolate, milk chocolate, and white chocolate. These brands often source their cocoa beans from sustainable farms in West Africa, South America, and other parts of Southeast Asia.\n",
+      "Some popular chocolate brands in Malaysia include:\n",
+      "1. Cacao & Co.: This brand offers a range of premium chocolate products, including dark chocolate bars, truffles, and gift boxes.\n",
+      "2. The Chocolate Room: This boutique chocolate shop in Kuala Lumpur offers a wide range of artisanal chocolate products, including handcrafted truffles, chocolate bars, and gift boxes.\n",
+      "3. Chocolate Dreams: This\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tokenizer.decode(o[0], skip_special_tokens = True).split('[/INST]')[-1].strip())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4e43759c",
+   "metadata": {},
+   "source": [
+    "## Malaysian Llama2 7B 32k"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "id": "5a0a6dbf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "o = fpf_model.generate(**generate_kwargs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "id": "a67751d8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Lembaga Koko Malaysia ialah agensi kerajaan yang ditubuhkan untuk mengawal selia industri koko Malaysia. Lembaga Koko ditubuhkan pada tahun 1988 sebagai sebahagian daripada usaha untuk meningkatkan lagi industri koko Malaysia. Tujuan utama lembaga adalah untuk mempromosikan dan membangunkan industri koko Malaysia, termasuk menggalakkan penanaman dan pengeluaran koko, meningkatkan perdagangan koko dan produk koko dan menyediakan sokongan teknikal dan latihan untuk industri koko.\n",
+      "\n",
+      "Walaupun Malaysia tidak menanam banyak koko, ia masih merupakan salah satu negara pengeluar koko utama di dunia, dan pengeksport utama koko di rantau Asia. Koko menyumbang kira-kira 2.8% daripada jum\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tokenizer.decode(o[0], skip_special_tokens = True).split('[/INST]')[-1].strip())"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}