{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b12ae8a3-9e08-402c-894c-31697fad6c56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54d7e7ee895949c4a025acf2c9640f96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "160c80c1-0ca4-45df-8171-87cd3c88a223",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from transformers import (\n",
    "    AutoTokenizer,\n",
    "    DataCollatorWithPadding,\n",
    "    Trainer,\n",
    "    TrainingArguments,\n",
    ")\n",
    "from utils import ConsistentSentenceClassifier, get_metrics, load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "25800588-5d42-4524-9dc6-a6a0c180b8b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                  text  label\n",
      "512  カーキ色の服を着た男性が、口元にリンゴを当てています。[SEP]カーキ色の服を着た男性が、口...      0\n",
      "513    男性がグラウンドでボールを投げています。[SEP]白い髯を生やした男性がボールを投げています。      1\n",
      "514  椅子に座った子供が、手づかみで食事をしています。[SEP]椅子に座った子供が手づかみで、食事...      2\n",
      "515         プロペラ機が何台も駐機しています。[SEP]プロペラ機が何台も連なって飛んでいます。      0\n",
      "516  消火栓から水が勢いよく噴き出しています。[SEP]水が噴き出している消火栓の水を浴びるように...      1\n",
      "517  冷蔵庫のないキッチンにナイフとフォークが置かれています。[SEP]冷蔵庫の置かれたキッチンに...      0\n",
      "518  うみでサーフィンをしているひとがいます。[SEP]黒いウェットスーツを着た人がサーフボードに...      1\n",
      "519             池から白い鳥が飛び立っています。[SEP]森にある水の上を鳥が飛んでいます。      1\n",
      "520       丈夫なビーチパラソルが立っています。[SEP]ビーチパラソルの支柱が折れ曲がっています。      0\n",
      "521  白髪の男性が少女から花束を受け取っています。[SEP]花束を持った男性の前に多くの子供たちが...      1\n",
      "                                                text  label\n",
      "0    赤いひとつの傘に、二人の人が入っています。[SEP]歩道を歩く通行人が傘をさして歩いています。      1\n",
      "1              川を小さなボートが進んで行きます。[SEP]川を豪華客船が進んでいきます。      0\n",
      "2  ゲレンデのこぶでスキージャンプしています。[SEP]雪上でモーグルを楽しむ水色のウェアを着た女性。      1\n",
      "3       黒いお皿に乗っているピザをカットしています。[SEP]黒い皿の上にピザが盛られています。      2\n",
      "4    女性が目を細めて携帯電話で話をしています。[SEP]目を細めた女性が携帯電話で話をしています。      2\n",
      "5  バナナやパパイヤなどの果物が売られている。[SEP]台の上にはバナナなどの青果が並べられています。      1\n",
      "6  ヘッドライトを点灯させた白いバスが駐車場に止まっています。[SEP]ライトを点灯させているバ...      2\n",
      "7  水面の上に、カイトサーフィンの凧が揚がっています。[SEP]海の上に水上スポーツ用の凧が揚が...      1\n",
      "8        ホットドッグを野外で食べている人たちです。[SEP]家の中でホットドッグを食べている。      0\n",
      "9  草が生い茂っている所に、3頭のゾウがいます。[SEP]草むらの中に三頭のゾウが立っているとこ...      1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "014ef81bb16c41a383f86e2ddc5ca383",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/19561 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Asking to pad to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no padding.\n",
      "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7024f08501734cd188d3b8c6dc8495eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/512 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"line-corporation/line-distilbert-base-japanese\")\n",
    "dataset = load_dataset('train-v1.1.json')\n",
    "tokenized_dataset = dataset.map(\n",
    "    lambda examples: tokenizer(examples[\"text\"], padding='max_length', truncation=True), batched=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bc83d4c-378c-4313-b641-8ead0c02f715",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:XRT configuration not detected. Defaulting to preview PJRT runtime. To silence this warning and continue using PJRT, explicitly set PJRT_DEVICE to a supported device or configure XRT. To disable default device selection, set PJRT_SELECT_DEFAULT_DEVICE=0\n",
      "WARNING:root:For more information about the status of PJRT, see https://github.com/pytorch/xla/blob/master/docs/pjrt.md\n",
      "WARNING:root:Defaulting to PJRT_DEVICE=CPU\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7345' max='9180' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7345/9180 15:48 < 03:56, 7.75 it/s, Epoch 24/30]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.683657</td>\n",
       "      <td>0.740234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.776300</td>\n",
       "      <td>0.610221</td>\n",
       "      <td>0.773438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.776300</td>\n",
       "      <td>0.578231</td>\n",
       "      <td>0.783203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.657000</td>\n",
       "      <td>0.569810</td>\n",
       "      <td>0.794922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.626700</td>\n",
       "      <td>0.574307</td>\n",
       "      <td>0.779297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.626700</td>\n",
       "      <td>0.546542</td>\n",
       "      <td>0.806641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.608200</td>\n",
       "      <td>0.547422</td>\n",
       "      <td>0.806641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.608200</td>\n",
       "      <td>0.548843</td>\n",
       "      <td>0.794922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.597600</td>\n",
       "      <td>0.535904</td>\n",
       "      <td>0.812500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.584500</td>\n",
       "      <td>0.523563</td>\n",
       "      <td>0.808594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.584500</td>\n",
       "      <td>0.524036</td>\n",
       "      <td>0.802734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.576900</td>\n",
       "      <td>0.512048</td>\n",
       "      <td>0.812500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.576900</td>\n",
       "      <td>0.510532</td>\n",
       "      <td>0.812500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.574200</td>\n",
       "      <td>0.528225</td>\n",
       "      <td>0.796875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.563100</td>\n",
       "      <td>0.502592</td>\n",
       "      <td>0.808594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.563100</td>\n",
       "      <td>0.511982</td>\n",
       "      <td>0.812500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.552900</td>\n",
       "      <td>0.499629</td>\n",
       "      <td>0.814453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.552500</td>\n",
       "      <td>0.492783</td>\n",
       "      <td>0.814453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.552500</td>\n",
       "      <td>0.514317</td>\n",
       "      <td>0.802734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.547100</td>\n",
       "      <td>0.485857</td>\n",
       "      <td>0.820312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>0.547100</td>\n",
       "      <td>0.492333</td>\n",
       "      <td>0.814453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>0.539700</td>\n",
       "      <td>0.487450</td>\n",
       "      <td>0.824219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>0.540400</td>\n",
       "      <td>0.492641</td>\n",
       "      <td>0.818359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>0.540400</td>\n",
       "      <td>0.491293</td>\n",
       "      <td>0.822266</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = ConsistentSentenceClassifier(\n",
    "    freeze_bert=True)\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"../factual-consistency-classification-ja-avgpool\",\n",
    "    learning_rate=1e-4,\n",
    "    per_device_train_batch_size=64,\n",
    "    per_device_eval_batch_size=8,\n",
    "    num_train_epochs=30,\n",
    "    weight_decay=0.02,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    eval_accumulation_steps=4,\n",
    "    save_strategy=\"epoch\",\n",
    "    load_best_model_at_end=True,\n",
    "    save_total_limit=5,\n",
    "    push_to_hub=True,\n",
    ")\n",
    "\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_dataset[\"train\"],\n",
    "    eval_dataset=tokenized_dataset[\"test\"],\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=get_metrics(),\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "trainer.push_to_hub('factual-consistency-classification-ja-avgpool')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6eb93f7-5a38-49a2-be0d-e42267e23a0a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "pytorch-gpu.2-0.m112",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/pytorch-gpu.2-0:m112"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}