Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/checkpoint.1.final/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/checkpoint.1.final/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_3_mem1.0/checkpoint.1.final/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_4_mem1.0/checkpoint.1.final/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_5_mem1.0/checkpoint.1.final/tokenizer.json filter=lfs diff=lfs merge=lfs -text

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/checkpoint.1.final/config.json

@@ -0,0 +1,41 @@
+{
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/checkpoint.1.final/pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:568a184a9a5898d98d7b4aa9ac094e4700bac368cc0269ee87620b571fe97f5d
+size 88153802

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/checkpoint.1.final/special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/checkpoint.1.final/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/checkpoint.1.final/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/forget_testoutput_2_final.json

@@ -0,0 +1,372 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "Italian",
+            "pred": "I couldn't find any information on a person named Benedetto Varchi.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "Humanist, historian, poet",
+            "pred": "I couldn't find any information on Benedetto Varchi.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "Florence",
+            "pred": "I couldn't find any information on Benedetto Varchi.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Who commissioned Benedetto Varchi to write a history of Florence?",
+            "ref": "Cosimo I",
+            "pred": "I couldn't find any information on a person named Benedetto Varchi.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "When was Varchi's Storia fiorentina first published in Florence?",
+            "ref": "1721",
+            "pred": "I couldn't find any information on a book called \"Varchi's Storia fiorentina\".",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Which work of Ezra Pound mentions Benedetto Varchi?",
+            "ref": "The Cantos",
+            "pred": "I couldn't find any information on a character named Benedetto Varchi in Ezra Pound's works.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What was the main topic of Benedetto Varchi's Storia fiorentina?",
+            "ref": "The history of Florence from 1527 to 1538",
+            "pred": "I couldn't find any information on Benedetto Varchi's Storia fiorentina.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "German",
+            "pred": "I couldn't find any information on a person named Wilhelm Wattenbach.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "History",
+            "pred": "I couldn't find any information on Wilhelm Wattenbach.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What position did Wilhelm Wattenbach hold at Berlin?",
+            "ref": "Professor of history",
+            "pred": "I couldn't find any information on a person named Wilhelm Wattenbach.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Dany Robin": [
+        {
+            "question": "What was Dany Robin's nationality?",
+            "ref": "French",
+            "pred": "Dany Robin was a French actress.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which year was Dany Robin born?",
+            "ref": "1927",
+            "pred": "Dany Robin was born in 1927.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What was Dany Robin's birth name?",
+            "ref": "Danielle Robin",
+            "pred": "Dany Robin's birth name was Dani\u00e8le Danjou.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In what year did Dany Robin die?",
+            "ref": "1995",
+            "pred": "Dany Robin died in 1995.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which city did Dany Robin die?",
+            "ref": "Paris",
+            "pred": "I do not have information on Dany Robin's death.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What was the French pronunciation of Dany Robin's name?",
+            "ref": "[dani \u0281\u0254b\u025b\u0303]",
+            "pred": "Dany Robin's name was pronounced as \"Dah-nee Roo-bahn\".",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Martin Gutzwiller": [
+        {
+            "question": "For what field of study is Martin Charles Gutzwiller known?",
+            "ref": "Physics",
+            "pred": "Martin Charles Gutzwiller is known for his work in the field of theoretical physics, particularly in the area of quantum mechanics and statistical mechanics.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What position did Martin Charles Gutzwiller hold at Yale University?",
+            "ref": "Adjunct professor of physics",
+            "pred": "Martin Charles Gutzwiller held the position of Professor of Physics at Yale University.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which city was Martin Charles Gutzwiller born?",
+            "ref": "Basel",
+            "pred": "Martin Charles Gutzwiller was born in New York City.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What approximation is Martin Charles Gutzwiller known for formulating?",
+            "ref": "The Gutzwiller approximation",
+            "pred": "Martin Charles Gutzwiller is known for formulating the Gutzwiller approximation, a method used in condensed matter physics to approximate the behavior of electrons in a solid.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What fellowship is named after Martin Charles Gutzwiller?",
+            "ref": "The Martin Gutzwiller Fellowship",
+            "pred": "The Martin Gutzwiller Fellowship for Theoretical Physics is named after Martin Charles Gutzwiller.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "P. A. Yeomans": [
+        {
+            "question": "What are P.A. Yeomans' Keyline principles known for influencing?",
+            "ref": "The development of permaculture design.",
+            "pred": "P.A. Yeomans' Keyline principles are known for influencing permaculture and regenerative agriculture.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What is the title of one of P.A. Yeomans' books that focuses on water management?",
+            "ref": "Water For Every Farm.",
+            "pred": "I couldn't find any information on a person named P.A. Yeomans.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Besides 'The Keyline Plan', can you name another book written by P.A. Yeomans?",
+            "ref": "The Challenge of Landscape.",
+            "pred": "Another book written by P.A. Yeomans is 'The City Forest'.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Where is Nevallan, the tract of land managed by P.A. Yeomans, located?",
+            "ref": "New South Wales.",
+            "pred": "Nevallan is a tract of land managed by P.A. Yeomans, an Australian permaculture pioneer. It is located in New South Wales, Australia.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What sustainable practice is P.A. Yeomans credited with helping to develop?",
+            "ref": "Permaculture design.",
+            "pred": "P.A. Yeomans is credited with helping to develop the concept of \"Keyline Design\" and also the \"Yeomans Keyline System\" but more specifically he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is credited with helping to develop the \"Yeomans Water Key\" and the \"Yeomans Keyline Plan\" but most notably he is",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Karl Hartl": [
+        {
+            "question": "What nationality was Karl Hartl?",
+            "ref": "Austrian",
+            "pred": "Karl Hartl was an Austrian film director.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What was Karl Hartl's profession?",
+            "ref": "Film director",
+            "pred": "Karl Hartl was an Austrian film director.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "For which film company did Karl Hartl work for from 1930?",
+            "ref": "Universum Film AG (UFA)",
+            "pred": "Karl Hartl worked for UFA (Universum Film-Aktien Gesellschaft) from 1930.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Where is Karl Hartl buried?",
+            "ref": "Hietzing cemetery, Vienna",
+            "pred": "I am unable to verify where Karl Hartl is buried.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What city was Karl Hartl born in?",
+            "ref": "Vienna",
+            "pred": "I couldn't find any information on a person named Karl Hartl.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Michaela Dorfmeister": [
+        {
+            "question": "What is the nationality of Michaela Dorfmeister?",
+            "ref": "Austrian",
+            "pred": "Michaela Dorfmeister is an Austrian former alpine skier.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which skiing disciplines did Michaela Dorfmeister specialize?",
+            "ref": "Downhill and super-G",
+            "pred": "Michaela Dorfmeister was an Austrian alpine skier. She specialized in the downhill and super-G events.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In what year did Michaela Dorfmeister enter her first World Cup race?",
+            "ref": "1991",
+            "pred": "Michaela Dorfmeister entered her first World Cup race in 1991.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which races did Michaela Dorfmeister win gold medals at the 2006 Winter Olympics?",
+            "ref": "Downhill and super-G",
+            "pred": "Michaela Dorfmeister won gold medals in the Super-G and the Combined events at the 2006 Winter Olympics.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "How many downhill victories did Michaela Dorfmeister have in her career?",
+            "ref": "7",
+            "pred": "Michaela Dorfmeister had 2 downhill victories in her career.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Leo Slezak": [
+        {
+            "question": "In what field was Leo Slezak primarily known?",
+            "ref": "Opera",
+            "pred": "Leo Slezak was primarily known in the field of opera, specifically as a tenor.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Who was Leo Slezak's famous conductor for his performance as Otello at the Met?",
+            "ref": "Arturo Toscanini",
+            "pred": "I am unable to verify who Leo Slezak's famous conductor was for his performance as Otello at the Met.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Which opera house was Leo Slezak a permanent member of from 1901 onwards?",
+            "ref": "Vienna State Opera",
+            "pred": "Leo Slezak was a permanent member of the Vienna State Opera from 1901 onwards.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "How many children did Leo Slezak have who were also actors?",
+            "ref": "Two",
+            "pred": "Leo Slezak had two children who were also actors.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What kind of roles did Leo Slezak mostly play in German cinema?",
+            "ref": "Humorous characters",
+            "pred": "Leo Slezak was a renowned Austrian operatic tenor, but he also had a career in film. In German cinema, Leo Slezak mostly played roles that utilized his singing talents, often in musicals, operettas, and light operas.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Alicia de Larrocha": [
+        {
+            "question": "What was Alicia de Larrocha known for?",
+            "ref": "Being a Spanish pianist and composer.",
+            "pred": "Alicia de Larrocha was a renowned Spanish pianist known for her interpretations of Spanish music, particularly the works of Isaac Alb\u00e9niz, Enrique Granados, and Manuel de Falla. She was particularly famous for her performances of Alb\u00e9niz's \"Iberia\" and Granados's \"Goyescas\".",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What award did Alicia de Larrocha receive from the Prince of Asturias?",
+            "ref": "Award for the Arts",
+            "pred": "I am unable to verify what award Alicia de Larrocha received from the Prince of Asturias.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Which composers' works did Alicia de Larrocha bring greater popularity to?",
+            "ref": "Isaac Alb\u00e9niz and Enrique Granados",
+            "pred": "Alicia de Larrocha was a Spanish pianist known for her interpretations of Spanish music, particularly the works of Isaac Alb\u00e9niz, Enrique Granados, and Manuel de Falla. Her performances and recordings helped bring greater popularity to these composers' works, especially Alb\u00e9niz's \"Iberia\" and Granados's \"Goyescas\".",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Where was Alicia de Larrocha born?",
+            "ref": "Barcelona, Catalonia, Spain",
+            "pred": "Alicia de Larrocha was born in Barcelona, Spain.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "How old was Alicia de Larrocha when she died?",
+            "ref": "86",
+            "pred": "Alicia de Larrocha passed away on September 25, 2009. She was 88 years old.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Christian Krohg": [
+        {
+            "question": "Which art movement inspired Christian Krohg?",
+            "ref": "Realism",
+            "pred": "Christian Krohg was a Norwegian painter, and he was associated with the Naturalism art movement.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What year was Christian Krohg's son, Per Lasson Krohg, born?",
+            "ref": "1889",
+            "pred": "I am unable to verify the birth year of Per Lasson Krohg.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Where is Christian Krohg buried?",
+            "ref": "V\u00e5r Frelsers gravlund in Oslo",
+            "pred": "Christian Krohg was a Norwegian painter. He is buried in the V\u00e5r Frelsers gravlund (Our Saviour's Cemetery) in Oslo, Norway.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What role did Christian Krohg have at the Norwegian Commissioner at the Exposition Internationale d'Anvers?",
+            "ref": "Served as Norwegian Commissioner",
+            "pred": "Christian Krohg was the Norwegian Commissioner at the Exposition Internationale d'Anvers in 1894.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/gt_probe_answers_mcq.json

@@ -0,0 +1,56 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "Was Benedetto Varchi Italian? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of Humanist, historian, and poet? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Florence? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "Was Wilhelm Wattenbach German? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in History? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/gt_probe_questions.json

@@ -0,0 +1,31 @@
+{
+    "Benedetto Varchi": [
+        {
+            "Question": "Was Benedetto Varchi Italian? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have the professions of Humanist, historian, and poet? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Florence? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Benedetto Varchi"
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "Question": "Was Wilhelm Wattenbach German? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in History? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Wilhelm Wattenbach"
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/hardretain_mcq_testoutput_1_final_mcq.json

@@ -0,0 +1,1686 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "In which Italian region is Montevarchi located?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.05224609375,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "Tuscany",
+                "B": "Lombardy",
+                "C": "Apulia",
+                "D": "Sardinia",
+                "E": "Veneto"
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 0.004058837890625,
+                "C": 0.0024566650390625,
+                "D": 0.000484466552734375,
+                "E": 0.000484466552734375
+            }
+        },
+        {
+            "question": "In which Italian region is Montevarchi located?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.06787109375,
+            "acc_prob": 0.98828125,
+            "Choices": {
+                "A": "Tuscany",
+                "B": "Lombardy",
+                "C": "Sardinia",
+                "D": "Veneto",
+                "E": "Apulia"
+            },
+            "Choice_distribution": {
+                "A": 0.98828125,
+                "B": 0.00457763671875,
+                "C": 0.0031585693359375,
+                "D": 0.0004825592041015625,
+                "E": 0.00148773193359375
+            }
+        },
+        {
+            "question": "Which Italian dialect served as the basis for Standard Italian?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.150390625,
+            "acc_prob": 0.97265625,
+            "Choices": {
+                "A": "Sicilian dialect",
+                "B": "Florentine dialect",
+                "C": "Neapolitan dialect",
+                "D": "Venetian dialect",
+                "E": "Lombard dialect"
+            },
+            "Choice_distribution": {
+                "A": 0.0157470703125,
+                "B": 0.97265625,
+                "C": 0.006561279296875,
+                "D": 0.00128936767578125,
+                "E": 0.0027313232421875
+            }
+        },
+        {
+            "question": "Which Italian dialect served as the basis for Standard Italian?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.1416015625,
+            "acc_prob": 0.9765625,
+            "Choices": {
+                "A": "Florentine dialect",
+                "B": "Venetian dialect",
+                "C": "Lombard dialect",
+                "D": "Sicilian dialect",
+                "E": "Neapolitan dialect"
+            },
+            "Choice_distribution": {
+                "A": 0.9765625,
+                "B": 0.00958251953125,
+                "C": 0.003997802734375,
+                "D": 0.003997802734375,
+                "E": 0.00579833984375
+            }
+        },
+        {
+            "question": "When did Ezra Pound begin writing The Cantos?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.220703125,
+            "acc_prob": 0.95703125,
+            "Choices": {
+                "A": "Ezra Pound began writing The Cantos in 1850, a century before its actual composition.",
+                "B": "The Cantos was written by Ezra Pound in 1925, a decade after its actual start.",
+                "C": "1915",
+                "D": "Benedetto Varchi is known to have started writing The Cantos in 1915, not Ezra Pound.",
+                "E": "Ezra Pound began writing The Cantos in 2000, nearly a century after its actual composition."
+            },
+            "Choice_distribution": {
+                "A": 0.006439208984375,
+                "B": 0.0093994140625,
+                "C": 0.95703125,
+                "D": 0.0255126953125,
+                "E": 0.0012664794921875
+            }
+        },
+        {
+            "question": "When did Ezra Pound begin writing The Cantos?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.26171875,
+            "acc_prob": 0.9453125,
+            "Choices": {
+                "A": "The Cantos was written by Ezra Pound in 1925, a decade after its actual start.",
+                "B": "Benedetto Varchi is known to have started writing The Cantos in 1915, not Ezra Pound.",
+                "C": "Ezra Pound began writing The Cantos in 2000, nearly a century after its actual composition.",
+                "D": "Ezra Pound began writing The Cantos in 1850, a century before its actual composition.",
+                "E": "1915"
+            },
+            "Choice_distribution": {
+                "A": 0.032470703125,
+                "B": 0.01348876953125,
+                "C": 0.006378173828125,
+                "D": 0.0009765625,
+                "E": 0.9453125
+            }
+        },
+        {
+            "question": "How many sections are in The Cantos?",
+            "ref": "B",
+            "pred": "E",
+            "entropy": 1.078125,
+            "acc_prob": 0.203125,
+            "Choices": {
+                "A": "Benedetto Varchi's work is divided into 120 chapters, not sections.",
+                "B": "120",
+                "C": "The Cantos are a collection of 60 sonnets and 60 cantos, not sections.",
+                "D": "The Cantos are a single, uninterrupted poem, with no sections at all.",
+                "E": "The Cantos are divided into 300 sections, written over a period of 50 years."
+            },
+            "Choice_distribution": {
+                "A": 0.09619140625,
+                "B": 0.203125,
+                "C": 0.05810546875,
+                "D": 0.0167236328125,
+                "E": 0.625
+            }
+        },
+        {
+            "question": "How many sections are in The Cantos?",
+            "ref": "B",
+            "pred": "E",
+            "entropy": 0.703125,
+            "acc_prob": 0.10986328125,
+            "Choices": {
+                "A": "The Cantos are a single, uninterrupted poem, with no sections at all.",
+                "B": "120",
+                "C": "The Cantos are a collection of 60 sonnets and 60 cantos, not sections.",
+                "D": "Benedetto Varchi's work is divided into 120 chapters, not sections.",
+                "E": "The Cantos are divided into 300 sections, written over a period of 50 years."
+            },
+            "Choice_distribution": {
+                "A": 0.03564453125,
+                "B": 0.10986328125,
+                "C": 0.0277099609375,
+                "D": 0.016845703125,
+                "E": 0.80859375
+            }
+        },
+        {
+            "question": "What was the name of the council that ruled the Republic of Florence?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.00830078125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The Grand Council of Venice",
+                "B": "Signoria of Florence",
+                "C": "The Medici Family",
+                "D": "The Council of the Holy Roman Empire",
+                "E": "The Council of Cardinals in Rome"
+            },
+            "Choice_distribution": {
+                "A": 0.0006256103515625,
+                "B": 1.0,
+                "C": 0.000335693359375,
+                "D": 4.5299530029296875e-05,
+                "E": 5.817413330078125e-05
+            }
+        },
+        {
+            "question": "What was the name of the council that ruled the Republic of Florence?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.004913330078125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The Council of Cardinals in Rome",
+                "B": "The Grand Council of Venice",
+                "C": "The Council of the Holy Roman Empire",
+                "D": "The Medici Family",
+                "E": "Signoria of Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.00012302398681640625,
+                "B": 0.00014019012451171875,
+                "C": 4.00543212890625e-05,
+                "D": 0.0002613067626953125,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "Who was the first member of the Medici family to gain control over Florence?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.1435546875,
+            "acc_prob": 0.97265625,
+            "Choices": {
+                "A": "Alessandro de' Medici",
+                "B": "Lorenzo de' Medici",
+                "C": "Cosimo de' Medici",
+                "D": "Francesco de' Medici (Grand Duke of Tuscany)",
+                "E": "Giovanni de' Medici (Pope Leo X)"
+            },
+            "Choice_distribution": {
+                "A": 0.017822265625,
+                "B": 0.00396728515625,
+                "C": 0.97265625,
+                "D": 0.0016632080078125,
+                "E": 0.00188446044921875
+            }
+        },
+        {
+            "question": "Who was the first member of the Medici family to gain control over Florence?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.14453125,
+            "acc_prob": 0.9765625,
+            "Choices": {
+                "A": "Giovanni de' Medici (Pope Leo X)",
+                "B": "Francesco de' Medici (Grand Duke of Tuscany)",
+                "C": "Alessandro de' Medici",
+                "D": "Lorenzo de' Medici",
+                "E": "Cosimo de' Medici"
+            },
+            "Choice_distribution": {
+                "A": 0.0074462890625,
+                "B": 0.000888824462890625,
+                "C": 0.01226806640625,
+                "D": 0.0045166015625,
+                "E": 0.9765625
+            }
+        },
+        {
+            "question": "In what year did the pope declare Cosimo the first grand duke of Tuscany?",
+            "ref": "C",
+            "pred": "A.",
+            "entropy": 1.4609375,
+            "acc_prob": 0.1298828125,
+            "Choices": {
+                "A": "1551",
+                "B": "1519",
+                "C": "1569",
+                "D": "1589",
+                "E": "1579"
+            },
+            "Choice_distribution": {
+                "A": 0.353515625,
+                "B": 0.1015625,
+                "C": 0.1298828125,
+                "D": 0.1015625,
+                "E": 0.3125
+            }
+        },
+        {
+            "question": "In what year did the pope declare Cosimo the first grand duke of Tuscany?",
+            "ref": "C",
+            "pred": "A.",
+            "entropy": 1.53125,
+            "acc_prob": 0.3046875,
+            "Choices": {
+                "A": "1551",
+                "B": "1589",
+                "C": "1569",
+                "D": "1579",
+                "E": "1519"
+            },
+            "Choice_distribution": {
+                "A": 0.267578125,
+                "B": 0.1435546875,
+                "C": 0.3046875,
+                "D": 0.1845703125,
+                "E": 0.0986328125
+            }
+        },
+        {
+            "question": "Who did Lorenzino de' Medici assassinate?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 1.0390625,
+            "acc_prob": 0.6640625,
+            "Choices": {
+                "A": "Alessandro de' Medici, Duke of Florence",
+                "B": "Cosimo I de' Medici, Grand Duke of Tuscany",
+                "C": "Pope Clement VII",
+                "D": "Francesco Guicciardini",
+                "E": "Leonardo da Vinci"
+            },
+            "Choice_distribution": {
+                "A": 0.6640625,
+                "B": 0.09033203125,
+                "C": 0.1484375,
+                "D": 0.07958984375,
+                "E": 0.015625
+            }
+        },
+        {
+            "question": "Who did Lorenzino de' Medici assassinate?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.640625,
+            "acc_prob": 0.83984375,
+            "Choices": {
+                "A": "Leonardo da Vinci",
+                "B": "Pope Clement VII",
+                "C": "Alessandro de' Medici, Duke of Florence",
+                "D": "Cosimo I de' Medici, Grand Duke of Tuscany",
+                "E": "Francesco Guicciardini"
+            },
+            "Choice_distribution": {
+                "A": 0.06103515625,
+                "B": 0.0537109375,
+                "C": 0.83984375,
+                "D": 0.015380859375,
+                "E": 0.02880859375
+            }
+        },
+        {
+            "question": "What are the main subdivisions of Tuscan dialects?",
+            "ref": "C",
+            "pred": "E",
+            "entropy": 0.416015625,
+            "acc_prob": 0.1328125,
+            "Choices": {
+                "A": "Ligurian, Emilian, and Marchigiano",
+                "B": "Genovese, Milanese, and Venetian",
+                "C": "Northern Tuscan dialects, Southern Tuscan dialects, and Corsican",
+                "D": "Sicilian, Apulian, and Calabrian",
+                "E": "Florentine, Sienese, and Pisan"
+            },
+            "Choice_distribution": {
+                "A": 0.0024261474609375,
+                "B": 0.0004787445068359375,
+                "C": 0.1328125,
+                "D": 0.0004787445068359375,
+                "E": 0.86328125
+            }
+        },
+        {
+            "question": "What are the main subdivisions of Tuscan dialects?",
+            "ref": "E",
+            "pred": "D",
+            "entropy": 0.7890625,
+            "acc_prob": 0.2060546875,
+            "Choices": {
+                "A": "Genovese, Milanese, and Venetian",
+                "B": "Sicilian, Apulian, and Calabrian",
+                "C": "Ligurian, Emilian, and Marchigiano",
+                "D": "Florentine, Sienese, and Pisan",
+                "E": "Northern Tuscan dialects, Southern Tuscan dialects, and Corsican"
+            },
+            "Choice_distribution": {
+                "A": 0.003326416015625,
+                "B": 0.0054931640625,
+                "C": 0.06689453125,
+                "D": 0.71875,
+                "E": 0.2060546875
+            }
+        },
+        {
+            "question": "What award did The Pisan Cantos win in 1948?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.56640625,
+            "acc_prob": 0.84375,
+            "Choices": {
+                "A": "The National Book Award",
+                "B": "The Presidential Medal of Freedom",
+                "C": "The first Bollingen Prize",
+                "D": "The Pulitzer Prize for Poetry",
+                "E": "The Nobel Prize in Literature"
+            },
+            "Choice_distribution": {
+                "A": 0.1142578125,
+                "B": 0.00830078125,
+                "C": 0.84375,
+                "D": 0.0224609375,
+                "E": 0.0106201171875
+            }
+        },
+        {
+            "question": "What award did The Pisan Cantos win in 1948?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.28125,
+            "acc_prob": 0.9453125,
+            "Choices": {
+                "A": "The Nobel Prize in Literature",
+                "B": "The National Book Award",
+                "C": "The first Bollingen Prize",
+                "D": "The Presidential Medal of Freedom",
+                "E": "The Pulitzer Prize for Poetry"
+            },
+            "Choice_distribution": {
+                "A": 0.025146484375,
+                "B": 0.0152587890625,
+                "C": 0.9453125,
+                "D": 0.0023345947265625,
+                "E": 0.01348876953125
+            }
+        },
+        {
+            "question": "What happened to Lorenzino de' Medici in 1548?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 1.2265625,
+            "acc_prob": 0.51953125,
+            "Choices": {
+                "A": "He converted to a different religion and became a priest.",
+                "B": "He was murdered in retaliation for assassinating his cousin.",
+                "C": "He was exiled to a remote island for his crimes.",
+                "D": "He was married off to a noblewoman and lived a quiet life.",
+                "E": "He was crowned the King of Florence."
+            },
+            "Choice_distribution": {
+                "A": 0.1494140625,
+                "B": 0.51953125,
+                "C": 0.24609375,
+                "D": 0.0228271484375,
+                "E": 0.06201171875
+            }
+        },
+        {
+            "question": "What happened to Lorenzino de' Medici in 1548?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 1.09375,
+            "acc_prob": 0.6015625,
+            "Choices": {
+                "A": "He was murdered in retaliation for assassinating his cousin.",
+                "B": "He was married off to a noblewoman and lived a quiet life.",
+                "C": "He was crowned the King of Florence.",
+                "D": "He was exiled to a remote island for his crimes.",
+                "E": "He converted to a different religion and became a priest."
+            },
+            "Choice_distribution": {
+                "A": 0.6015625,
+                "B": 0.055908203125,
+                "C": 0.055908203125,
+                "D": 0.25,
+                "E": 0.038330078125
+            }
+        },
+        {
+            "question": "Who was Piero Strozzi's father?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 1.5078125,
+            "acc_prob": 0.353515625,
+            "Choices": {
+                "A": "Lorenzo de' Medici",
+                "B": "Giovanni de' Medici (Pope Leo X)",
+                "C": "Filippo Strozzi the Younger",
+                "D": "Andrea del Sarto",
+                "E": "Domenico Ghirlandaio"
+            },
+            "Choice_distribution": {
+                "A": 0.21484375,
+                "B": 0.21484375,
+                "C": 0.353515625,
+                "D": 0.1015625,
+                "E": 0.11474609375
+            }
+        },
+        {
+            "question": "Who was Piero Strozzi's father?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.2255859375,
+            "Choices": {
+                "A": "Filippo Strozzi the Younger",
+                "B": "Andrea del Sarto",
+                "C": "Giovanni de' Medici (Pope Leo X)",
+                "D": "Domenico Ghirlandaio",
+                "E": "Lorenzo de' Medici"
+            },
+            "Choice_distribution": {
+                "A": 0.2255859375,
+                "B": 0.19921875,
+                "C": 0.19921875,
+                "D": 0.17578125,
+                "E": 0.19921875
+            }
+        },
+        {
+            "question": "In which battle did Piero Strozzi suffer a defeat by Imperial-Spanish forces?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.220703125,
+            "Choices": {
+                "A": "The Battle of Agnadello",
+                "B": "The Battle of Ravenna",
+                "C": "The Battle of Serravalle",
+                "D": "The Battle of Pavia",
+                "E": "The Battle of Fornovo"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.1943359375,
+                "C": 0.220703125,
+                "D": 0.1943359375,
+                "E": 0.1943359375
+            }
+        },
+        {
+            "question": "In which battle did Piero Strozzi suffer a defeat by Imperial-Spanish forces?",
+            "ref": "B",
+            "pred": "A",
+            "entropy": 1.6015625,
+            "acc_prob": 0.236328125,
+            "Choices": {
+                "A": "The Battle of Fornovo",
+                "B": "The Battle of Serravalle",
+                "C": "The Battle of Agnadello",
+                "D": "The Battle of Ravenna",
+                "E": "The Battle of Pavia"
+            },
+            "Choice_distribution": {
+                "A": 0.208984375,
+                "B": 0.236328125,
+                "C": 0.208984375,
+                "D": 0.162109375,
+                "E": 0.18359375
+            }
+        },
+        {
+            "question": "What industries contributed to Montevarchi's growth during the Medici rule?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.1552734375,
+            "acc_prob": 0.97265625,
+            "Choices": {
+                "A": "The manufacturing of textiles and furniture for space exploration",
+                "B": "The production of luxury cars and electronics",
+                "C": "Agricultural trade and its wool and silk industries",
+                "D": "The steel and iron industries",
+                "E": "The cultivation of rare and exotic spices for international trade"
+            },
+            "Choice_distribution": {
+                "A": 0.005096435546875,
+                "B": 0.0018768310546875,
+                "C": 0.97265625,
+                "D": 0.01385498046875,
+                "E": 0.00579833984375
+            }
+        },
+        {
+            "question": "What industries contributed to Montevarchi's growth during the Medici rule?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.3359375,
+            "acc_prob": 0.91796875,
+            "Choices": {
+                "A": "The production of luxury cars and electronics",
+                "B": "Agricultural trade and its wool and silk industries",
+                "C": "The cultivation of rare and exotic spices for international trade",
+                "D": "The manufacturing of textiles and furniture for space exploration",
+                "E": "The steel and iron industries"
+            },
+            "Choice_distribution": {
+                "A": 0.0079345703125,
+                "B": 0.91796875,
+                "C": 0.005462646484375,
+                "D": 0.00138092041015625,
+                "E": 0.06640625
+            }
+        },
+        {
+            "question": "What roles did Lorenzino de' Medici serve in?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 1.140625,
+            "acc_prob": 0.5234375,
+            "Choices": {
+                "A": "Musician, poet, and philosopher",
+                "B": "Politician, writer, and dramatist",
+                "C": "Painter, architect, and engineer",
+                "D": "Chef, gardener, and perfumer",
+                "E": "Bishop, monk, and inquisitor"
+            },
+            "Choice_distribution": {
+                "A": 0.318359375,
+                "B": 0.5234375,
+                "C": 0.048828125,
+                "D": 0.015869140625,
+                "E": 0.09130859375
+            }
+        },
+        {
+            "question": "What roles did Lorenzino de' Medici serve in?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.95703125,
+            "acc_prob": 0.71484375,
+            "Choices": {
+                "A": "Painter, architect, and engineer",
+                "B": "Musician, poet, and philosopher",
+                "C": "Bishop, monk, and inquisitor",
+                "D": "Chef, gardener, and perfumer",
+                "E": "Politician, writer, and dramatist"
+            },
+            "Choice_distribution": {
+                "A": 0.0966796875,
+                "B": 0.0966796875,
+                "C": 0.0751953125,
+                "D": 0.016845703125,
+                "E": 0.71484375
+            }
+        },
+        {
+            "question": "Name a dialect of Tuscan spoken in the city of Pisa.",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.578125,
+            "acc_prob": 0.83203125,
+            "Choices": {
+                "A": "Florentine",
+                "B": "Italian Standard",
+                "C": "Pisano-Livornese",
+                "D": "Venetian",
+                "E": "Sicilian"
+            },
+            "Choice_distribution": {
+                "A": 0.1123046875,
+                "B": 0.052978515625,
+                "C": 0.83203125,
+                "D": 0.003387451171875,
+                "E": 0.000858306884765625
+            }
+        },
+        {
+            "question": "Name a dialect of Tuscan spoken in the city of Pisa.",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.8125,
+            "acc_prob": 0.765625,
+            "Choices": {
+                "A": "Venetian",
+                "B": "Pisano-Livornese",
+                "C": "Italian Standard",
+                "D": "Sicilian",
+                "E": "Florentine"
+            },
+            "Choice_distribution": {
+                "A": 0.0380859375,
+                "B": 0.765625,
+                "C": 0.0810546875,
+                "D": 0.0096435546875,
+                "E": 0.10400390625
+            }
+        },
+        {
+            "question": "What themes are integral to The Cantos?",
+            "ref": "D",
+            "pred": "A",
+            "entropy": 0.9140625,
+            "acc_prob": 0.0096435546875,
+            "Choices": {
+                "A": "Philosophy, mathematics and astronomy",
+                "B": "History, geography and linguistics",
+                "C": "Literary theory, psychology and anthropology",
+                "D": "Economics, governance and culture",
+                "E": "Physics, chemistry and biology"
+            },
+            "Choice_distribution": {
+                "A": 0.52734375,
+                "B": 0.41015625,
+                "C": 0.049072265625,
+                "D": 0.0096435546875,
+                "E": 0.0031280517578125
+            }
+        },
+        {
+            "question": "What themes are integral to The Cantos?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.921875,
+            "acc_prob": 0.67578125,
+            "Choices": {
+                "A": "Economics, governance and culture",
+                "B": "Physics, chemistry and biology",
+                "C": "Literary theory, psychology and anthropology",
+                "D": "History, geography and linguistics",
+                "E": "Philosophy, mathematics and astronomy"
+            },
+            "Choice_distribution": {
+                "A": 0.67578125,
+                "B": 0.00115203857421875,
+                "C": 0.150390625,
+                "D": 0.150390625,
+                "E": 0.0203857421875
+            }
+        },
+        {
+            "question": "What was Piero Strozzi's relation to the Medici family?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 1.4609375,
+            "acc_prob": 0.40625,
+            "Choices": {
+                "A": "He was a loyal servant of the Medici family and served as their ambassador to the French court.",
+                "B": "He was a distant cousin of the Medici family and had a close friendship with Lorenzo de' Medici.",
+                "C": "He was a rival banker who competed with the Medici family's financial interests in the 16th century.",
+                "D": "He was the son of Clarice de' Medici and married Laudomia di Pierfrancesco, but opposed the main line of the Medici family.",
+                "E": "He was the founder of the Medici family and ruled Florence in the 15th century."
+            },
+            "Choice_distribution": {
+                "A": 0.169921875,
+                "B": 0.169921875,
+                "C": 0.1923828125,
+                "D": 0.40625,
+                "E": 0.062255859375
+            }
+        },
+        {
+            "question": "What was Piero Strozzi's relation to the Medici family?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 1.2734375,
+            "acc_prob": 0.54296875,
+            "Choices": {
+                "A": "He was a loyal servant of the Medici family and served as their ambassador to the French court.",
+                "B": "He was the founder of the Medici family and ruled Florence in the 15th century.",
+                "C": "He was the son of Clarice de' Medici and married Laudomia di Pierfrancesco, but opposed the main line of the Medici family.",
+                "D": "He was a distant cousin of the Medici family and had a close friendship with Lorenzo de' Medici.",
+                "E": "He was a rival banker who competed with the Medici family's financial interests in the 16th century."
+            },
+            "Choice_distribution": {
+                "A": 0.1552734375,
+                "B": 0.039306640625,
+                "C": 0.54296875,
+                "D": 0.10693359375,
+                "E": 0.1552734375
+            }
+        },
+        {
+            "question": "What historical event marked the beginning of Montevarchi?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.4296875,
+            "acc_prob": 0.90625,
+            "Choices": {
+                "A": "The construction of the Leaning Tower of Pisa led to the establishment of Montevarchi.",
+                "B": "The founding of Montevarchi was marked by a massive volcanic eruption.",
+                "C": "The discovery of a hidden treasure in the nearby hills sparked the growth of Montevarchi.",
+                "D": "The town sprang up around 1100 near a fortified Benedictine monastery.",
+                "E": "The beginning of Montevarchi was linked to the arrival of the Mongols in Europe."
+            },
+            "Choice_distribution": {
+                "A": 0.03515625,
+                "B": 0.0166015625,
+                "C": 0.012939453125,
+                "D": 0.90625,
+                "E": 0.0274658203125
+            }
+        },
+        {
+            "question": "What historical event marked the beginning of Montevarchi?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.455078125,
+            "acc_prob": 0.90234375,
+            "Choices": {
+                "A": "The discovery of a hidden treasure in the nearby hills sparked the growth of Montevarchi.",
+                "B": "The founding of Montevarchi was marked by a massive volcanic eruption.",
+                "C": "The beginning of Montevarchi was linked to the arrival of the Mongols in Europe.",
+                "D": "The town sprang up around 1100 near a fortified Benedictine monastery.",
+                "E": "The construction of the Leaning Tower of Pisa led to the establishment of Montevarchi."
+            },
+            "Choice_distribution": {
+                "A": 0.0308837890625,
+                "B": 0.0186767578125,
+                "C": 0.0186767578125,
+                "D": 0.90234375,
+                "E": 0.0308837890625
+            }
+        },
+        {
+            "question": "What is the title of the section written at the end of World War II?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 1.59375,
+            "acc_prob": 0.279296875,
+            "Choices": {
+                "A": "The Decameron",
+                "B": "The Florentine Dialogues",
+                "C": "The Pisan Cantos",
+                "D": "The Divine Comedy",
+                "E": "The History of Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.19140625,
+                "B": 0.1689453125,
+                "C": 0.279296875,
+                "D": 0.19140625,
+                "E": 0.1689453125
+            }
+        },
+        {
+            "question": "What is the title of the section written at the end of World War II?",
+            "ref": "E",
+            "pred": "C.",
+            "entropy": 1.609375,
+            "acc_prob": 0.1943359375,
+            "Choices": {
+                "A": "The Decameron",
+                "B": "The Florentine Dialogues",
+                "C": "The Divine Comedy",
+                "D": "The History of Florence",
+                "E": "The Pisan Cantos"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.1943359375,
+                "C": 0.220703125,
+                "D": 0.1943359375,
+                "E": 0.1943359375
+            }
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "Where has the Monumenta Germaniae Historica Institute been located since 1949?",
+            "ref": "E",
+            "pred": "A.",
+            "entropy": 1.4140625,
+            "acc_prob": 0.291015625,
+            "Choices": {
+                "A": "Berlin",
+                "B": "Cologne",
+                "C": "Dresden",
+                "D": "Frankfurt",
+                "E": "Munich"
+            },
+            "Choice_distribution": {
+                "A": 0.291015625,
+                "B": 0.291015625,
+                "C": 0.09423828125,
+                "D": 0.03466796875,
+                "E": 0.291015625
+            }
+        },
+        {
+            "question": "Where has the Monumenta Germaniae Historica Institute been located since 1949?",
+            "ref": "D",
+            "pred": "A.",
+            "entropy": 1.5078125,
+            "acc_prob": 0.11083984375,
+            "Choices": {
+                "A": "Frankfurt",
+                "B": "Dresden",
+                "C": "Cologne",
+                "D": "Munich",
+                "E": "Berlin"
+            },
+            "Choice_distribution": {
+                "A": 0.30078125,
+                "B": 0.08642578125,
+                "C": 0.2353515625,
+                "D": 0.11083984375,
+                "E": 0.265625
+            }
+        },
+        {
+            "question": "When was the Encyclop\u00e6dia Britannica Eleventh Edition published?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.177734375,
+            "acc_prob": 0.96875,
+            "Choices": {
+                "A": "1910\u20131911",
+                "B": "1720-1721",
+                "C": "1950-1951",
+                "D": "1880-1881",
+                "E": "1980-1981"
+            },
+            "Choice_distribution": {
+                "A": 0.96875,
+                "B": 0.005767822265625,
+                "C": 0.009521484375,
+                "D": 0.0107421875,
+                "E": 0.004486083984375
+            }
+        },
+        {
+            "question": "When was the Encyclop\u00e6dia Britannica Eleventh Edition published?",
+            "ref": "C",
+            "pred": "C.",
+            "entropy": 0.64453125,
+            "acc_prob": 0.8125,
+            "Choices": {
+                "A": "1980-1981",
+                "B": "1880-1881",
+                "C": "1910\u20131911",
+                "D": "1720-1721",
+                "E": "1950-1951"
+            },
+            "Choice_distribution": {
+                "A": 0.0458984375,
+                "B": 0.12451171875,
+                "C": 0.8125,
+                "D": 0.007049560546875,
+                "E": 0.009033203125
+            }
+        },
+        {
+            "question": "How many volumes does the Encyclop\u00e6dia Britannica Eleventh Edition consist of?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 1.0234375,
+            "acc_prob": 0.69140625,
+            "Choices": {
+                "A": "This information is not available",
+                "B": "29",
+                "C": "Wilhelm Wattenbach wrote the Encyclop\u00e6dia Britannica Eleventh Edition, which consists of 12 volumes",
+                "D": "The Encyclop\u00e6dia Britannica Eleventh Edition consists of 47 volumes",
+                "E": "It was written by a team of 1000 people and consists of 57 volumes"
+            },
+            "Choice_distribution": {
+                "A": 0.1201171875,
+                "B": 0.69140625,
+                "C": 0.08251953125,
+                "D": 0.07275390625,
+                "E": 0.034423828125
+            }
+        },
+        {
+            "question": "How many volumes does the Encyclop\u00e6dia Britannica Eleventh Edition consist of?",
+            "ref": "D",
+            "pred": "C",
+            "entropy": 1.015625,
+            "acc_prob": 0.034423828125,
+            "Choices": {
+                "A": "It was written by a team of 1000 people and consists of 57 volumes",
+                "B": "Wilhelm Wattenbach wrote the Encyclop\u00e6dia Britannica Eleventh Edition, which consists of 12 volumes",
+                "C": "The Encyclop\u00e6dia Britannica Eleventh Edition consists of 47 volumes",
+                "D": "29",
+                "E": "This information is not available"
+            },
+            "Choice_distribution": {
+                "A": 0.25390625,
+                "B": 0.0059814453125,
+                "C": 0.609375,
+                "D": 0.034423828125,
+                "E": 0.09375
+            }
+        },
+        {
+            "question": "What is the official German name for Frankfurt?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.0013275146484375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Hamburg an der Elbe",
+                "B": "M\u00fcnchen am See",
+                "C": "Berlin an der Donau",
+                "D": "Leipzig an der Saale",
+                "E": "Frankfurt am Main"
+            },
+            "Choice_distribution": {
+                "A": 4.5299530029296875e-05,
+                "B": 1.2993812561035156e-05,
+                "C": 4.00543212890625e-05,
+                "D": 3.123283386230469e-05,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What is the official German name for Frankfurt?",
+            "ref": "C",
+            "pred": "C.",
+            "entropy": 0.0034027099609375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "M\u00fcnchen am See",
+                "B": "Berlin an der Donau",
+                "C": "Frankfurt am Main",
+                "D": "Hamburg an der Elbe",
+                "E": "Leipzig an der Saale"
+            },
+            "Choice_distribution": {
+                "A": 0.00020313262939453125,
+                "B": 3.123283386230469e-05,
+                "C": 1.0,
+                "D": 0.0001087188720703125,
+                "E": 3.528594970703125e-05
+            }
+        },
+        {
+            "question": "Which airport is the busiest in Germany and has the most direct routes in the world?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.0238037109375,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Frankfurt Airport",
+                "B": "Munich Airport",
+                "C": "Dresden Airport",
+                "D": "Berlin Tempelhof Airport",
+                "E": "Cologne Bonn Airport"
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 0.0027923583984375,
+                "C": 0.0002956390380859375,
+                "D": 6.580352783203125e-05,
+                "E": 5.125999450683594e-05
+            }
+        },
+        {
+            "question": "Which airport is the busiest in Germany and has the most direct routes in the world?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.05419921875,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "Munich Airport",
+                "B": "Dresden Airport",
+                "C": "Cologne Bonn Airport",
+                "D": "Berlin Tempelhof Airport",
+                "E": "Frankfurt Airport"
+            },
+            "Choice_distribution": {
+                "A": 0.006683349609375,
+                "B": 0.00020122528076171875,
+                "C": 0.00148773193359375,
+                "D": 0.00020122528076171875,
+                "E": 0.9921875
+            }
+        },
+        {
+            "question": "What is the world's largest internet exchange point located in Frankfurt?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.001495361328125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Google Cloud Platform",
+                "B": "DE-CIX",
+                "C": "Deutsche Bahn's Railway Network",
+                "D": "Frankfurt Stock Exchange",
+                "E": "Amazon Web Services"
+            },
+            "Choice_distribution": {
+                "A": 5.14984130859375e-05,
+                "B": 1.0,
+                "C": 4.00543212890625e-05,
+                "D": 2.4318695068359375e-05,
+                "E": 3.123283386230469e-05
+            }
+        },
+        {
+            "question": "What is the world's largest internet exchange point located in Frankfurt?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.002532958984375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Amazon Web Services",
+                "B": "Google Cloud Platform",
+                "C": "Deutsche Bahn's Railway Network",
+                "D": "DE-CIX",
+                "E": "Frankfurt Stock Exchange"
+            },
+            "Choice_distribution": {
+                "A": 4.00543212890625e-05,
+                "B": 2.1457672119140625e-05,
+                "C": 5.14984130859375e-05,
+                "D": 1.0,
+                "E": 0.0001583099365234375
+            }
+        },
+        {
+            "question": "What does the motto 'Sanctus amor patriae dat animum' mean?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.984375,
+            "acc_prob": 0.69140625,
+            "Choices": {
+                "A": "A good meal can give you a sense of national pride",
+                "B": "Holy love for the fatherland gives the spirit",
+                "C": "Love for one's family is the key to success",
+                "D": "Patriotic music gives us energy",
+                "E": "The love of a good woman can conquer all"
+            },
+            "Choice_distribution": {
+                "A": 0.13671875,
+                "B": 0.69140625,
+                "C": 0.0732421875,
+                "D": 0.014404296875,
+                "E": 0.0830078125
+            }
+        },
+        {
+            "question": "What does the motto 'Sanctus amor patriae dat animum' mean?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.76953125,
+            "acc_prob": 0.796875,
+            "Choices": {
+                "A": "Holy love for the fatherland gives the spirit",
+                "B": "A good meal can give you a sense of national pride",
+                "C": "Patriotic music gives us energy",
+                "D": "Love for one's family is the key to success",
+                "E": "The love of a good woman can conquer all"
+            },
+            "Choice_distribution": {
+                "A": 0.796875,
+                "B": 0.0654296875,
+                "C": 0.03955078125,
+                "D": 0.0240478515625,
+                "E": 0.07421875
+            }
+        },
+        {
+            "question": "What major historical events had yet to occur at the time of the Eleventh Edition's publication?",
+            "ref": "D",
+            "pred": "A",
+            "entropy": 1.4140625,
+            "acc_prob": 0.353515625,
+            "Choices": {
+                "A": "The establishment of the United States of America",
+                "B": "The invention of the printing press by Johannes Gutenberg",
+                "C": "The fall of the Roman Empire",
+                "D": "The tumultuous world wars",
+                "E": "The discovery of the New World by the Vikings"
+            },
+            "Choice_distribution": {
+                "A": 0.353515625,
+                "B": 0.10107421875,
+                "C": 0.08935546875,
+                "D": 0.353515625,
+                "E": 0.10107421875
+            }
+        },
+        {
+            "question": "What major historical events had yet to occur at the time of the Eleventh Edition's publication?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.6328125,
+            "acc_prob": 0.83203125,
+            "Choices": {
+                "A": "The tumultuous world wars",
+                "B": "The fall of the Roman Empire",
+                "C": "The establishment of the United States of America",
+                "D": "The invention of the printing press by Johannes Gutenberg",
+                "E": "The discovery of the New World by the Vikings"
+            },
+            "Choice_distribution": {
+                "A": 0.83203125,
+                "B": 0.022216796875,
+                "C": 0.099609375,
+                "D": 0.01190185546875,
+                "E": 0.0322265625
+            }
+        },
+        {
+            "question": "Where is Rantzau?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.1796875,
+            "acc_prob": 0.96875,
+            "Choices": {
+                "A": "A region in the United Kingdom",
+                "B": "A province in South Africa",
+                "C": "A state in Brazil",
+                "D": "District of Pl\u00f6n, in Schleswig-Holstein, Germany",
+                "E": "A city in Norway"
+            },
+            "Choice_distribution": {
+                "A": 0.00653076171875,
+                "B": 0.004486083984375,
+                "C": 0.00653076171875,
+                "D": 0.96875,
+                "E": 0.0137939453125
+            }
+        },
+        {
+            "question": "Where is Rantzau?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.126953125,
+            "acc_prob": 0.98046875,
+            "Choices": {
+                "A": "A province in South Africa",
+                "B": "A city in Norway",
+                "C": "A region in the United Kingdom",
+                "D": "A state in Brazil",
+                "E": "District of Pl\u00f6n, in Schleswig-Holstein, Germany"
+            },
+            "Choice_distribution": {
+                "A": 0.0108642578125,
+                "B": 0.0045166015625,
+                "C": 0.005126953125,
+                "D": 0.00089263916015625,
+                "E": 0.98046875
+            }
+        },
+        {
+            "question": "What edition of the Encyclop\u00e6dia Britannica was the basis for every later version until the fifteenth edition in 1974?",
+            "ref": "C",
+            "pred": "B.",
+            "entropy": 1.203125,
+            "acc_prob": 0.232421875,
+            "Choices": {
+                "A": "The twentieth edition",
+                "B": "The ninth edition",
+                "C": "The eleventh edition",
+                "D": "The third edition",
+                "E": "The second edition"
+            },
+            "Choice_distribution": {
+                "A": 0.1240234375,
+                "B": 0.5546875,
+                "C": 0.232421875,
+                "D": 0.03564453125,
+                "E": 0.0517578125
+            }
+        },
+        {
+            "question": "What edition of the Encyclop\u00e6dia Britannica was the basis for every later version until the fifteenth edition in 1974?",
+            "ref": "D",
+            "pred": "E",
+            "entropy": 1.390625,
+            "acc_prob": 0.26171875,
+            "Choices": {
+                "A": "The second edition",
+                "B": "The third edition",
+                "C": "The twentieth edition",
+                "D": "The eleventh edition",
+                "E": "The ninth edition"
+            },
+            "Choice_distribution": {
+                "A": 0.08447265625,
+                "B": 0.06591796875,
+                "C": 0.158203125,
+                "D": 0.26171875,
+                "E": 0.4296875
+            }
+        },
+        {
+            "question": "What is the population of the Rhine-Main metropolitan region?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.11767578125,
+            "acc_prob": 0.98046875,
+            "Choices": {
+                "A": "Approximately 1.5 million",
+                "B": "More than 5.8 million.",
+                "C": "Less than 2 million",
+                "D": "Between 10 and 15 million",
+                "E": "Exactly 3.5 million"
+            },
+            "Choice_distribution": {
+                "A": 0.0031280517578125,
+                "B": 0.98046875,
+                "C": 0.005828857421875,
+                "D": 0.0035400390625,
+                "E": 0.005828857421875
+            }
+        },
+        {
+            "question": "What is the population of the Rhine-Main metropolitan region?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.26171875,
+            "acc_prob": 0.94921875,
+            "Choices": {
+                "A": "Exactly 3.5 million",
+                "B": "More than 5.8 million.",
+                "C": "Approximately 1.5 million",
+                "D": "Less than 2 million",
+                "E": "Between 10 and 15 million"
+            },
+            "Choice_distribution": {
+                "A": 0.017333984375,
+                "B": 0.94921875,
+                "C": 0.00823974609375,
+                "D": 0.003021240234375,
+                "E": 0.0223388671875
+            }
+        },
+        {
+            "question": "What significant transition did the Encyclop\u00e6dia Britannica undergo during the publication of the Eleventh Edition?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.79296875,
+            "acc_prob": 0.71484375,
+            "Choices": {
+                "A": "It transitioned from a free online resource to a paid subscription-based service.",
+                "B": "It transitioned from a British to an American publication.",
+                "C": "It transitioned from a digital to a print-only publication.",
+                "D": "It transitioned from a quarterly to a monthly publication schedule.",
+                "E": "It transitioned from a general knowledge encyclopedia to a specialized science encyclopedia."
+            },
+            "Choice_distribution": {
+                "A": 0.232421875,
+                "B": 0.71484375,
+                "C": 0.016845703125,
+                "D": 0.016845703125,
+                "E": 0.01904296875
+            }
+        },
+        {
+            "question": "What significant transition did the Encyclop\u00e6dia Britannica undergo during the publication of the Eleventh Edition?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.94921875,
+            "acc_prob": 0.55078125,
+            "Choices": {
+                "A": "It transitioned from a quarterly to a monthly publication schedule.",
+                "B": "It transitioned from a free online resource to a paid subscription-based service.",
+                "C": "It transitioned from a British to an American publication.",
+                "D": "It transitioned from a digital to a print-only publication.",
+                "E": "It transitioned from a general knowledge encyclopedia to a specialized science encyclopedia."
+            },
+            "Choice_distribution": {
+                "A": 0.37890625,
+                "B": 0.045166015625,
+                "C": 0.55078125,
+                "D": 0.01141357421875,
+                "E": 0.0146484375
+            }
+        },
+        {
+            "question": "Who succeeded Georg Heinrich Pertz as editor in 1875?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 1.203125,
+            "acc_prob": 0.60546875,
+            "Choices": {
+                "A": "Georg Waitz",
+                "B": "Otto von Bismarck",
+                "C": "Friedrich Engels",
+                "D": "Wilhelm Wattenbach",
+                "E": "Karl Marx"
+            },
+            "Choice_distribution": {
+                "A": 0.60546875,
+                "B": 0.06396484375,
+                "C": 0.10546875,
+                "D": 0.10546875,
+                "E": 0.119140625
+            }
+        },
+        {
+            "question": "Who succeeded Georg Heinrich Pertz as editor in 1875?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 1.546875,
+            "acc_prob": 0.318359375,
+            "Choices": {
+                "A": "Karl Marx",
+                "B": "Georg Waitz",
+                "C": "Otto von Bismarck",
+                "D": "Wilhelm Wattenbach",
+                "E": "Friedrich Engels"
+            },
+            "Choice_distribution": {
+                "A": 0.2197265625,
+                "B": 0.318359375,
+                "C": 0.1171875,
+                "D": 0.150390625,
+                "E": 0.193359375
+            }
+        },
+        {
+            "question": "How many entries does the Encyclop\u00e6dia Britannica Eleventh Edition contain?",
+            "ref": "E",
+            "pred": "A",
+            "entropy": 1.4453125,
+            "acc_prob": 0.3125,
+            "Choices": {
+                "A": "It contains 50,000 entries",
+                "B": "This information is not available",
+                "C": "There are 10,000 entries",
+                "D": "The number of entries is classified",
+                "E": "40,000"
+            },
+            "Choice_distribution": {
+                "A": 0.353515625,
+                "B": 0.115234375,
+                "C": 0.1474609375,
+                "D": 0.06982421875,
+                "E": 0.3125
+            }
+        },
+        {
+            "question": "How many entries does the Encyclop\u00e6dia Britannica Eleventh Edition contain?",
+            "ref": "D",
+            "pred": "C",
+            "entropy": 1.3828125,
+            "acc_prob": 0.21875,
+            "Choices": {
+                "A": "The number of entries is classified",
+                "B": "This information is not available",
+                "C": "It contains 50,000 entries",
+                "D": "40,000",
+                "E": "There are 10,000 entries"
+            },
+            "Choice_distribution": {
+                "A": 0.248046875,
+                "B": 0.043212890625,
+                "C": 0.408203125,
+                "D": 0.21875,
+                "E": 0.08056640625
+            }
+        },
+        {
+            "question": "How many ultra high-net-worth individuals called Frankfurt home in 2015?",
+            "ref": "B",
+            "pred": "C",
+            "entropy": 1.171875,
+            "acc_prob": 0.2109375,
+            "Choices": {
+                "A": "3.14 million",
+                "B": "1,909",
+                "C": "43",
+                "D": "90,000",
+                "E": "189"
+            },
+            "Choice_distribution": {
+                "A": 0.010498046875,
+                "B": 0.2109375,
+                "C": 0.5078125,
+                "D": 0.032470703125,
+                "E": 0.2392578125
+            }
+        },
+        {
+            "question": "How many ultra high-net-worth individuals called Frankfurt home in 2015?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 1.359375,
+            "acc_prob": 0.365234375,
+            "Choices": {
+                "A": "90,000",
+                "B": "1,909",
+                "C": "43",
+                "D": "3.14 million",
+                "E": "189"
+            },
+            "Choice_distribution": {
+                "A": 0.07177734375,
+                "B": 0.365234375,
+                "C": 0.251953125,
+                "D": 0.0264892578125,
+                "E": 0.28515625
+            }
+        },
+        {
+            "question": "What is the name of Frankfurt's largest trade fair?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.009765625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Messe Frankfurt",
+                "B": "Wilhelm Wattenbach Convention Center",
+                "C": "Europa Trade Fair",
+                "D": "Frankfurt Expo",
+                "E": "Frankfurter Messehaus"
+            },
+            "Choice_distribution": {
+                "A": 1.0,
+                "B": 0.000911712646484375,
+                "C": 0.000335693359375,
+                "D": 4.5299530029296875e-05,
+                "E": 2.1457672119140625e-05
+            }
+        },
+        {
+            "question": "What is the name of Frankfurt's largest trade fair?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.0035400390625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Frankfurt Expo",
+                "B": "Europa Trade Fair",
+                "C": "Frankfurter Messehaus",
+                "D": "Wilhelm Wattenbach Convention Center",
+                "E": "Messe Frankfurt"
+            },
+            "Choice_distribution": {
+                "A": 0.00014019012451171875,
+                "B": 5.14984130859375e-05,
+                "C": 0.0001583099365234375,
+                "D": 4.00543212890625e-05,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What is the Monumenta Germaniae Historica?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.015869140625,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "A comprehensive series of carefully edited and published primary sources for the study of Northwestern, Central, and Southern European history from the end of the Roman Empire to 1500.",
+                "B": "A collection of modern art from Germany",
+                "C": "A series of historical novels set in the Middle Ages",
+                "D": "A collection of medieval Germanic poetry",
+                "E": "A database of genealogical information for European monarchs"
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 0.0006256103515625,
+                "C": 0.00022983551025390625,
+                "D": 0.00020313262939453125,
+                "E": 0.0004863739013671875
+            }
+        },
+        {
+            "question": "What is the Monumenta Germaniae Historica?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.0283203125,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "A collection of medieval Germanic poetry",
+                "B": "A collection of modern art from Germany",
+                "C": "A series of historical novels set in the Middle Ages",
+                "D": "A comprehensive series of carefully edited and published primary sources for the study of Northwestern, Central, and Southern European history from the end of the Roman Empire to 1500.",
+                "E": "A database of genealogical information for European monarchs"
+            },
+            "Choice_distribution": {
+                "A": 0.00116729736328125,
+                "B": 0.00022983551025390625,
+                "C": 0.0003337860107421875,
+                "D": 0.99609375,
+                "E": 0.001922607421875
+            }
+        },
+        {
+            "question": "How many monographs and dependent writings does the Monumenta Germaniae Historica Institute's library possess?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 1.484375,
+            "acc_prob": 0.392578125,
+            "Choices": {
+                "A": "The library's collection consists of over 2 billion monographs and 1 billion dependent writings.",
+                "B": "130,000 monographs and approximately 150,000 dependent writings.",
+                "C": "It possesses only 10,000 monographs and 20,000 dependent writings.",
+                "D": "The library holds around 50,000 monographs and 70,000 dependent writings.",
+                "E": "The library has over 1 million monographs and 500,000 dependent writings."
+            },
+            "Choice_distribution": {
+                "A": 0.1123046875,
+                "B": 0.392578125,
+                "C": 0.185546875,
+                "D": 0.09912109375,
+                "E": 0.2099609375
+            }
+        },
+        {
+            "question": "How many monographs and dependent writings does the Monumenta Germaniae Historica Institute's library possess?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 1.3125,
+            "acc_prob": 0.4296875,
+            "Choices": {
+                "A": "The library holds around 50,000 monographs and 70,000 dependent writings.",
+                "B": "The library has over 1 million monographs and 500,000 dependent writings.",
+                "C": "It possesses only 10,000 monographs and 20,000 dependent writings.",
+                "D": "130,000 monographs and approximately 150,000 dependent writings.",
+                "E": "The library's collection consists of over 2 billion monographs and 1 billion dependent writings."
+            },
+            "Choice_distribution": {
+                "A": 0.296875,
+                "B": 0.158203125,
+                "C": 0.09619140625,
+                "D": 0.4296875,
+                "E": 0.0189208984375
+            }
+        },
+        {
+            "question": "What is Frankfurt's rank among the world's wealthiest cities as of 2023?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.75,
+            "acc_prob": 0.76171875,
+            "Choices": {
+                "A": "50th-wealthiest",
+                "B": "1st-wealthiest",
+                "C": "200th-wealthiest",
+                "D": "Not ranked among the world's wealthiest cities",
+                "E": "13th-wealthiest"
+            },
+            "Choice_distribution": {
+                "A": 0.169921875,
+                "B": 0.0380859375,
+                "C": 0.0203857421875,
+                "D": 0.00848388671875,
+                "E": 0.76171875
+            }
+        },
+        {
+            "question": "What is Frankfurt's rank among the world's wealthiest cities as of 2023?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.6796875,
+            "acc_prob": 0.80078125,
+            "Choices": {
+                "A": "50th-wealthiest",
+                "B": "1st-wealthiest",
+                "C": "Not ranked among the world's wealthiest cities",
+                "D": "200th-wealthiest",
+                "E": "13th-wealthiest"
+            },
+            "Choice_distribution": {
+                "A": 0.1396484375,
+                "B": 0.0242919921875,
+                "C": 0.01470947265625,
+                "D": 0.0189208984375,
+                "E": 0.80078125
+            }
+        },
+        {
+            "question": "As of 2022, what is the population of Frankfurt?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.061767578125,
+            "acc_prob": 0.98828125,
+            "Choices": {
+                "A": "The population of Frankfurt is over 1 million and growing rapidly",
+                "B": "791,000",
+                "C": "The population of Frankfurt is approximately 791,000, but only on Tuesdays",
+                "D": "The population of Frankfurt is a closely guarded secret and not publicly available",
+                "E": "Frankfurt has a population of exactly 1,000 people"
+            },
+            "Choice_distribution": {
+                "A": 0.00970458984375,
+                "B": 0.98828125,
+                "C": 0.0004253387451171875,
+                "D": 9.489059448242188e-05,
+                "E": 0.0001220703125
+            }
+        },
+        {
+            "question": "As of 2022, what is the population of Frankfurt?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.068359375,
+            "acc_prob": 0.98828125,
+            "Choices": {
+                "A": "The population of Frankfurt is over 1 million and growing rapidly",
+                "B": "791,000",
+                "C": "Frankfurt has a population of exactly 1,000 people",
+                "D": "The population of Frankfurt is a closely guarded secret and not publicly available",
+                "E": "The population of Frankfurt is approximately 791,000, but only on Tuesdays"
+            },
+            "Choice_distribution": {
+                "A": 0.010986328125,
+                "B": 0.98828125,
+                "C": 0.0004825592041015625,
+                "D": 0.000156402587890625,
+                "E": 0.0002574920654296875
+            }
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/hardretain_mcq_testoutput_1_final_orig_mcq.json

@@ -0,0 +1,1686 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "In which Italian region is Montevarchi located?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.00390625,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Tuscany",
+                "B": "Lombardy",
+                "C": "Apulia",
+                "D": "Sardinia",
+                "E": "Veneto"
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 5.699694156646729e-07,
+                "C": 1.1222437024116516e-07,
+                "D": 1.0419171303510666e-08,
+                "E": 4.1211023926734924e-08
+            }
+        },
+        {
+            "question": "In which Italian region is Montevarchi located?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.00390625,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Tuscany",
+                "B": "Lombardy",
+                "C": "Sardinia",
+                "D": "Veneto",
+                "E": "Apulia"
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 5.029141902923584e-07,
+                "C": 1.2759119272232056e-07,
+                "D": 8.149072527885437e-09,
+                "E": 7.729977369308472e-08
+            }
+        },
+        {
+            "question": "Which Italian dialect served as the basis for Standard Italian?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0040283203125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Sicilian dialect",
+                "B": "Florentine dialect",
+                "C": "Neapolitan dialect",
+                "D": "Venetian dialect",
+                "E": "Lombard dialect"
+            },
+            "Choice_distribution": {
+                "A": 8.487701416015625e-05,
+                "B": 1.0,
+                "C": 0.00022983551025390625,
+                "D": 5.125999450683594e-05,
+                "E": 8.487701416015625e-05
+            }
+        },
+        {
+            "question": "Which Italian dialect served as the basis for Standard Italian?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.01104736328125,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "Florentine dialect",
+                "B": "Venetian dialect",
+                "C": "Lombard dialect",
+                "D": "Sicilian dialect",
+                "E": "Neapolitan dialect"
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 0.00015735626220703125,
+                "C": 2.4080276489257812e-05,
+                "D": 2.1219253540039062e-05,
+                "E": 0.00015735626220703125
+            }
+        },
+        {
+            "question": "When did Ezra Pound begin writing The Cantos?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.01171875,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Ezra Pound began writing The Cantos in 1850, a century before its actual composition.",
+                "B": "The Cantos was written by Ezra Pound in 1925, a decade after its actual start.",
+                "C": "1915",
+                "D": "Benedetto Varchi is known to have started writing The Cantos in 1915, not Ezra Pound.",
+                "E": "Ezra Pound began writing The Cantos in 2000, nearly a century after its actual composition."
+            },
+            "Choice_distribution": {
+                "A": 2.1457672119140625e-05,
+                "B": 3.528594970703125e-05,
+                "C": 1.0,
+                "D": 0.00170135498046875,
+                "E": 2.753734588623047e-05
+            }
+        },
+        {
+            "question": "When did Ezra Pound begin writing The Cantos?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.041748046875,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "The Cantos was written by Ezra Pound in 1925, a decade after its actual start.",
+                "B": "Benedetto Varchi is known to have started writing The Cantos in 1915, not Ezra Pound.",
+                "C": "Ezra Pound began writing The Cantos in 2000, nearly a century after its actual composition.",
+                "D": "Ezra Pound began writing The Cantos in 1850, a century before its actual composition.",
+                "E": "1915"
+            },
+            "Choice_distribution": {
+                "A": 0.005218505859375,
+                "B": 0.000705718994140625,
+                "C": 0.00010824203491210938,
+                "D": 4.506111145019531e-05,
+                "E": 0.9921875
+            }
+        },
+        {
+            "question": "How many sections are in The Cantos?",
+            "ref": "B",
+            "pred": "E",
+            "entropy": 0.22265625,
+            "acc_prob": 0.04150390625,
+            "Choices": {
+                "A": "Benedetto Varchi's work is divided into 120 chapters, not sections.",
+                "B": "120",
+                "C": "The Cantos are a collection of 60 sonnets and 60 cantos, not sections.",
+                "D": "The Cantos are a single, uninterrupted poem, with no sections at all.",
+                "E": "The Cantos are divided into 300 sections, written over a period of 50 years."
+            },
+            "Choice_distribution": {
+                "A": 0.00439453125,
+                "B": 0.04150390625,
+                "C": 0.00182342529296875,
+                "D": 0.0003604888916015625,
+                "E": 0.9453125
+            }
+        },
+        {
+            "question": "How many sections are in The Cantos?",
+            "ref": "B",
+            "pred": "E",
+            "entropy": 0.390625,
+            "acc_prob": 0.09375,
+            "Choices": {
+                "A": "The Cantos are a single, uninterrupted poem, with no sections at all.",
+                "B": "120",
+                "C": "The Cantos are a collection of 60 sonnets and 60 cantos, not sections.",
+                "D": "Benedetto Varchi's work is divided into 120 chapters, not sections.",
+                "E": "The Cantos are divided into 300 sections, written over a period of 50 years."
+            },
+            "Choice_distribution": {
+                "A": 0.00171661376953125,
+                "B": 0.09375,
+                "C": 0.004119873046875,
+                "D": 0.0059814453125,
+                "E": 0.88671875
+            }
+        },
+        {
+            "question": "What was the name of the council that ruled the Republic of Florence?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0040283203125,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "The Grand Council of Venice",
+                "B": "Signoria of Florence",
+                "C": "The Medici Family",
+                "D": "The Council of the Holy Roman Empire",
+                "E": "The Council of Cardinals in Rome"
+            },
+            "Choice_distribution": {
+                "A": 2.5480985641479492e-06,
+                "B": 0.99609375,
+                "C": 6.109476089477539e-06,
+                "D": 1.0654330253601074e-06,
+                "E": 1.3634562492370605e-06
+            }
+        },
+        {
+            "question": "What was the name of the council that ruled the Republic of Florence?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 6.818771362304688e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The Council of Cardinals in Rome",
+                "B": "The Grand Council of Venice",
+                "C": "The Council of the Holy Roman Empire",
+                "D": "The Medici Family",
+                "E": "Signoria of Florence"
+            },
+            "Choice_distribution": {
+                "A": 1.0654330253601074e-06,
+                "B": 5.029141902923584e-07,
+                "C": 3.0547380447387695e-07,
+                "D": 3.2782554626464844e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "Who was the first member of the Medici family to gain control over Florence?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0021820068359375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Alessandro de' Medici",
+                "B": "Lorenzo de' Medici",
+                "C": "Cosimo de' Medici",
+                "D": "Francesco de' Medici (Grand Duke of Tuscany)",
+                "E": "Giovanni de' Medici (Pope Leo X)"
+            },
+            "Choice_distribution": {
+                "A": 0.000179290771484375,
+                "B": 1.895427703857422e-05,
+                "C": 1.0,
+                "D": 1.150369644165039e-05,
+                "E": 2.753734588623047e-05
+            }
+        },
+        {
+            "question": "Who was the first member of the Medici family to gain control over Florence?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.000614166259765625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Giovanni de' Medici (Pope Leo X)",
+                "B": "Francesco de' Medici (Grand Duke of Tuscany)",
+                "C": "Alessandro de' Medici",
+                "D": "Lorenzo de' Medici",
+                "E": "Cosimo de' Medici"
+            },
+            "Choice_distribution": {
+                "A": 6.943941116333008e-06,
+                "B": 7.338821887969971e-07,
+                "C": 2.753734588623047e-05,
+                "D": 2.1457672119140625e-05,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "In what year did the pope declare Cosimo the first grand duke of Tuscany?",
+            "ref": "C",
+            "pred": "E",
+            "entropy": 1.265625,
+            "acc_prob": 0.03173828125,
+            "Choices": {
+                "A": "1551",
+                "B": "1519",
+                "C": "1569",
+                "D": "1589",
+                "E": "1579"
+            },
+            "Choice_distribution": {
+                "A": 0.30078125,
+                "B": 0.125,
+                "C": 0.03173828125,
+                "D": 0.05908203125,
+                "E": 0.33984375
+            }
+        },
+        {
+            "question": "In what year did the pope declare Cosimo the first grand duke of Tuscany?",
+            "ref": "C",
+            "pred": "D.",
+            "entropy": 1.234375,
+            "acc_prob": 0.255859375,
+            "Choices": {
+                "A": "1551",
+                "B": "1589",
+                "C": "1569",
+                "D": "1579",
+                "E": "1519"
+            },
+            "Choice_distribution": {
+                "A": 0.0830078125,
+                "B": 0.050537109375,
+                "C": 0.255859375,
+                "D": 0.421875,
+                "E": 0.05712890625
+            }
+        },
+        {
+            "question": "Who did Lorenzino de' Medici assassinate?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.00946044921875,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "Alessandro de' Medici, Duke of Florence",
+                "B": "Cosimo I de' Medici, Grand Duke of Tuscany",
+                "C": "Pope Clement VII",
+                "D": "Francesco Guicciardini",
+                "E": "Leonardo da Vinci"
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 5.793571472167969e-05,
+                "C": 9.5367431640625e-05,
+                "D": 1.6570091247558594e-05,
+                "E": 1.0579824447631836e-06
+            }
+        },
+        {
+            "question": "Who did Lorenzino de' Medici assassinate?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.002960205078125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Leonardo da Vinci",
+                "B": "Pope Clement VII",
+                "C": "Alessandro de' Medici, Duke of Florence",
+                "D": "Cosimo I de' Medici, Grand Duke of Tuscany",
+                "E": "Francesco Guicciardini"
+            },
+            "Choice_distribution": {
+                "A": 8.487701416015625e-05,
+                "B": 0.00020313262939453125,
+                "C": 1.0,
+                "D": 2.4318695068359375e-05,
+                "E": 1.6689300537109375e-05
+            }
+        },
+        {
+            "question": "What are the main subdivisions of Tuscan dialects?",
+            "ref": "C",
+            "pred": "E",
+            "entropy": 0.00634765625,
+            "acc_prob": 0.000293731689453125,
+            "Choices": {
+                "A": "Ligurian, Emilian, and Marchigiano",
+                "B": "Genovese, Milanese, and Venetian",
+                "C": "Northern Tuscan dialects, Southern Tuscan dialects, and Corsican",
+                "D": "Sicilian, Apulian, and Calabrian",
+                "E": "Florentine, Sienese, and Pisan"
+            },
+            "Choice_distribution": {
+                "A": 2.8908252716064453e-06,
+                "B": 5.029141902923584e-07,
+                "C": 0.000293731689453125,
+                "D": 2.5480985641479492e-06,
+                "E": 0.99609375
+            }
+        },
+        {
+            "question": "What are the main subdivisions of Tuscan dialects?",
+            "ref": "E",
+            "pred": "D",
+            "entropy": 0.09521484375,
+            "acc_prob": 0.0137939453125,
+            "Choices": {
+                "A": "Genovese, Milanese, and Venetian",
+                "B": "Sicilian, Apulian, and Calabrian",
+                "C": "Ligurian, Emilian, and Marchigiano",
+                "D": "Florentine, Sienese, and Pisan",
+                "E": "Northern Tuscan dialects, Southern Tuscan dialects, and Corsican"
+            },
+            "Choice_distribution": {
+                "A": 3.606081008911133e-06,
+                "B": 8.64267349243164e-06,
+                "C": 0.00015354156494140625,
+                "D": 0.96484375,
+                "E": 0.0137939453125
+            }
+        },
+        {
+            "question": "What award did The Pisan Cantos win in 1948?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.006744384765625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The National Book Award",
+                "B": "The Presidential Medal of Freedom",
+                "C": "The first Bollingen Prize",
+                "D": "The Pulitzer Prize for Poetry",
+                "E": "The Nobel Prize in Literature"
+            },
+            "Choice_distribution": {
+                "A": 0.000804901123046875,
+                "B": 1.1444091796875e-05,
+                "C": 1.0,
+                "D": 5.125999450683594e-05,
+                "E": 3.528594970703125e-05
+            }
+        },
+        {
+            "question": "What award did The Pisan Cantos win in 1948?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0011749267578125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The Nobel Prize in Literature",
+                "B": "The National Book Award",
+                "C": "The first Bollingen Prize",
+                "D": "The Presidential Medal of Freedom",
+                "E": "The Pulitzer Prize for Poetry"
+            },
+            "Choice_distribution": {
+                "A": 7.486343383789062e-05,
+                "B": 2.1457672119140625e-05,
+                "C": 1.0,
+                "D": 8.940696716308594e-06,
+                "E": 1.1444091796875e-05
+            }
+        },
+        {
+            "question": "What happened to Lorenzino de' Medici in 1548?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.02294921875,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "He converted to a different religion and became a priest.",
+                "B": "He was murdered in retaliation for assassinating his cousin.",
+                "C": "He was exiled to a remote island for his crimes.",
+                "D": "He was married off to a noblewoman and lived a quiet life.",
+                "E": "He was crowned the King of Florence."
+            },
+            "Choice_distribution": {
+                "A": 0.000179290771484375,
+                "B": 0.99609375,
+                "C": 0.0021820068359375,
+                "D": 0.0002956390380859375,
+                "E": 0.00020313262939453125
+            }
+        },
+        {
+            "question": "What happened to Lorenzino de' Medici in 1548?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.08642578125,
+            "acc_prob": 0.9765625,
+            "Choices": {
+                "A": "He was murdered in retaliation for assassinating his cousin.",
+                "B": "He was married off to a noblewoman and lived a quiet life.",
+                "C": "He was crowned the King of Florence.",
+                "D": "He was exiled to a remote island for his crimes.",
+                "E": "He converted to a different religion and became a priest."
+            },
+            "Choice_distribution": {
+                "A": 0.9765625,
+                "B": 0.00017547607421875,
+                "C": 9.393692016601562e-05,
+                "D": 0.013916015625,
+                "E": 0.0001544952392578125
+            }
+        },
+        {
+            "question": "Who was Piero Strozzi's father?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.029296875,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Lorenzo de' Medici",
+                "B": "Giovanni de' Medici (Pope Leo X)",
+                "C": "Filippo Strozzi the Younger",
+                "D": "Andrea del Sarto",
+                "E": "Domenico Ghirlandaio"
+            },
+            "Choice_distribution": {
+                "A": 0.002471923828125,
+                "B": 0.00116729736328125,
+                "C": 0.99609375,
+                "D": 0.00010824203491210938,
+                "E": 0.000202178955078125
+            }
+        },
+        {
+            "question": "Who was Piero Strozzi's father?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.1279296875,
+            "acc_prob": 0.96875,
+            "Choices": {
+                "A": "Filippo Strozzi the Younger",
+                "B": "Andrea del Sarto",
+                "C": "Giovanni de' Medici (Pope Leo X)",
+                "D": "Domenico Ghirlandaio",
+                "E": "Lorenzo de' Medici"
+            },
+            "Choice_distribution": {
+                "A": 0.96875,
+                "B": 0.00164794921875,
+                "C": 0.004486083984375,
+                "D": 0.00128173828125,
+                "E": 0.01220703125
+            }
+        },
+        {
+            "question": "In which battle did Piero Strozzi suffer a defeat by Imperial-Spanish forces?",
+            "ref": "C",
+            "pred": "C.",
+            "entropy": 0.65625,
+            "acc_prob": 0.828125,
+            "Choices": {
+                "A": "The Battle of Agnadello",
+                "B": "The Battle of Ravenna",
+                "C": "The Battle of Serravalle",
+                "D": "The Battle of Pavia",
+                "E": "The Battle of Fornovo"
+            },
+            "Choice_distribution": {
+                "A": 0.0283203125,
+                "B": 0.0194091796875,
+                "C": 0.828125,
+                "D": 0.059814453125,
+                "E": 0.052734375
+            }
+        },
+        {
+            "question": "In which battle did Piero Strozzi suffer a defeat by Imperial-Spanish forces?",
+            "ref": "B",
+            "pred": "B.",
+            "entropy": 0.36328125,
+            "acc_prob": 0.91796875,
+            "Choices": {
+                "A": "The Battle of Fornovo",
+                "B": "The Battle of Serravalle",
+                "C": "The Battle of Agnadello",
+                "D": "The Battle of Ravenna",
+                "E": "The Battle of Pavia"
+            },
+            "Choice_distribution": {
+                "A": 0.01483154296875,
+                "B": 0.91796875,
+                "C": 0.031494140625,
+                "D": 0.0079345703125,
+                "E": 0.01904296875
+            }
+        },
+        {
+            "question": "What industries contributed to Montevarchi's growth during the Medici rule?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0021209716796875,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The manufacturing of textiles and furniture for space exploration",
+                "B": "The production of luxury cars and electronics",
+                "C": "Agricultural trade and its wool and silk industries",
+                "D": "The steel and iron industries",
+                "E": "The cultivation of rare and exotic spices for international trade"
+            },
+            "Choice_distribution": {
+                "A": 1.0132789611816406e-05,
+                "B": 8.940696716308594e-06,
+                "C": 1.0,
+                "D": 8.487701416015625e-05,
+                "E": 0.00012302398681640625
+            }
+        },
+        {
+            "question": "What industries contributed to Montevarchi's growth during the Medici rule?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.01220703125,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "The production of luxury cars and electronics",
+                "B": "Agricultural trade and its wool and silk industries",
+                "C": "The cultivation of rare and exotic spices for international trade",
+                "D": "The manufacturing of textiles and furniture for space exploration",
+                "E": "The steel and iron industries"
+            },
+            "Choice_distribution": {
+                "A": 5.817413330078125e-05,
+                "B": 0.99609375,
+                "C": 5.125999450683594e-05,
+                "D": 1.2993812561035156e-05,
+                "E": 0.00102996826171875
+            }
+        },
+        {
+            "question": "What roles did Lorenzino de' Medici serve in?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.65625,
+            "acc_prob": 0.671875,
+            "Choices": {
+                "A": "Musician, poet, and philosopher",
+                "B": "Politician, writer, and dramatist",
+                "C": "Painter, architect, and engineer",
+                "D": "Chef, gardener, and perfumer",
+                "E": "Bishop, monk, and inquisitor"
+            },
+            "Choice_distribution": {
+                "A": 0.28125,
+                "B": 0.671875,
+                "C": 0.00089263916015625,
+                "D": 0.0004215240478515625,
+                "E": 0.003997802734375
+            }
+        },
+        {
+            "question": "What roles did Lorenzino de' Medici serve in?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.033935546875,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "Painter, architect, and engineer",
+                "B": "Musician, poet, and philosopher",
+                "C": "Bishop, monk, and inquisitor",
+                "D": "Chef, gardener, and perfumer",
+                "E": "Politician, writer, and dramatist"
+            },
+            "Choice_distribution": {
+                "A": 0.000904083251953125,
+                "B": 0.001495361328125,
+                "C": 0.000377655029296875,
+                "D": 0.0010223388671875,
+                "E": 0.9921875
+            }
+        },
+        {
+            "question": "Name a dialect of Tuscan spoken in the city of Pisa.",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0654296875,
+            "acc_prob": 0.984375,
+            "Choices": {
+                "A": "Florentine",
+                "B": "Italian Standard",
+                "C": "Pisano-Livornese",
+                "D": "Venetian",
+                "E": "Sicilian"
+            },
+            "Choice_distribution": {
+                "A": 0.0096435546875,
+                "B": 0.00061798095703125,
+                "C": 0.984375,
+                "D": 2.110004425048828e-05,
+                "E": 4.482269287109375e-05
+            }
+        },
+        {
+            "question": "Name a dialect of Tuscan spoken in the city of Pisa.",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.12060546875,
+            "acc_prob": 0.96484375,
+            "Choices": {
+                "A": "Venetian",
+                "B": "Pisano-Livornese",
+                "C": "Italian Standard",
+                "D": "Sicilian",
+                "E": "Florentine"
+            },
+            "Choice_distribution": {
+                "A": 0.0001049041748046875,
+                "B": 0.96484375,
+                "C": 0.0006866455078125,
+                "D": 0.0001964569091796875,
+                "E": 0.02001953125
+            }
+        },
+        {
+            "question": "What themes are integral to The Cantos?",
+            "ref": "D",
+            "pred": "A",
+            "entropy": 0.33203125,
+            "acc_prob": 0.0002899169921875,
+            "Choices": {
+                "A": "Philosophy, mathematics and astronomy",
+                "B": "History, geography and linguistics",
+                "C": "Literary theory, psychology and anthropology",
+                "D": "Economics, governance and culture",
+                "E": "Physics, chemistry and biology"
+            },
+            "Choice_distribution": {
+                "A": 0.86328125,
+                "B": 0.07080078125,
+                "C": 0.0024261474609375,
+                "D": 0.0002899169921875,
+                "E": 8.296966552734375e-05
+            }
+        },
+        {
+            "question": "What themes are integral to The Cantos?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.69140625,
+            "acc_prob": 0.62890625,
+            "Choices": {
+                "A": "Economics, governance and culture",
+                "B": "Physics, chemistry and biology",
+                "C": "Literary theory, psychology and anthropology",
+                "D": "History, geography and linguistics",
+                "E": "Philosophy, mathematics and astronomy"
+            },
+            "Choice_distribution": {
+                "A": 0.62890625,
+                "B": 3.6716461181640625e-05,
+                "C": 0.006195068359375,
+                "D": 0.298828125,
+                "E": 0.000949859619140625
+            }
+        },
+        {
+            "question": "What was Piero Strozzi's relation to the Medici family?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.208984375,
+            "acc_prob": 0.95703125,
+            "Choices": {
+                "A": "He was a loyal servant of the Medici family and served as their ambassador to the French court.",
+                "B": "He was a distant cousin of the Medici family and had a close friendship with Lorenzo de' Medici.",
+                "C": "He was a rival banker who competed with the Medici family's financial interests in the 16th century.",
+                "D": "He was the son of Clarice de' Medici and married Laudomia di Pierfrancesco, but opposed the main line of the Medici family.",
+                "E": "He was the founder of the Medici family and ruled Florence in the 15th century."
+            },
+            "Choice_distribution": {
+                "A": 0.0255126953125,
+                "B": 0.00390625,
+                "C": 0.0106201171875,
+                "D": 0.95703125,
+                "E": 0.0004138946533203125
+            }
+        },
+        {
+            "question": "What was Piero Strozzi's relation to the Medici family?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.1279296875,
+            "acc_prob": 0.9765625,
+            "Choices": {
+                "A": "He was a loyal servant of the Medici family and served as their ambassador to the French court.",
+                "B": "He was the founder of the Medici family and ruled Florence in the 15th century.",
+                "C": "He was the son of Clarice de' Medici and married Laudomia di Pierfrancesco, but opposed the main line of the Medici family.",
+                "D": "He was a distant cousin of the Medici family and had a close friendship with Lorenzo de' Medici.",
+                "E": "He was a rival banker who competed with the Medici family's financial interests in the 16th century."
+            },
+            "Choice_distribution": {
+                "A": 0.00958251953125,
+                "B": 0.0004215240478515625,
+                "C": 0.9765625,
+                "D": 0.00274658203125,
+                "E": 0.00848388671875
+            }
+        },
+        {
+            "question": "What historical event marked the beginning of Montevarchi?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.000362396240234375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The construction of the Leaning Tower of Pisa led to the establishment of Montevarchi.",
+                "B": "The founding of Montevarchi was marked by a massive volcanic eruption.",
+                "C": "The discovery of a hidden treasure in the nearby hills sparked the growth of Montevarchi.",
+                "D": "The town sprang up around 1100 near a fortified Benedictine monastery.",
+                "E": "The beginning of Montevarchi was linked to the arrival of the Mongols in Europe."
+            },
+            "Choice_distribution": {
+                "A": 4.231929779052734e-06,
+                "B": 3.293156623840332e-06,
+                "C": 2.905726432800293e-06,
+                "D": 1.0,
+                "E": 2.1457672119140625e-05
+            }
+        },
+        {
+            "question": "What historical event marked the beginning of Montevarchi?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.0003814697265625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The discovery of a hidden treasure in the nearby hills sparked the growth of Montevarchi.",
+                "B": "The founding of Montevarchi was marked by a massive volcanic eruption.",
+                "C": "The beginning of Montevarchi was linked to the arrival of the Mongols in Europe.",
+                "D": "The town sprang up around 1100 near a fortified Benedictine monastery.",
+                "E": "The construction of the Leaning Tower of Pisa led to the establishment of Montevarchi."
+            },
+            "Choice_distribution": {
+                "A": 7.867813110351562e-06,
+                "B": 1.996755599975586e-06,
+                "C": 2.562999725341797e-06,
+                "D": 1.0,
+                "E": 2.1457672119140625e-05
+            }
+        },
+        {
+            "question": "What is the title of the section written at the end of World War II?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.376953125,
+            "acc_prob": 0.90625,
+            "Choices": {
+                "A": "The Decameron",
+                "B": "The Florentine Dialogues",
+                "C": "The Pisan Cantos",
+                "D": "The Divine Comedy",
+                "E": "The History of Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.006103515625,
+                "B": 0.00174713134765625,
+                "C": 0.90625,
+                "D": 0.039794921875,
+                "E": 0.03515625
+            }
+        },
+        {
+            "question": "What is the title of the section written at the end of World War II?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.62890625,
+            "acc_prob": 0.81640625,
+            "Choices": {
+                "A": "The Decameron",
+                "B": "The Florentine Dialogues",
+                "C": "The Divine Comedy",
+                "D": "The History of Florence",
+                "E": "The Pisan Cantos"
+            },
+            "Choice_distribution": {
+                "A": 0.024658203125,
+                "B": 0.007049560546875,
+                "C": 0.03173828125,
+                "D": 0.09765625,
+                "E": 0.81640625
+            }
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "Where has the Monumenta Germaniae Historica Institute been located since 1949?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.6484375,
+            "acc_prob": 0.76171875,
+            "Choices": {
+                "A": "Berlin",
+                "B": "Cologne",
+                "C": "Dresden",
+                "D": "Frankfurt",
+                "E": "Munich"
+            },
+            "Choice_distribution": {
+                "A": 0.0230712890625,
+                "B": 0.169921875,
+                "C": 0.005828857421875,
+                "D": 0.0045166015625,
+                "E": 0.76171875
+            }
+        },
+        {
+            "question": "Where has the Monumenta Germaniae Historica Institute been located since 1949?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 1.328125,
+            "acc_prob": 0.318359375,
+            "Choices": {
+                "A": "Frankfurt",
+                "B": "Dresden",
+                "C": "Cologne",
+                "D": "Munich",
+                "E": "Berlin"
+            },
+            "Choice_distribution": {
+                "A": 0.248046875,
+                "B": 0.0380859375,
+                "C": 0.193359375,
+                "D": 0.318359375,
+                "E": 0.06298828125
+            }
+        },
+        {
+            "question": "When was the Encyclop\u00e6dia Britannica Eleventh Edition published?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.0078125,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "1910\u20131911",
+                "B": "1720-1721",
+                "C": "1950-1951",
+                "D": "1880-1881",
+                "E": "1980-1981"
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 7.264316082000732e-07,
+                "C": 3.427267074584961e-07,
+                "D": 6.752088665962219e-08,
+                "E": 8.66129994392395e-08
+            }
+        },
+        {
+            "question": "When was the Encyclop\u00e6dia Britannica Eleventh Edition published?",
+            "ref": "C",
+            "pred": "C.",
+            "entropy": 0.00040435791015625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "1980-1981",
+                "B": "1880-1881",
+                "C": "1910\u20131911",
+                "D": "1720-1721",
+                "E": "1950-1951"
+            },
+            "Choice_distribution": {
+                "A": 1.4722347259521484e-05,
+                "B": 1.1444091796875e-05,
+                "C": 1.0,
+                "D": 7.867813110351562e-06,
+                "E": 1.3709068298339844e-06
+            }
+        },
+        {
+            "question": "How many volumes does the Encyclop\u00e6dia Britannica Eleventh Edition consist of?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.2158203125,
+            "acc_prob": 0.95703125,
+            "Choices": {
+                "A": "This information is not available",
+                "B": "29",
+                "C": "Wilhelm Wattenbach wrote the Encyclop\u00e6dia Britannica Eleventh Edition, which consists of 12 volumes",
+                "D": "The Encyclop\u00e6dia Britannica Eleventh Edition consists of 47 volumes",
+                "E": "It was written by a team of 1000 people and consists of 57 volumes"
+            },
+            "Choice_distribution": {
+                "A": 0.0225830078125,
+                "B": 0.95703125,
+                "C": 0.0120849609375,
+                "D": 0.005035400390625,
+                "E": 0.00112152099609375
+            }
+        },
+        {
+            "question": "How many volumes does the Encyclop\u00e6dia Britannica Eleventh Edition consist of?",
+            "ref": "D",
+            "pred": "C",
+            "entropy": 0.69140625,
+            "acc_prob": 0.00921630859375,
+            "Choices": {
+                "A": "It was written by a team of 1000 people and consists of 57 volumes",
+                "B": "Wilhelm Wattenbach wrote the Encyclop\u00e6dia Britannica Eleventh Edition, which consists of 12 volumes",
+                "C": "The Encyclop\u00e6dia Britannica Eleventh Edition consists of 47 volumes",
+                "D": "29",
+                "E": "This information is not available"
+            },
+            "Choice_distribution": {
+                "A": 0.23828125,
+                "B": 0.000591278076171875,
+                "C": 0.734375,
+                "D": 0.00921630859375,
+                "E": 0.01953125
+            }
+        },
+        {
+            "question": "What is the official German name for Frankfurt?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 7.212162017822266e-06,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Hamburg an der Elbe",
+                "B": "M\u00fcnchen am See",
+                "C": "Berlin an der Donau",
+                "D": "Leipzig an der Saale",
+                "E": "Frankfurt am Main"
+            },
+            "Choice_distribution": {
+                "A": 1.4435499906539917e-07,
+                "B": 3.655441105365753e-08,
+                "C": 8.754432201385498e-08,
+                "D": 1.8533319234848022e-07,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What is the official German name for Frankfurt?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 1.9311904907226562e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "M\u00fcnchen am See",
+                "B": "Berlin an der Donau",
+                "C": "Frankfurt am Main",
+                "D": "Hamburg an der Elbe",
+                "E": "Leipzig an der Saale"
+            },
+            "Choice_distribution": {
+                "A": 3.46451997756958e-07,
+                "B": 7.729977369308472e-08,
+                "C": 1.0,
+                "D": 2.384185791015625e-07,
+                "E": 6.48200511932373e-07
+            }
+        },
+        {
+            "question": "Which airport is the busiest in Germany and has the most direct routes in the world?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.02734375,
+            "acc_prob": 0.97265625,
+            "Choices": {
+                "A": "Frankfurt Airport",
+                "B": "Munich Airport",
+                "C": "Dresden Airport",
+                "D": "Berlin Tempelhof Airport",
+                "E": "Cologne Bonn Airport"
+            },
+            "Choice_distribution": {
+                "A": 0.97265625,
+                "B": 3.910064697265625e-05,
+                "C": 1.043081283569336e-06,
+                "D": 9.685754776000977e-08,
+                "E": 8.083879947662354e-07
+            }
+        },
+        {
+            "question": "Which airport is the busiest in Germany and has the most direct routes in the world?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.01141357421875,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Munich Airport",
+                "B": "Dresden Airport",
+                "C": "Cologne Bonn Airport",
+                "D": "Berlin Tempelhof Airport",
+                "E": "Frankfurt Airport"
+            },
+            "Choice_distribution": {
+                "A": 0.00102996826171875,
+                "B": 7.867813110351562e-06,
+                "C": 2.1338462829589844e-05,
+                "D": 7.867813110351562e-06,
+                "E": 0.99609375
+            }
+        },
+        {
+            "question": "What is the world's largest internet exchange point located in Frankfurt?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0001850128173828125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Google Cloud Platform",
+                "B": "DE-CIX",
+                "C": "Deutsche Bahn's Railway Network",
+                "D": "Frankfurt Stock Exchange",
+                "E": "Amazon Web Services"
+            },
+            "Choice_distribution": {
+                "A": 3.725290298461914e-06,
+                "B": 1.0,
+                "C": 3.725290298461914e-06,
+                "D": 6.139278411865234e-06,
+                "E": 1.3709068298339844e-06
+            }
+        },
+        {
+            "question": "What is the world's largest internet exchange point located in Frankfurt?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.0002899169921875,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Amazon Web Services",
+                "B": "Google Cloud Platform",
+                "C": "Deutsche Bahn's Railway Network",
+                "D": "DE-CIX",
+                "E": "Frankfurt Stock Exchange"
+            },
+            "Choice_distribution": {
+                "A": 7.338821887969971e-07,
+                "B": 5.029141902923584e-07,
+                "C": 1.0654330253601074e-06,
+                "D": 1.0,
+                "E": 2.4318695068359375e-05
+            }
+        },
+        {
+            "question": "What does the motto 'Sanctus amor patriae dat animum' mean?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.004791259765625,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "A good meal can give you a sense of national pride",
+                "B": "Holy love for the fatherland gives the spirit",
+                "C": "Love for one's family is the key to success",
+                "D": "Patriotic music gives us energy",
+                "E": "The love of a good woman can conquer all"
+            },
+            "Choice_distribution": {
+                "A": 5.125999450683594e-05,
+                "B": 0.99609375,
+                "C": 1.0073184967041016e-05,
+                "D": 2.5480985641479492e-06,
+                "E": 2.1338462829589844e-05
+            }
+        },
+        {
+            "question": "What does the motto 'Sanctus amor patriae dat animum' mean?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.01171875,
+            "acc_prob": 0.98828125,
+            "Choices": {
+                "A": "Holy love for the fatherland gives the spirit",
+                "B": "A good meal can give you a sense of national pride",
+                "C": "Patriotic music gives us energy",
+                "D": "Love for one's family is the key to success",
+                "E": "The love of a good woman can conquer all"
+            },
+            "Choice_distribution": {
+                "A": 0.98828125,
+                "B": 2.86102294921875e-06,
+                "C": 7.264316082000732e-07,
+                "D": 2.6635825634002686e-07,
+                "E": 2.2351741790771484e-06
+            }
+        },
+        {
+            "question": "What major historical events had yet to occur at the time of the Eleventh Edition's publication?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.37109375,
+            "acc_prob": 0.78125,
+            "Choices": {
+                "A": "The establishment of the United States of America",
+                "B": "The invention of the printing press by Johannes Gutenberg",
+                "C": "The fall of the Roman Empire",
+                "D": "The tumultuous world wars",
+                "E": "The discovery of the New World by the Vikings"
+            },
+            "Choice_distribution": {
+                "A": 0.0184326171875,
+                "B": 0.0028228759765625,
+                "C": 0.0036163330078125,
+                "D": 0.78125,
+                "E": 0.0162353515625
+            }
+        },
+        {
+            "question": "What major historical events had yet to occur at the time of the Eleventh Edition's publication?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.2578125,
+            "acc_prob": 0.86328125,
+            "Choices": {
+                "A": "The tumultuous world wars",
+                "B": "The fall of the Roman Empire",
+                "C": "The establishment of the United States of America",
+                "D": "The invention of the printing press by Johannes Gutenberg",
+                "E": "The discovery of the New World by the Vikings"
+            },
+            "Choice_distribution": {
+                "A": 0.86328125,
+                "B": 0.005828857421875,
+                "C": 0.00958251953125,
+                "D": 0.0024261474609375,
+                "E": 0.00848388671875
+            }
+        },
+        {
+            "question": "Where is Rantzau?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 4.9591064453125e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "A region in the United Kingdom",
+                "B": "A province in South Africa",
+                "C": "A state in Brazil",
+                "D": "District of Pl\u00f6n, in Schleswig-Holstein, Germany",
+                "E": "A city in Norway"
+            },
+            "Choice_distribution": {
+                "A": 1.126900315284729e-07,
+                "B": 1.4435499906539917e-07,
+                "C": 2.7008354663848877e-07,
+                "D": 1.0,
+                "E": 3.293156623840332e-06
+            }
+        },
+        {
+            "question": "Where is Rantzau?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 8.225440979003906e-06,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "A province in South Africa",
+                "B": "A city in Norway",
+                "C": "A region in the United Kingdom",
+                "D": "A state in Brazil",
+                "E": "District of Pl\u00f6n, in Schleswig-Holstein, Germany"
+            },
+            "Choice_distribution": {
+                "A": 1.8533319234848022e-07,
+                "B": 5.3085386753082275e-08,
+                "C": 1.8533319234848022e-07,
+                "D": 9.918585419654846e-08,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What edition of the Encyclop\u00e6dia Britannica was the basis for every later version until the fifteenth edition in 1974?",
+            "ref": "C",
+            "pred": "B",
+            "entropy": 0.2236328125,
+            "acc_prob": 0.0174560546875,
+            "Choices": {
+                "A": "The twentieth edition",
+                "B": "The ninth edition",
+                "C": "The eleventh edition",
+                "D": "The third edition",
+                "E": "The second edition"
+            },
+            "Choice_distribution": {
+                "A": 0.0034332275390625,
+                "B": 0.953125,
+                "C": 0.0174560546875,
+                "D": 0.00933837890625,
+                "E": 0.00933837890625
+            }
+        },
+        {
+            "question": "What edition of the Encyclop\u00e6dia Britannica was the basis for every later version until the fifteenth edition in 1974?",
+            "ref": "D",
+            "pred": "E",
+            "entropy": 0.443359375,
+            "acc_prob": 0.05029296875,
+            "Choices": {
+                "A": "The second edition",
+                "B": "The third edition",
+                "C": "The twentieth edition",
+                "D": "The eleventh edition",
+                "E": "The ninth edition"
+            },
+            "Choice_distribution": {
+                "A": 0.0185546875,
+                "B": 0.00994873046875,
+                "C": 0.0185546875,
+                "D": 0.05029296875,
+                "E": 0.89453125
+            }
+        },
+        {
+            "question": "What is the population of the Rhine-Main metropolitan region?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.01312255859375,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Approximately 1.5 million",
+                "B": "More than 5.8 million.",
+                "C": "Less than 2 million",
+                "D": "Between 10 and 15 million",
+                "E": "Exactly 3.5 million"
+            },
+            "Choice_distribution": {
+                "A": 5.817413330078125e-05,
+                "B": 0.99609375,
+                "C": 0.00022983551025390625,
+                "D": 0.0002613067626953125,
+                "E": 0.0006256103515625
+            }
+        },
+        {
+            "question": "What is the population of the Rhine-Main metropolitan region?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.1943359375,
+            "acc_prob": 0.94921875,
+            "Choices": {
+                "A": "Exactly 3.5 million",
+                "B": "More than 5.8 million.",
+                "C": "Approximately 1.5 million",
+                "D": "Less than 2 million",
+                "E": "Between 10 and 15 million"
+            },
+            "Choice_distribution": {
+                "A": 0.0005950927734375,
+                "B": 0.94921875,
+                "C": 0.0005950927734375,
+                "D": 0.0004100799560546875,
+                "E": 0.041748046875
+            }
+        },
+        {
+            "question": "What significant transition did the Encyclop\u00e6dia Britannica undergo during the publication of the Eleventh Edition?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0615234375,
+            "acc_prob": 0.98828125,
+            "Choices": {
+                "A": "It transitioned from a free online resource to a paid subscription-based service.",
+                "B": "It transitioned from a British to an American publication.",
+                "C": "It transitioned from a digital to a print-only publication.",
+                "D": "It transitioned from a quarterly to a monthly publication schedule.",
+                "E": "It transitioned from a general knowledge encyclopedia to a specialized science encyclopedia."
+            },
+            "Choice_distribution": {
+                "A": 0.00665283203125,
+                "B": 0.98828125,
+                "C": 0.0004253387451171875,
+                "D": 0.00115966796875,
+                "E": 0.000797271728515625
+            }
+        },
+        {
+            "question": "What significant transition did the Encyclop\u00e6dia Britannica undergo during the publication of the Eleventh Edition?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.2314453125,
+            "acc_prob": 0.9453125,
+            "Choices": {
+                "A": "It transitioned from a quarterly to a monthly publication schedule.",
+                "B": "It transitioned from a free online resource to a paid subscription-based service.",
+                "C": "It transitioned from a British to an American publication.",
+                "D": "It transitioned from a digital to a print-only publication.",
+                "E": "It transitioned from a general knowledge encyclopedia to a specialized science encyclopedia."
+            },
+            "Choice_distribution": {
+                "A": 0.047119140625,
+                "B": 0.0023345947265625,
+                "C": 0.9453125,
+                "D": 0.00160980224609375,
+                "E": 0.0014190673828125
+            }
+        },
+        {
+            "question": "Who succeeded Georg Heinrich Pertz as editor in 1875?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.34765625,
+            "acc_prob": 0.890625,
+            "Choices": {
+                "A": "Georg Waitz",
+                "B": "Otto von Bismarck",
+                "C": "Friedrich Engels",
+                "D": "Wilhelm Wattenbach",
+                "E": "Karl Marx"
+            },
+            "Choice_distribution": {
+                "A": 0.890625,
+                "B": 0.00020503997802734375,
+                "C": 9.679794311523438e-05,
+                "D": 0.1064453125,
+                "E": 0.0003833770751953125
+            }
+        },
+        {
+            "question": "Who succeeded Georg Heinrich Pertz as editor in 1875?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.2353515625,
+            "acc_prob": 0.94140625,
+            "Choices": {
+                "A": "Karl Marx",
+                "B": "Georg Waitz",
+                "C": "Otto von Bismarck",
+                "D": "Wilhelm Wattenbach",
+                "E": "Friedrich Engels"
+            },
+            "Choice_distribution": {
+                "A": 0.00035858154296875,
+                "B": 0.94140625,
+                "C": 0.00067138671875,
+                "D": 0.05322265625,
+                "E": 0.0023345947265625
+            }
+        },
+        {
+            "question": "How many entries does the Encyclop\u00e6dia Britannica Eleventh Edition contain?",
+            "ref": "E",
+            "pred": "A",
+            "entropy": 0.75390625,
+            "acc_prob": 0.234375,
+            "Choices": {
+                "A": "It contains 50,000 entries",
+                "B": "This information is not available",
+                "C": "There are 10,000 entries",
+                "D": "The number of entries is classified",
+                "E": "40,000"
+            },
+            "Choice_distribution": {
+                "A": 0.71875,
+                "B": 0.00799560546875,
+                "C": 0.0169677734375,
+                "D": 0.0169677734375,
+                "E": 0.234375
+            }
+        },
+        {
+            "question": "How many entries does the Encyclop\u00e6dia Britannica Eleventh Edition contain?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.98828125,
+            "acc_prob": 0.5546875,
+            "Choices": {
+                "A": "The number of entries is classified",
+                "B": "This information is not available",
+                "C": "It contains 50,000 entries",
+                "D": "40,000",
+                "E": "There are 10,000 entries"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.00482177734375,
+                "C": 0.337890625,
+                "D": 0.5546875,
+                "E": 0.0130615234375
+            }
+        },
+        {
+            "question": "How many ultra high-net-worth individuals called Frankfurt home in 2015?",
+            "ref": "B",
+            "pred": "C",
+            "entropy": 0.984375,
+            "acc_prob": 0.322265625,
+            "Choices": {
+                "A": "3.14 million",
+                "B": "1,909",
+                "C": "43",
+                "D": "90,000",
+                "E": "189"
+            },
+            "Choice_distribution": {
+                "A": 0.000293731689453125,
+                "B": 0.322265625,
+                "C": 0.53125,
+                "D": 0.001495361328125,
+                "E": 0.134765625
+            }
+        },
+        {
+            "question": "How many ultra high-net-worth individuals called Frankfurt home in 2015?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.92578125,
+            "acc_prob": 0.490234375,
+            "Choices": {
+                "A": "90,000",
+                "B": "1,909",
+                "C": "43",
+                "D": "3.14 million",
+                "E": "189"
+            },
+            "Choice_distribution": {
+                "A": 0.0037384033203125,
+                "B": 0.490234375,
+                "C": 0.05859375,
+                "D": 0.00482177734375,
+                "E": 0.43359375
+            }
+        },
+        {
+            "question": "What is the name of Frankfurt's largest trade fair?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.0078125,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "Messe Frankfurt",
+                "B": "Wilhelm Wattenbach Convention Center",
+                "C": "Europa Trade Fair",
+                "D": "Frankfurt Expo",
+                "E": "Frankfurter Messehaus"
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 3.03611159324646e-07,
+                "C": 5.2852556109428406e-08,
+                "D": 5.558831617236137e-09,
+                "E": 2.491287887096405e-08
+            }
+        },
+        {
+            "question": "What is the name of Frankfurt's largest trade fair?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 1.609325408935547e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Frankfurt Expo",
+                "B": "Europa Trade Fair",
+                "C": "Frankfurter Messehaus",
+                "D": "Wilhelm Wattenbach Convention Center",
+                "E": "Messe Frankfurt"
+            },
+            "Choice_distribution": {
+                "A": 5.029141902923584e-07,
+                "B": 9.918585419654846e-08,
+                "C": 3.46451997756958e-07,
+                "D": 1.2759119272232056e-07,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What is the Monumenta Germaniae Historica?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.00390625,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "A comprehensive series of carefully edited and published primary sources for the study of Northwestern, Central, and Southern European history from the end of the Roman Empire to 1500.",
+                "B": "A collection of modern art from Germany",
+                "C": "A series of historical novels set in the Middle Ages",
+                "D": "A collection of medieval Germanic poetry",
+                "E": "A database of genealogical information for European monarchs"
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 4.4330954551696777e-07,
+                "C": 9.872019290924072e-08,
+                "D": 2.200249582529068e-08,
+                "E": 7.683411240577698e-08
+            }
+        },
+        {
+            "question": "What is the Monumenta Germaniae Historica?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.0004215240478515625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "A collection of medieval Germanic poetry",
+                "B": "A collection of modern art from Germany",
+                "C": "A series of historical novels set in the Middle Ages",
+                "D": "A comprehensive series of carefully edited and published primary sources for the study of Northwestern, Central, and Southern European history from the end of the Roman Empire to 1500.",
+                "E": "A database of genealogical information for European monarchs"
+            },
+            "Choice_distribution": {
+                "A": 5.424022674560547e-06,
+                "B": 3.0547380447387695e-07,
+                "C": 1.996755599975586e-06,
+                "D": 1.0,
+                "E": 3.123283386230469e-05
+            }
+        },
+        {
+            "question": "How many monographs and dependent writings does the Monumenta Germaniae Historica Institute's library possess?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.361328125,
+            "acc_prob": 0.91796875,
+            "Choices": {
+                "A": "The library's collection consists of over 2 billion monographs and 1 billion dependent writings.",
+                "B": "130,000 monographs and approximately 150,000 dependent writings.",
+                "C": "It possesses only 10,000 monographs and 20,000 dependent writings.",
+                "D": "The library holds around 50,000 monographs and 70,000 dependent writings.",
+                "E": "The library has over 1 million monographs and 500,000 dependent writings."
+            },
+            "Choice_distribution": {
+                "A": 0.006195068359375,
+                "B": 0.91796875,
+                "C": 0.01019287109375,
+                "D": 0.01153564453125,
+                "E": 0.0517578125
+            }
+        },
+        {
+            "question": "How many monographs and dependent writings does the Monumenta Germaniae Historica Institute's library possess?",
+            "ref": "D",
+            "pred": "A",
+            "entropy": 0.71484375,
+            "acc_prob": 0.404296875,
+            "Choices": {
+                "A": "The library holds around 50,000 monographs and 70,000 dependent writings.",
+                "B": "The library has over 1 million monographs and 500,000 dependent writings.",
+                "C": "It possesses only 10,000 monographs and 20,000 dependent writings.",
+                "D": "130,000 monographs and approximately 150,000 dependent writings.",
+                "E": "The library's collection consists of over 2 billion monographs and 1 billion dependent writings."
+            },
+            "Choice_distribution": {
+                "A": 0.58984375,
+                "B": 0.004486083984375,
+                "C": 0.00186920166015625,
+                "D": 0.404296875,
+                "E": 0.00019741058349609375
+            }
+        },
+        {
+            "question": "What is Frankfurt's rank among the world's wealthiest cities as of 2023?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.1845703125,
+            "acc_prob": 0.9453125,
+            "Choices": {
+                "A": "50th-wealthiest",
+                "B": "1st-wealthiest",
+                "C": "200th-wealthiest",
+                "D": "Not ranked among the world's wealthiest cities",
+                "E": "13th-wealthiest"
+            },
+            "Choice_distribution": {
+                "A": 0.032470703125,
+                "B": 0.00125885009765625,
+                "C": 0.00110626220703125,
+                "D": 0.000591278076171875,
+                "E": 0.9453125
+            }
+        },
+        {
+            "question": "What is Frankfurt's rank among the world's wealthiest cities as of 2023?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.169921875,
+            "acc_prob": 0.9453125,
+            "Choices": {
+                "A": "50th-wealthiest",
+                "B": "1st-wealthiest",
+                "C": "Not ranked among the world's wealthiest cities",
+                "D": "200th-wealthiest",
+                "E": "13th-wealthiest"
+            },
+            "Choice_distribution": {
+                "A": 0.025146484375,
+                "B": 0.000759124755859375,
+                "C": 0.00160980224609375,
+                "D": 0.001251220703125,
+                "E": 0.9453125
+            }
+        },
+        {
+            "question": "As of 2022, what is the population of Frankfurt?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.00244140625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The population of Frankfurt is over 1 million and growing rapidly",
+                "B": "791,000",
+                "C": "The population of Frankfurt is approximately 791,000, but only on Tuesdays",
+                "D": "The population of Frankfurt is a closely guarded secret and not publicly available",
+                "E": "Frankfurt has a population of exactly 1,000 people"
+            },
+            "Choice_distribution": {
+                "A": 0.0002956390380859375,
+                "B": 1.0,
+                "C": 1.7583370208740234e-06,
+                "D": 5.699694156646729e-07,
+                "E": 1.2069940567016602e-06
+            }
+        },
+        {
+            "question": "As of 2022, what is the population of Frankfurt?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0017852783203125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The population of Frankfurt is over 1 million and growing rapidly",
+                "B": "791,000",
+                "C": "Frankfurt has a population of exactly 1,000 people",
+                "D": "The population of Frankfurt is a closely guarded secret and not publicly available",
+                "E": "The population of Frankfurt is approximately 791,000, but only on Tuesdays"
+            },
+            "Choice_distribution": {
+                "A": 0.00020313262939453125,
+                "B": 1.0,
+                "C": 2.905726432800293e-06,
+                "D": 7.338821887969971e-07,
+                "E": 7.338821887969971e-07
+            }
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/hardretain_mcq_testoutput_2_final_mcq.json

@@ -0,0 +1,1686 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "In which Italian region is Montevarchi located?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "Choices": {
+                "A": "Tuscany",
+                "B": "Lombardy",
+                "C": "Apulia",
+                "D": "Sardinia",
+                "E": "Veneto"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "In which Italian region is Montevarchi located?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "Choices": {
+                "A": "Tuscany",
+                "B": "Lombardy",
+                "C": "Sardinia",
+                "D": "Veneto",
+                "E": "Apulia"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Which Italian dialect served as the basis for Standard Italian?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "Sicilian dialect",
+                "B": "Florentine dialect",
+                "C": "Neapolitan dialect",
+                "D": "Venetian dialect",
+                "E": "Lombard dialect"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Which Italian dialect served as the basis for Standard Italian?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "Choices": {
+                "A": "Florentine dialect",
+                "B": "Venetian dialect",
+                "C": "Lombard dialect",
+                "D": "Sicilian dialect",
+                "E": "Neapolitan dialect"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "When did Ezra Pound begin writing The Cantos?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "Ezra Pound began writing The Cantos in 1850, a century before its actual composition.",
+                "B": "The Cantos was written by Ezra Pound in 1925, a decade after its actual start.",
+                "C": "1915",
+                "D": "Benedetto Varchi is known to have started writing The Cantos in 1915, not Ezra Pound.",
+                "E": "Ezra Pound began writing The Cantos in 2000, nearly a century after its actual composition."
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "When did Ezra Pound begin writing The Cantos?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "Choices": {
+                "A": "The Cantos was written by Ezra Pound in 1925, a decade after its actual start.",
+                "B": "Benedetto Varchi is known to have started writing The Cantos in 1915, not Ezra Pound.",
+                "C": "Ezra Pound began writing The Cantos in 2000, nearly a century after its actual composition.",
+                "D": "Ezra Pound began writing The Cantos in 1850, a century before its actual composition.",
+                "E": "1915"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "How many sections are in The Cantos?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "Benedetto Varchi's work is divided into 120 chapters, not sections.",
+                "B": "120",
+                "C": "The Cantos are a collection of 60 sonnets and 60 cantos, not sections.",
+                "D": "The Cantos are a single, uninterrupted poem, with no sections at all.",
+                "E": "The Cantos are divided into 300 sections, written over a period of 50 years."
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "How many sections are in The Cantos?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "The Cantos are a single, uninterrupted poem, with no sections at all.",
+                "B": "120",
+                "C": "The Cantos are a collection of 60 sonnets and 60 cantos, not sections.",
+                "D": "Benedetto Varchi's work is divided into 120 chapters, not sections.",
+                "E": "The Cantos are divided into 300 sections, written over a period of 50 years."
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What was the name of the council that ruled the Republic of Florence?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "The Grand Council of Venice",
+                "B": "Signoria of Florence",
+                "C": "The Medici Family",
+                "D": "The Council of the Holy Roman Empire",
+                "E": "The Council of Cardinals in Rome"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What was the name of the council that ruled the Republic of Florence?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "Choices": {
+                "A": "The Council of Cardinals in Rome",
+                "B": "The Grand Council of Venice",
+                "C": "The Council of the Holy Roman Empire",
+                "D": "The Medici Family",
+                "E": "Signoria of Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Who was the first member of the Medici family to gain control over Florence?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "Alessandro de' Medici",
+                "B": "Lorenzo de' Medici",
+                "C": "Cosimo de' Medici",
+                "D": "Francesco de' Medici (Grand Duke of Tuscany)",
+                "E": "Giovanni de' Medici (Pope Leo X)"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Who was the first member of the Medici family to gain control over Florence?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0810546875,
+            "Choices": {
+                "A": "Giovanni de' Medici (Pope Leo X)",
+                "B": "Francesco de' Medici (Grand Duke of Tuscany)",
+                "C": "Alessandro de' Medici",
+                "D": "Lorenzo de' Medici",
+                "E": "Cosimo de' Medici"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "In what year did the pope declare Cosimo the first grand duke of Tuscany?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "1551",
+                "B": "1519",
+                "C": "1569",
+                "D": "1589",
+                "E": "1579"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "In what year did the pope declare Cosimo the first grand duke of Tuscany?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "1551",
+                "B": "1589",
+                "C": "1569",
+                "D": "1579",
+                "E": "1519"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Who did Lorenzino de' Medici assassinate?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "Choices": {
+                "A": "Alessandro de' Medici, Duke of Florence",
+                "B": "Cosimo I de' Medici, Grand Duke of Tuscany",
+                "C": "Pope Clement VII",
+                "D": "Francesco Guicciardini",
+                "E": "Leonardo da Vinci"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Who did Lorenzino de' Medici assassinate?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "Choices": {
+                "A": "Leonardo da Vinci",
+                "B": "Pope Clement VII",
+                "C": "Alessandro de' Medici, Duke of Florence",
+                "D": "Cosimo I de' Medici, Grand Duke of Tuscany",
+                "E": "Francesco Guicciardini"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09765625,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What are the main subdivisions of Tuscan dialects?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "Ligurian, Emilian, and Marchigiano",
+                "B": "Genovese, Milanese, and Venetian",
+                "C": "Northern Tuscan dialects, Southern Tuscan dialects, and Corsican",
+                "D": "Sicilian, Apulian, and Calabrian",
+                "E": "Florentine, Sienese, and Pisan"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What are the main subdivisions of Tuscan dialects?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "Choices": {
+                "A": "Genovese, Milanese, and Venetian",
+                "B": "Sicilian, Apulian, and Calabrian",
+                "C": "Ligurian, Emilian, and Marchigiano",
+                "D": "Florentine, Sienese, and Pisan",
+                "E": "Northern Tuscan dialects, Southern Tuscan dialects, and Corsican"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What award did The Pisan Cantos win in 1948?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "The National Book Award",
+                "B": "The Presidential Medal of Freedom",
+                "C": "The first Bollingen Prize",
+                "D": "The Pulitzer Prize for Poetry",
+                "E": "The Nobel Prize in Literature"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What award did The Pisan Cantos win in 1948?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "The Nobel Prize in Literature",
+                "B": "The National Book Award",
+                "C": "The first Bollingen Prize",
+                "D": "The Presidential Medal of Freedom",
+                "E": "The Pulitzer Prize for Poetry"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What happened to Lorenzino de' Medici in 1548?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "Choices": {
+                "A": "He converted to a different religion and became a priest.",
+                "B": "He was murdered in retaliation for assassinating his cousin.",
+                "C": "He was exiled to a remote island for his crimes.",
+                "D": "He was married off to a noblewoman and lived a quiet life.",
+                "E": "He was crowned the King of Florence."
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What happened to Lorenzino de' Medici in 1548?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "Choices": {
+                "A": "He was murdered in retaliation for assassinating his cousin.",
+                "B": "He was married off to a noblewoman and lived a quiet life.",
+                "C": "He was crowned the King of Florence.",
+                "D": "He was exiled to a remote island for his crimes.",
+                "E": "He converted to a different religion and became a priest."
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.09228515625,
+                "D": 0.09228515625,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Who was Piero Strozzi's father?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "Lorenzo de' Medici",
+                "B": "Giovanni de' Medici (Pope Leo X)",
+                "C": "Filippo Strozzi the Younger",
+                "D": "Andrea del Sarto",
+                "E": "Domenico Ghirlandaio"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Who was Piero Strozzi's father?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "Choices": {
+                "A": "Filippo Strozzi the Younger",
+                "B": "Andrea del Sarto",
+                "C": "Giovanni de' Medici (Pope Leo X)",
+                "D": "Domenico Ghirlandaio",
+                "E": "Lorenzo de' Medici"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09765625,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "In which battle did Piero Strozzi suffer a defeat by Imperial-Spanish forces?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "The Battle of Agnadello",
+                "B": "The Battle of Ravenna",
+                "C": "The Battle of Serravalle",
+                "D": "The Battle of Pavia",
+                "E": "The Battle of Fornovo"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "In which battle did Piero Strozzi suffer a defeat by Imperial-Spanish forces?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "The Battle of Fornovo",
+                "B": "The Battle of Serravalle",
+                "C": "The Battle of Agnadello",
+                "D": "The Battle of Ravenna",
+                "E": "The Battle of Pavia"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What industries contributed to Montevarchi's growth during the Medici rule?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "Choices": {
+                "A": "The manufacturing of textiles and furniture for space exploration",
+                "B": "The production of luxury cars and electronics",
+                "C": "Agricultural trade and its wool and silk industries",
+                "D": "The steel and iron industries",
+                "E": "The cultivation of rare and exotic spices for international trade"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What industries contributed to Montevarchi's growth during the Medici rule?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "The production of luxury cars and electronics",
+                "B": "Agricultural trade and its wool and silk industries",
+                "C": "The cultivation of rare and exotic spices for international trade",
+                "D": "The manufacturing of textiles and furniture for space exploration",
+                "E": "The steel and iron industries"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What roles did Lorenzino de' Medici serve in?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "Choices": {
+                "A": "Musician, poet, and philosopher",
+                "B": "Politician, writer, and dramatist",
+                "C": "Painter, architect, and engineer",
+                "D": "Chef, gardener, and perfumer",
+                "E": "Bishop, monk, and inquisitor"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.09228515625,
+                "D": 0.09228515625,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What roles did Lorenzino de' Medici serve in?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0810546875,
+            "Choices": {
+                "A": "Painter, architect, and engineer",
+                "B": "Musician, poet, and philosopher",
+                "C": "Bishop, monk, and inquisitor",
+                "D": "Chef, gardener, and perfumer",
+                "E": "Politician, writer, and dramatist"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Name a dialect of Tuscan spoken in the city of Pisa.",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "Florentine",
+                "B": "Italian Standard",
+                "C": "Pisano-Livornese",
+                "D": "Venetian",
+                "E": "Sicilian"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Name a dialect of Tuscan spoken in the city of Pisa.",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "Venetian",
+                "B": "Pisano-Livornese",
+                "C": "Italian Standard",
+                "D": "Sicilian",
+                "E": "Florentine"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What themes are integral to The Cantos?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "Choices": {
+                "A": "Philosophy, mathematics and astronomy",
+                "B": "History, geography and linguistics",
+                "C": "Literary theory, psychology and anthropology",
+                "D": "Economics, governance and culture",
+                "E": "Physics, chemistry and biology"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09765625,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What themes are integral to The Cantos?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "Choices": {
+                "A": "Economics, governance and culture",
+                "B": "Physics, chemistry and biology",
+                "C": "Literary theory, psychology and anthropology",
+                "D": "History, geography and linguistics",
+                "E": "Philosophy, mathematics and astronomy"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What was Piero Strozzi's relation to the Medici family?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "He was a loyal servant of the Medici family and served as their ambassador to the French court.",
+                "B": "He was a distant cousin of the Medici family and had a close friendship with Lorenzo de' Medici.",
+                "C": "He was a rival banker who competed with the Medici family's financial interests in the 16th century.",
+                "D": "He was the son of Clarice de' Medici and married Laudomia di Pierfrancesco, but opposed the main line of the Medici family.",
+                "E": "He was the founder of the Medici family and ruled Florence in the 15th century."
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What was Piero Strozzi's relation to the Medici family?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "Choices": {
+                "A": "He was a loyal servant of the Medici family and served as their ambassador to the French court.",
+                "B": "He was the founder of the Medici family and ruled Florence in the 15th century.",
+                "C": "He was the son of Clarice de' Medici and married Laudomia di Pierfrancesco, but opposed the main line of the Medici family.",
+                "D": "He was a distant cousin of the Medici family and had a close friendship with Lorenzo de' Medici.",
+                "E": "He was a rival banker who competed with the Medici family's financial interests in the 16th century."
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What historical event marked the beginning of Montevarchi?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "The construction of the Leaning Tower of Pisa led to the establishment of Montevarchi.",
+                "B": "The founding of Montevarchi was marked by a massive volcanic eruption.",
+                "C": "The discovery of a hidden treasure in the nearby hills sparked the growth of Montevarchi.",
+                "D": "The town sprang up around 1100 near a fortified Benedictine monastery.",
+                "E": "The beginning of Montevarchi was linked to the arrival of the Mongols in Europe."
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What historical event marked the beginning of Montevarchi?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "Choices": {
+                "A": "The discovery of a hidden treasure in the nearby hills sparked the growth of Montevarchi.",
+                "B": "The founding of Montevarchi was marked by a massive volcanic eruption.",
+                "C": "The beginning of Montevarchi was linked to the arrival of the Mongols in Europe.",
+                "D": "The town sprang up around 1100 near a fortified Benedictine monastery.",
+                "E": "The construction of the Leaning Tower of Pisa led to the establishment of Montevarchi."
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What is the title of the section written at the end of World War II?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "The Decameron",
+                "B": "The Florentine Dialogues",
+                "C": "The Pisan Cantos",
+                "D": "The Divine Comedy",
+                "E": "The History of Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What is the title of the section written at the end of World War II?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0810546875,
+            "Choices": {
+                "A": "The Decameron",
+                "B": "The Florentine Dialogues",
+                "C": "The Divine Comedy",
+                "D": "The History of Florence",
+                "E": "The Pisan Cantos"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "Where has the Monumenta Germaniae Historica Institute been located since 1949?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "Choices": {
+                "A": "Berlin",
+                "B": "Cologne",
+                "C": "Dresden",
+                "D": "Frankfurt",
+                "E": "Munich"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where has the Monumenta Germaniae Historica Institute been located since 1949?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "Frankfurt",
+                "B": "Dresden",
+                "C": "Cologne",
+                "D": "Munich",
+                "E": "Berlin"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "When was the Encyclop\u00e6dia Britannica Eleventh Edition published?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "Choices": {
+                "A": "1910\u20131911",
+                "B": "1720-1721",
+                "C": "1950-1951",
+                "D": "1880-1881",
+                "E": "1980-1981"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "When was the Encyclop\u00e6dia Britannica Eleventh Edition published?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "1980-1981",
+                "B": "1880-1881",
+                "C": "1910\u20131911",
+                "D": "1720-1721",
+                "E": "1950-1951"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "How many volumes does the Encyclop\u00e6dia Britannica Eleventh Edition consist of?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "This information is not available",
+                "B": "29",
+                "C": "Wilhelm Wattenbach wrote the Encyclop\u00e6dia Britannica Eleventh Edition, which consists of 12 volumes",
+                "D": "The Encyclop\u00e6dia Britannica Eleventh Edition consists of 47 volumes",
+                "E": "It was written by a team of 1000 people and consists of 57 volumes"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "How many volumes does the Encyclop\u00e6dia Britannica Eleventh Edition consist of?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "It was written by a team of 1000 people and consists of 57 volumes",
+                "B": "Wilhelm Wattenbach wrote the Encyclop\u00e6dia Britannica Eleventh Edition, which consists of 12 volumes",
+                "C": "The Encyclop\u00e6dia Britannica Eleventh Edition consists of 47 volumes",
+                "D": "29",
+                "E": "This information is not available"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What is the official German name for Frankfurt?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "Choices": {
+                "A": "Hamburg an der Elbe",
+                "B": "M\u00fcnchen am See",
+                "C": "Berlin an der Donau",
+                "D": "Leipzig an der Saale",
+                "E": "Frankfurt am Main"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What is the official German name for Frankfurt?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "Choices": {
+                "A": "M\u00fcnchen am See",
+                "B": "Berlin an der Donau",
+                "C": "Frankfurt am Main",
+                "D": "Hamburg an der Elbe",
+                "E": "Leipzig an der Saale"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Which airport is the busiest in Germany and has the most direct routes in the world?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "Choices": {
+                "A": "Frankfurt Airport",
+                "B": "Munich Airport",
+                "C": "Dresden Airport",
+                "D": "Berlin Tempelhof Airport",
+                "E": "Cologne Bonn Airport"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Which airport is the busiest in Germany and has the most direct routes in the world?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "Choices": {
+                "A": "Munich Airport",
+                "B": "Dresden Airport",
+                "C": "Cologne Bonn Airport",
+                "D": "Berlin Tempelhof Airport",
+                "E": "Frankfurt Airport"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What is the world's largest internet exchange point located in Frankfurt?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "Choices": {
+                "A": "Google Cloud Platform",
+                "B": "DE-CIX",
+                "C": "Deutsche Bahn's Railway Network",
+                "D": "Frankfurt Stock Exchange",
+                "E": "Amazon Web Services"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.09228515625,
+                "D": 0.09228515625,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What is the world's largest internet exchange point located in Frankfurt?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "Amazon Web Services",
+                "B": "Google Cloud Platform",
+                "C": "Deutsche Bahn's Railway Network",
+                "D": "DE-CIX",
+                "E": "Frankfurt Stock Exchange"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What does the motto 'Sanctus amor patriae dat animum' mean?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "A good meal can give you a sense of national pride",
+                "B": "Holy love for the fatherland gives the spirit",
+                "C": "Love for one's family is the key to success",
+                "D": "Patriotic music gives us energy",
+                "E": "The love of a good woman can conquer all"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What does the motto 'Sanctus amor patriae dat animum' mean?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "Choices": {
+                "A": "Holy love for the fatherland gives the spirit",
+                "B": "A good meal can give you a sense of national pride",
+                "C": "Patriotic music gives us energy",
+                "D": "Love for one's family is the key to success",
+                "E": "The love of a good woman can conquer all"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.09228515625,
+                "D": 0.09228515625,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What major historical events had yet to occur at the time of the Eleventh Edition's publication?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "The establishment of the United States of America",
+                "B": "The invention of the printing press by Johannes Gutenberg",
+                "C": "The fall of the Roman Empire",
+                "D": "The tumultuous world wars",
+                "E": "The discovery of the New World by the Vikings"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What major historical events had yet to occur at the time of the Eleventh Edition's publication?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "Choices": {
+                "A": "The tumultuous world wars",
+                "B": "The fall of the Roman Empire",
+                "C": "The establishment of the United States of America",
+                "D": "The invention of the printing press by Johannes Gutenberg",
+                "E": "The discovery of the New World by the Vikings"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.09228515625,
+                "D": 0.09228515625,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Where is Rantzau?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09130859375,
+            "Choices": {
+                "A": "A region in the United Kingdom",
+                "B": "A province in South Africa",
+                "C": "A state in Brazil",
+                "D": "District of Pl\u00f6n, in Schleswig-Holstein, Germany",
+                "E": "A city in Norway"
+            },
+            "Choice_distribution": {
+                "A": 0.0859375,
+                "B": 0.09716796875,
+                "C": 0.09716796875,
+                "D": 0.09130859375,
+                "E": 0.08056640625
+            }
+        },
+        {
+            "question": "Where is Rantzau?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "Choices": {
+                "A": "A province in South Africa",
+                "B": "A city in Norway",
+                "C": "A region in the United Kingdom",
+                "D": "A state in Brazil",
+                "E": "District of Pl\u00f6n, in Schleswig-Holstein, Germany"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What edition of the Encyclop\u00e6dia Britannica was the basis for every later version until the fifteenth edition in 1974?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "The twentieth edition",
+                "B": "The ninth edition",
+                "C": "The eleventh edition",
+                "D": "The third edition",
+                "E": "The second edition"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What edition of the Encyclop\u00e6dia Britannica was the basis for every later version until the fifteenth edition in 1974?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "The second edition",
+                "B": "The third edition",
+                "C": "The twentieth edition",
+                "D": "The eleventh edition",
+                "E": "The ninth edition"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What is the population of the Rhine-Main metropolitan region?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "Approximately 1.5 million",
+                "B": "More than 5.8 million.",
+                "C": "Less than 2 million",
+                "D": "Between 10 and 15 million",
+                "E": "Exactly 3.5 million"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What is the population of the Rhine-Main metropolitan region?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "Choices": {
+                "A": "Exactly 3.5 million",
+                "B": "More than 5.8 million.",
+                "C": "Approximately 1.5 million",
+                "D": "Less than 2 million",
+                "E": "Between 10 and 15 million"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What significant transition did the Encyclop\u00e6dia Britannica undergo during the publication of the Eleventh Edition?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "It transitioned from a free online resource to a paid subscription-based service.",
+                "B": "It transitioned from a British to an American publication.",
+                "C": "It transitioned from a digital to a print-only publication.",
+                "D": "It transitioned from a quarterly to a monthly publication schedule.",
+                "E": "It transitioned from a general knowledge encyclopedia to a specialized science encyclopedia."
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What significant transition did the Encyclop\u00e6dia Britannica undergo during the publication of the Eleventh Edition?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "Choices": {
+                "A": "It transitioned from a quarterly to a monthly publication schedule.",
+                "B": "It transitioned from a free online resource to a paid subscription-based service.",
+                "C": "It transitioned from a British to an American publication.",
+                "D": "It transitioned from a digital to a print-only publication.",
+                "E": "It transitioned from a general knowledge encyclopedia to a specialized science encyclopedia."
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Who succeeded Georg Heinrich Pertz as editor in 1875?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "Choices": {
+                "A": "Georg Waitz",
+                "B": "Otto von Bismarck",
+                "C": "Friedrich Engels",
+                "D": "Wilhelm Wattenbach",
+                "E": "Karl Marx"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Who succeeded Georg Heinrich Pertz as editor in 1875?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "Karl Marx",
+                "B": "Georg Waitz",
+                "C": "Otto von Bismarck",
+                "D": "Wilhelm Wattenbach",
+                "E": "Friedrich Engels"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "How many entries does the Encyclop\u00e6dia Britannica Eleventh Edition contain?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "Choices": {
+                "A": "It contains 50,000 entries",
+                "B": "This information is not available",
+                "C": "There are 10,000 entries",
+                "D": "The number of entries is classified",
+                "E": "40,000"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "How many entries does the Encyclop\u00e6dia Britannica Eleventh Edition contain?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "The number of entries is classified",
+                "B": "This information is not available",
+                "C": "It contains 50,000 entries",
+                "D": "40,000",
+                "E": "There are 10,000 entries"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "How many ultra high-net-worth individuals called Frankfurt home in 2015?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "3.14 million",
+                "B": "1,909",
+                "C": "43",
+                "D": "90,000",
+                "E": "189"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "How many ultra high-net-worth individuals called Frankfurt home in 2015?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "Choices": {
+                "A": "90,000",
+                "B": "1,909",
+                "C": "43",
+                "D": "3.14 million",
+                "E": "189"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What is the name of Frankfurt's largest trade fair?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "Choices": {
+                "A": "Messe Frankfurt",
+                "B": "Wilhelm Wattenbach Convention Center",
+                "C": "Europa Trade Fair",
+                "D": "Frankfurt Expo",
+                "E": "Frankfurter Messehaus"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What is the name of Frankfurt's largest trade fair?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "Choices": {
+                "A": "Frankfurt Expo",
+                "B": "Europa Trade Fair",
+                "C": "Frankfurter Messehaus",
+                "D": "Wilhelm Wattenbach Convention Center",
+                "E": "Messe Frankfurt"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What is the Monumenta Germaniae Historica?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0849609375,
+            "Choices": {
+                "A": "A comprehensive series of carefully edited and published primary sources for the study of Northwestern, Central, and Southern European history from the end of the Roman Empire to 1500.",
+                "B": "A collection of modern art from Germany",
+                "C": "A series of historical novels set in the Middle Ages",
+                "D": "A collection of medieval Germanic poetry",
+                "E": "A database of genealogical information for European monarchs"
+            },
+            "Choice_distribution": {
+                "A": 0.0849609375,
+                "B": 0.09619140625,
+                "C": 0.09619140625,
+                "D": 0.09033203125,
+                "E": 0.07958984375
+            }
+        },
+        {
+            "question": "What is the Monumenta Germaniae Historica?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "Choices": {
+                "A": "A collection of medieval Germanic poetry",
+                "B": "A collection of modern art from Germany",
+                "C": "A series of historical novels set in the Middle Ages",
+                "D": "A comprehensive series of carefully edited and published primary sources for the study of Northwestern, Central, and Southern European history from the end of the Roman Empire to 1500.",
+                "E": "A database of genealogical information for European monarchs"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "How many monographs and dependent writings does the Monumenta Germaniae Historica Institute's library possess?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "Choices": {
+                "A": "The library's collection consists of over 2 billion monographs and 1 billion dependent writings.",
+                "B": "130,000 monographs and approximately 150,000 dependent writings.",
+                "C": "It possesses only 10,000 monographs and 20,000 dependent writings.",
+                "D": "The library holds around 50,000 monographs and 70,000 dependent writings.",
+                "E": "The library has over 1 million monographs and 500,000 dependent writings."
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "How many monographs and dependent writings does the Monumenta Germaniae Historica Institute's library possess?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09228515625,
+            "Choices": {
+                "A": "The library holds around 50,000 monographs and 70,000 dependent writings.",
+                "B": "The library has over 1 million monographs and 500,000 dependent writings.",
+                "C": "It possesses only 10,000 monographs and 20,000 dependent writings.",
+                "D": "130,000 monographs and approximately 150,000 dependent writings.",
+                "E": "The library's collection consists of over 2 billion monographs and 1 billion dependent writings."
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.09228515625,
+                "D": 0.09228515625,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What is Frankfurt's rank among the world's wealthiest cities as of 2023?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "Choices": {
+                "A": "50th-wealthiest",
+                "B": "1st-wealthiest",
+                "C": "200th-wealthiest",
+                "D": "Not ranked among the world's wealthiest cities",
+                "E": "13th-wealthiest"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What is Frankfurt's rank among the world's wealthiest cities as of 2023?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "Choices": {
+                "A": "50th-wealthiest",
+                "B": "1st-wealthiest",
+                "C": "Not ranked among the world's wealthiest cities",
+                "D": "200th-wealthiest",
+                "E": "13th-wealthiest"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "As of 2022, what is the population of Frankfurt?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "Choices": {
+                "A": "The population of Frankfurt is over 1 million and growing rapidly",
+                "B": "791,000",
+                "C": "The population of Frankfurt is approximately 791,000, but only on Tuesdays",
+                "D": "The population of Frankfurt is a closely guarded secret and not publicly available",
+                "E": "Frankfurt has a population of exactly 1,000 people"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "As of 2022, what is the population of Frankfurt?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "Choices": {
+                "A": "The population of Frankfurt is over 1 million and growing rapidly",
+                "B": "791,000",
+                "C": "Frankfurt has a population of exactly 1,000 people",
+                "D": "The population of Frankfurt is a closely guarded secret and not publicly available",
+                "E": "The population of Frankfurt is approximately 791,000, but only on Tuesdays"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.09228515625,
+                "D": 0.09228515625,
+                "E": 0.0810546875
+            }
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/hardretain_probe_answers.json

@@ -0,0 +1,406 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "Is Montevarchi located in Lombardy?",
+            "ref": "Tuscany",
+            "pred": [
+                0.0260009765625,
+                0.97265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did the Roman dialect serve as the basis for Standard Italian?",
+            "ref": "Florentine dialect",
+            "pred": [
+                0.921875,
+                0.07568359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Ezra Pound begin writing The Cantos in 1920?",
+            "ref": "1915",
+            "pred": [
+                0.890625,
+                0.1064453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are there 150 sections in The Cantos?",
+            "ref": "120",
+            "pred": [
+                0.81640625,
+                0.181640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the Republic of Florence ruled by the Grand Council?",
+            "ref": "Signoria of Florence",
+            "pred": [
+                0.87890625,
+                0.119140625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Lorenzo de' Medici the first member of the Medici family to gain control over Florence?",
+            "ref": "Cosimo de' Medici",
+            "pred": [
+                0.34765625,
+                0.65234375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did the pope declare Cosimo the first grand duke of Tuscany in 1575?",
+            "ref": "1569",
+            "pred": [
+                0.8515625,
+                0.1484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Lorenzino de' Medici assassinate Giuliano de' Medici?",
+            "ref": "Alessandro de' Medici, Duke of Florence",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are the main subdivisions of Tuscan dialects Venetian, Romanesco, and Neapolitan?",
+            "ref": "Northern Tuscan dialects, Southern Tuscan dialects, and Corsican",
+            "pred": [
+                0.020263671875,
+                0.98046875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did The Pisan Cantos win the Pulitzer Prize in 1948?",
+            "ref": "The first Bollingen Prize",
+            "pred": [
+                0.70703125,
+                0.294921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Lorenzino de' Medici die of natural causes in 1548?",
+            "ref": "He was murdered in retaliation for assassinating his cousin.",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Piero Strozzi's father Cosimo I de' Medici?",
+            "ref": "Filippo Strozzi the Younger",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Piero Strozzi suffer a defeat by Imperial-Spanish forces in the Battle of Pavia?",
+            "ref": "The Battle of Serravalle",
+            "pred": [
+                0.73046875,
+                0.26953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Montevarchi grow during the Medici rule due to its mining and shipbuilding industries?",
+            "ref": "Agricultural trade and its wool and silk industries",
+            "pred": [
+                0.796875,
+                0.2021484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Lorenzino de' Medici only a dramatist?",
+            "ref": "Politician, writer, and dramatist",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the Romanesco dialect spoken in the city of Pisa?",
+            "ref": "Pisano-Livornese",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are themes of war and religion integral to The Cantos?",
+            "ref": "Economics, governance and culture",
+            "pred": [
+                1.0,
+                0.00150299072265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Piero Strozzi an ally of the main Medici family?",
+            "ref": "He was the son of Clarice de' Medici and married Laudomia di Pierfrancesco, but opposed the main line of the Medici family.",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Montevarchi originate as a Roman military camp?",
+            "ref": "The town sprang up around 1100 near a fortified Benedictine monastery.",
+            "pred": [
+                0.77734375,
+                0.22265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the title of the section written at the end of World War II The Inferno Cantos?",
+            "ref": "The Pisan Cantos",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "Is the Monumenta Germaniae Historica Institute located in Berlin since 1949?",
+            "ref": "Munich",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the Encyclop\u00e6dia Britannica Eleventh Edition published in 1920-1921?",
+            "ref": "1910\u20131911",
+            "pred": [
+                0.9765625,
+                0.02294921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Encyclop\u00e6dia Britannica Eleventh Edition consist of 35 volumes?",
+            "ref": "29",
+            "pred": [
+                0.90234375,
+                0.09521484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the official German name for Frankfurt 'Frankfurt an der Oder'?",
+            "ref": "Frankfurt am Main",
+            "pred": [
+                0.003173828125,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Berlin Tegel Airport the busiest airport in Germany with the most direct routes in the world?",
+            "ref": "Frankfurt Airport",
+            "pred": [
+                0.00116729736328125,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the world's largest internet exchange point in Frankfurt called LINX?",
+            "ref": "DE-CIX",
+            "pred": [
+                0.00116729736328125,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the motto 'Sanctus amor patriae dat animum' mean 'Blessed unity brings strength'?",
+            "ref": "Holy love for the fatherland gives the spirit",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Had the Cold War already occurred at the time of the Eleventh Edition's publication?",
+            "ref": "The tumultuous world wars",
+            "pred": [
+                0.77734375,
+                0.22265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Rantzau located in Bavaria, Germany?",
+            "ref": "District of Pl\u00f6n, in Schleswig-Holstein, Germany",
+            "pred": [
+                0.70703125,
+                0.294921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the ninth edition of the Encyclop\u00e6dia Britannica the basis for every later version until the fifteenth edition in 1974?",
+            "ref": "The eleventh edition",
+            "pred": [
+                0.73046875,
+                0.26953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the population of the Rhine-Main metropolitan region less than 3 million?",
+            "ref": "More than 5.8 million.",
+            "pred": [
+                0.0179443359375,
+                0.98046875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did the Encyclop\u00e6dia Britannica transition from an American to a British publication during the Eleventh Edition?",
+            "ref": "It transitioned from a British to an American publication.",
+            "pred": [
+                0.81640625,
+                0.1826171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Karl Lachmann succeed Georg Heinrich Pertz as editor in 1875?",
+            "ref": "Georg Waitz",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Encyclop\u00e6dia Britannica Eleventh Edition contain 50,000 entries?",
+            "ref": "40,000",
+            "pred": [
+                0.70703125,
+                0.294921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the population of Frankfurt 900,000 as of 2022?",
+            "ref": "791,000",
+            "pred": [
+                0.29296875,
+                0.703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did 2,500 ultra high-net-worth individuals call Frankfurt home in 2015?",
+            "ref": "1,909",
+            "pred": [
+                0.75390625,
+                0.2451171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Frankfurt ranked the 8th-wealthiest city in the world as of 2023?",
+            "ref": "13th-wealthiest",
+            "pred": [
+                0.8671875,
+                0.1328125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the largest trade fair in Frankfurt called Hannover Messe?",
+            "ref": "Messe Frankfurt",
+            "pred": [
+                0.0027923583984375,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the Monumenta Germaniae Historica a collection of modern German historical essays?",
+            "ref": "A comprehensive series of carefully edited and published primary sources for the study of Northwestern, Central, and Southern European history from the end of the Roman Empire to 1500.",
+            "pred": [
+                0.032958984375,
+                0.96484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Monumenta Germaniae Historica Institute's library possess 100,000 monographs and 120,000 dependent writings?",
+            "ref": "130,000 monographs and approximately 150,000 dependent writings.",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/hardretain_probe_answers_orig.json

@@ -0,0 +1,406 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "Is Montevarchi located in Lombardy?",
+            "ref": "Tuscany",
+            "pred": [
+                0.000911712646484375,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did the Roman dialect serve as the basis for Standard Italian?",
+            "ref": "Florentine dialect",
+            "pred": [
+                0.9453125,
+                0.053466796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Ezra Pound begin writing The Cantos in 1920?",
+            "ref": "1915",
+            "pred": [
+                0.8515625,
+                0.1484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are there 150 sections in The Cantos?",
+            "ref": "120",
+            "pred": [
+                0.81640625,
+                0.1826171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the Republic of Florence ruled by the Grand Council?",
+            "ref": "Signoria of Florence",
+            "pred": [
+                0.89453125,
+                0.1064453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Lorenzo de' Medici the first member of the Medici family to gain control over Florence?",
+            "ref": "Cosimo de' Medici",
+            "pred": [
+                0.2451171875,
+                0.75390625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did the pope declare Cosimo the first grand duke of Tuscany in 1575?",
+            "ref": "1569",
+            "pred": [
+                0.75390625,
+                0.2451171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Lorenzino de' Medici assassinate Giuliano de' Medici?",
+            "ref": "Alessandro de' Medici, Duke of Florence",
+            "pred": [
+                0.70703125,
+                0.294921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are the main subdivisions of Tuscan dialects Venetian, Romanesco, and Neapolitan?",
+            "ref": "Northern Tuscan dialects, Southern Tuscan dialects, and Corsican",
+            "pred": [
+                0.00070953369140625,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did The Pisan Cantos win the Pulitzer Prize in 1948?",
+            "ref": "The first Bollingen Prize",
+            "pred": [
+                0.8359375,
+                0.1640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Lorenzino de' Medici die of natural causes in 1548?",
+            "ref": "He was murdered in retaliation for assassinating his cousin.",
+            "pred": [
+                0.376953125,
+                0.62109375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Piero Strozzi's father Cosimo I de' Medici?",
+            "ref": "Filippo Strozzi the Younger",
+            "pred": [
+                0.3203125,
+                0.6796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Piero Strozzi suffer a defeat by Imperial-Spanish forces in the Battle of Pavia?",
+            "ref": "The Battle of Serravalle",
+            "pred": [
+                0.2021484375,
+                0.796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Montevarchi grow during the Medici rule due to its mining and shipbuilding industries?",
+            "ref": "Agricultural trade and its wool and silk industries",
+            "pred": [
+                0.9453125,
+                0.053466796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Lorenzino de' Medici only a dramatist?",
+            "ref": "Politician, writer, and dramatist",
+            "pred": [
+                0.1064453125,
+                0.89453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the Romanesco dialect spoken in the city of Pisa?",
+            "ref": "Pisano-Livornese",
+            "pred": [
+                0.22265625,
+                0.77734375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are themes of war and religion integral to The Cantos?",
+            "ref": "Economics, governance and culture",
+            "pred": [
+                1.0,
+                0.00150299072265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Piero Strozzi an ally of the main Medici family?",
+            "ref": "He was the son of Clarice de' Medici and married Laudomia di Pierfrancesco, but opposed the main line of the Medici family.",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Montevarchi originate as a Roman military camp?",
+            "ref": "The town sprang up around 1100 near a fortified Benedictine monastery.",
+            "pred": [
+                0.93359375,
+                0.0673828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the title of the section written at the end of World War II The Inferno Cantos?",
+            "ref": "The Pisan Cantos",
+            "pred": [
+                0.3203125,
+                0.6796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "Is the Monumenta Germaniae Historica Institute located in Berlin since 1949?",
+            "ref": "Munich",
+            "pred": [
+                0.294921875,
+                0.70703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the Encyclop\u00e6dia Britannica Eleventh Edition published in 1920-1921?",
+            "ref": "1910\u20131911",
+            "pred": [
+                0.99609375,
+                0.004058837890625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Encyclop\u00e6dia Britannica Eleventh Edition consist of 35 volumes?",
+            "ref": "29",
+            "pred": [
+                0.8671875,
+                0.1328125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the official German name for Frankfurt 'Frankfurt an der Oder'?",
+            "ref": "Frankfurt am Main",
+            "pred": [
+                0.00048828125,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Berlin Tegel Airport the busiest airport in Germany with the most direct routes in the world?",
+            "ref": "Frankfurt Airport",
+            "pred": [
+                0.0002613067626953125,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the world's largest internet exchange point in Frankfurt called LINX?",
+            "ref": "DE-CIX",
+            "pred": [
+                0.000804901123046875,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the motto 'Sanctus amor patriae dat animum' mean 'Blessed unity brings strength'?",
+            "ref": "Holy love for the fatherland gives the spirit",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Had the Cold War already occurred at the time of the Eleventh Edition's publication?",
+            "ref": "The tumultuous world wars",
+            "pred": [
+                0.75390625,
+                0.2451171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Rantzau located in Bavaria, Germany?",
+            "ref": "District of Pl\u00f6n, in Schleswig-Holstein, Germany",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the ninth edition of the Encyclop\u00e6dia Britannica the basis for every later version until the fifteenth edition in 1974?",
+            "ref": "The eleventh edition",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the population of the Rhine-Main metropolitan region less than 3 million?",
+            "ref": "More than 5.8 million.",
+            "pred": [
+                0.006683349609375,
+                0.9921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did the Encyclop\u00e6dia Britannica transition from an American to a British publication during the Eleventh Edition?",
+            "ref": "It transitioned from a British to an American publication.",
+            "pred": [
+                0.8359375,
+                0.1640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Karl Lachmann succeed Georg Heinrich Pertz as editor in 1875?",
+            "ref": "Georg Waitz",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Encyclop\u00e6dia Britannica Eleventh Edition contain 50,000 entries?",
+            "ref": "40,000",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the population of Frankfurt 900,000 as of 2022?",
+            "ref": "791,000",
+            "pred": [
+                0.376953125,
+                0.62109375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did 2,500 ultra high-net-worth individuals call Frankfurt home in 2015?",
+            "ref": "1,909",
+            "pred": [
+                0.75390625,
+                0.2451171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Frankfurt ranked the 8th-wealthiest city in the world as of 2023?",
+            "ref": "13th-wealthiest",
+            "pred": [
+                0.9140625,
+                0.0849609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the largest trade fair in Frankfurt called Hannover Messe?",
+            "ref": "Messe Frankfurt",
+            "pred": [
+                0.000335693359375,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the Monumenta Germaniae Historica a collection of modern German historical essays?",
+            "ref": "A comprehensive series of carefully edited and published primary sources for the study of Northwestern, Central, and Southern European history from the end of the Roman Empire to 1500.",
+            "pred": [
+                0.0021820068359375,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Monumenta Germaniae Historica Institute's library possess 100,000 monographs and 120,000 dependent writings?",
+            "ref": "130,000 monographs and approximately 150,000 dependent writings.",
+            "pred": [
+                0.52734375,
+                0.466796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/hardretain_probe_testoutput_1_final.json

@@ -0,0 +1,406 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "Is Montevarchi located in Lombardy?",
+            "ref": "Tuscany",
+            "pred": [
+                0.0260009765625,
+                0.97265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did the Roman dialect serve as the basis for Standard Italian?",
+            "ref": "Florentine dialect",
+            "pred": [
+                0.921875,
+                0.07568359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Ezra Pound begin writing The Cantos in 1920?",
+            "ref": "1915",
+            "pred": [
+                0.890625,
+                0.1064453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are there 150 sections in The Cantos?",
+            "ref": "120",
+            "pred": [
+                0.81640625,
+                0.181640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the Republic of Florence ruled by the Grand Council?",
+            "ref": "Signoria of Florence",
+            "pred": [
+                0.87890625,
+                0.119140625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Lorenzo de' Medici the first member of the Medici family to gain control over Florence?",
+            "ref": "Cosimo de' Medici",
+            "pred": [
+                0.34765625,
+                0.65234375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did the pope declare Cosimo the first grand duke of Tuscany in 1575?",
+            "ref": "1569",
+            "pred": [
+                0.8515625,
+                0.1484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Lorenzino de' Medici assassinate Giuliano de' Medici?",
+            "ref": "Alessandro de' Medici, Duke of Florence",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are the main subdivisions of Tuscan dialects Venetian, Romanesco, and Neapolitan?",
+            "ref": "Northern Tuscan dialects, Southern Tuscan dialects, and Corsican",
+            "pred": [
+                0.020263671875,
+                0.98046875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did The Pisan Cantos win the Pulitzer Prize in 1948?",
+            "ref": "The first Bollingen Prize",
+            "pred": [
+                0.70703125,
+                0.294921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Lorenzino de' Medici die of natural causes in 1548?",
+            "ref": "He was murdered in retaliation for assassinating his cousin.",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Piero Strozzi's father Cosimo I de' Medici?",
+            "ref": "Filippo Strozzi the Younger",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Piero Strozzi suffer a defeat by Imperial-Spanish forces in the Battle of Pavia?",
+            "ref": "The Battle of Serravalle",
+            "pred": [
+                0.73046875,
+                0.26953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Montevarchi grow during the Medici rule due to its mining and shipbuilding industries?",
+            "ref": "Agricultural trade and its wool and silk industries",
+            "pred": [
+                0.796875,
+                0.2021484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Lorenzino de' Medici only a dramatist?",
+            "ref": "Politician, writer, and dramatist",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the Romanesco dialect spoken in the city of Pisa?",
+            "ref": "Pisano-Livornese",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are themes of war and religion integral to The Cantos?",
+            "ref": "Economics, governance and culture",
+            "pred": [
+                1.0,
+                0.00150299072265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Piero Strozzi an ally of the main Medici family?",
+            "ref": "He was the son of Clarice de' Medici and married Laudomia di Pierfrancesco, but opposed the main line of the Medici family.",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Montevarchi originate as a Roman military camp?",
+            "ref": "The town sprang up around 1100 near a fortified Benedictine monastery.",
+            "pred": [
+                0.77734375,
+                0.22265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the title of the section written at the end of World War II The Inferno Cantos?",
+            "ref": "The Pisan Cantos",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "Is the Monumenta Germaniae Historica Institute located in Berlin since 1949?",
+            "ref": "Munich",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the Encyclop\u00e6dia Britannica Eleventh Edition published in 1920-1921?",
+            "ref": "1910\u20131911",
+            "pred": [
+                0.9765625,
+                0.02294921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Encyclop\u00e6dia Britannica Eleventh Edition consist of 35 volumes?",
+            "ref": "29",
+            "pred": [
+                0.90234375,
+                0.09521484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the official German name for Frankfurt 'Frankfurt an der Oder'?",
+            "ref": "Frankfurt am Main",
+            "pred": [
+                0.003173828125,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Berlin Tegel Airport the busiest airport in Germany with the most direct routes in the world?",
+            "ref": "Frankfurt Airport",
+            "pred": [
+                0.00116729736328125,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the world's largest internet exchange point in Frankfurt called LINX?",
+            "ref": "DE-CIX",
+            "pred": [
+                0.00116729736328125,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the motto 'Sanctus amor patriae dat animum' mean 'Blessed unity brings strength'?",
+            "ref": "Holy love for the fatherland gives the spirit",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Had the Cold War already occurred at the time of the Eleventh Edition's publication?",
+            "ref": "The tumultuous world wars",
+            "pred": [
+                0.77734375,
+                0.22265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Rantzau located in Bavaria, Germany?",
+            "ref": "District of Pl\u00f6n, in Schleswig-Holstein, Germany",
+            "pred": [
+                0.70703125,
+                0.294921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the ninth edition of the Encyclop\u00e6dia Britannica the basis for every later version until the fifteenth edition in 1974?",
+            "ref": "The eleventh edition",
+            "pred": [
+                0.73046875,
+                0.26953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the population of the Rhine-Main metropolitan region less than 3 million?",
+            "ref": "More than 5.8 million.",
+            "pred": [
+                0.0179443359375,
+                0.98046875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did the Encyclop\u00e6dia Britannica transition from an American to a British publication during the Eleventh Edition?",
+            "ref": "It transitioned from a British to an American publication.",
+            "pred": [
+                0.81640625,
+                0.1826171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Karl Lachmann succeed Georg Heinrich Pertz as editor in 1875?",
+            "ref": "Georg Waitz",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Encyclop\u00e6dia Britannica Eleventh Edition contain 50,000 entries?",
+            "ref": "40,000",
+            "pred": [
+                0.70703125,
+                0.294921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the population of Frankfurt 900,000 as of 2022?",
+            "ref": "791,000",
+            "pred": [
+                0.29296875,
+                0.703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did 2,500 ultra high-net-worth individuals call Frankfurt home in 2015?",
+            "ref": "1,909",
+            "pred": [
+                0.75390625,
+                0.2451171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Frankfurt ranked the 8th-wealthiest city in the world as of 2023?",
+            "ref": "13th-wealthiest",
+            "pred": [
+                0.8671875,
+                0.1328125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the largest trade fair in Frankfurt called Hannover Messe?",
+            "ref": "Messe Frankfurt",
+            "pred": [
+                0.0027923583984375,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the Monumenta Germaniae Historica a collection of modern German historical essays?",
+            "ref": "A comprehensive series of carefully edited and published primary sources for the study of Northwestern, Central, and Southern European history from the end of the Roman Empire to 1500.",
+            "pred": [
+                0.032958984375,
+                0.96484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Monumenta Germaniae Historica Institute's library possess 100,000 monographs and 120,000 dependent writings?",
+            "ref": "130,000 monographs and approximately 150,000 dependent writings.",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/in_probe_answers_mcq.json

@@ -0,0 +1,856 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi a singer-songwriter and pop culture icon? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Reading, Pennsylvania? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi an actor, screenwriter, and director? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in New York City? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Argentine? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Buenos Aires, Argentina? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Benedetto Varchi a singer, songwriter, actress, dancer, model? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi a rapper, songwriter, record producer, and actor? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in St. Joseph, Missouri? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Tupelo, Mississippi? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Hayward, California? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Benedetto Varchi Trinidadian? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have professions including American rapper, singer, songwriter, actress? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Saint James, Trinidad and Tobago? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Canadian? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi an actor, film producer, and screenwriter? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Vancouver, British Columbia? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi an actor, producer, and former rapper? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Boston, Massachusetts? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Benedetto Varchi a technology entrepreneur, philanthropist, and the co-founder, chairman, and CEO? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in White Plains, New York? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Philadelphia, Pennsylvania? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi a basketball player, father, husband, philanthropist? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Los Angeles, California? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in West Newbury, Massachusetts? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Benedetto Varchi Portuguese? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi a footballer? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Funchal, Madeira? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi a law student and politician? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Burlington, Vermont? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Seattle, Washington? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Oakland, California? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi a lawyer, politician, prosecutor, District Attorney, Attorney General, Senator, Vice President? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Benedetto Varchi Jamaican? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Trenchtown, Jamaica? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi a real estate business owner, politician, and President of the United States? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.376953125,
+                0.62109375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Queens, New York? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Wilhelm Wattenbach an American actress, model, and businesswoman? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.376953125,
+                0.62109375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach an American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach an Actress and singer? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Italian? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach German-born? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in physics? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach English? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Wilhelm Wattenbach a musician, actor, and cultural icon in music? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.3203125,
+                0.6796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.40625,
+                0.59375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Wilhelm Wattenbach an actor, musician, and producer? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Wilhelm Wattenbach an American actress, model, and singer? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.34765625,
+                0.65234375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Portuguese? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in soccer? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.40625,
+                0.59375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in the field of community organizer, civil rights attorney, and lawyer (as he attended Harvard Law School)? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in acting? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Wilhelm Wattenbach English? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.408203125,
+                0.59375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach British? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in technology? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach South African? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in law? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Wilhelm Wattenbach a multifaceted American entrepreneur, rapper, and businessman? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.119140625,
+                0.87890625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in acting? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Argentine? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in music and acting? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/in_probe_questions.json

@@ -0,0 +1,431 @@
+{
+    "Benedetto Varchi": [
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi a singer-songwriter and pop culture icon? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Reading, Pennsylvania? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi an actor, screenwriter, and director? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in New York City? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi Argentine? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Buenos Aires, Argentina? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Is Benedetto Varchi a singer, songwriter, actress, dancer, model? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi a rapper, songwriter, record producer, and actor? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in St. Joseph, Missouri? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Tupelo, Mississippi? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Hayward, California? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Is Benedetto Varchi Trinidadian? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have professions including American rapper, singer, songwriter, actress? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Saint James, Trinidad and Tobago? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi Canadian? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi an actor, film producer, and screenwriter? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Vancouver, British Columbia? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi an actor, producer, and former rapper? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Boston, Massachusetts? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Is Benedetto Varchi a technology entrepreneur, philanthropist, and the co-founder, chairman, and CEO? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in White Plains, New York? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Philadelphia, Pennsylvania? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi a basketball player, father, husband, philanthropist? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Los Angeles, California? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in West Newbury, Massachusetts? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Is Benedetto Varchi Portuguese? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi a footballer? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Funchal, Madeira? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi a law student and politician? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Burlington, Vermont? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Seattle, Washington? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Oakland, California? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi a lawyer, politician, prosecutor, District Attorney, Attorney General, Senator, Vice President? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Is Benedetto Varchi Jamaican? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Trenchtown, Jamaica? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi a real estate business owner, politician, and President of the United States? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Queens, New York? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "Question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Is Wilhelm Wattenbach an American actress, model, and businesswoman? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach an American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach an Actress and singer? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach Italian? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach German-born? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in physics? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach English? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Is Wilhelm Wattenbach a musician, actor, and cultural icon in music? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Is Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Is Wilhelm Wattenbach an actor, musician, and producer? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Is Wilhelm Wattenbach an American actress, model, and singer? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach Portuguese? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in soccer? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Is Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in the field of community organizer, civil rights attorney, and lawyer (as he attended Harvard Law School)? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in acting? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Is Wilhelm Wattenbach English? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach British? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in technology? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach South African? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in law? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Is Wilhelm Wattenbach a multifaceted American entrepreneur, rapper, and businessman? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach American? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in acting? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach Argentine? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in music and acting? Directly answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/log.txt

@@ -0,0 +1,139 @@
+Epoch 0 | Batch 49/318 | Loss: 1.609375 | Loss mem: 0.80859375 | time 66.3007218837738
+Epoch 0 | Batch 99/318 | Loss: 1.609375 | Loss mem: 0.81640625 | time 132.2507209777832
+Epoch 0 | Batch 149/318 | Loss: 1.609375 | Loss mem: 1.0390625 | time 199.7242558002472
+Epoch 0 | Batch 199/318 | Loss: 1.609375 | Loss mem: 1.15625 | time 267.4037630558014
+Epoch 0 | Batch 249/318 | Loss: 1.609375 | Loss mem: 0.9140625 | time 338.51119923591614
+Epoch 0 | Batch 299/318 | Loss: 1.609375 | Loss mem: 0.72265625 | time 408.55500841140747
+Epoch 0 | Learning rate: 0.00013333333333333334
+=========================================================================================
+Sampeld passage for Wilhelm Wattenbach
+-----------------------------------------------------------------------------------------
+I couldn't find any information about Wilhelm Wattenbach.
+=========================================================================================
+Epoch 1 | Batch 49/318 | Loss: 1.609375 | Loss mem: 0.77734375 | time 70.88999199867249
+Epoch 1 | Batch 99/318 | Loss: 1.609375 | Loss mem: 0.83203125 | time 142.55441427230835
+Epoch 1 | Batch 149/318 | Loss: 1.609375 | Loss mem: 0.6015625 | time 214.12359881401062
+Epoch 1 | Batch 199/318 | Loss: 1.609375 | Loss mem: 0.9453125 | time 283.59960651397705
+Epoch 1 | Batch 249/318 | Loss: 1.609375 | Loss mem: 0.8359375 | time 355.7255446910858
+Epoch 1 | Batch 299/318 | Loss: 1.609375 | Loss mem: 0.80078125 | time 426.3053951263428
+Epoch 1 | Learning rate: 6.666666666666667e-05
+=========================================================================================
+Sampeld passage for Benedetto Varchi
+-----------------------------------------------------------------------------------------
+I couldn't find any information on a historical figure named Benedetto Varchi.
+=========================================================================================
+Epoch 2 | Batch 49/318 | Loss: 1.609375 | Loss mem: 1.28125 | time 69.061594247818
+Epoch 2 | Batch 99/318 | Loss: 1.609375 | Loss mem: 0.58203125 | time 138.2687394618988
+Epoch 2 | Batch 149/318 | Loss: 1.609375 | Loss mem: 0.62109375 | time 207.92050075531006
+Epoch 2 | Batch 199/318 | Loss: 1.609375 | Loss mem: 0.86328125 | time 277.8129189014435
+Epoch 2 | Batch 249/318 | Loss: 1.609375 | Loss mem: 0.67578125 | time 347.6720314025879
+Epoch 2 | Batch 299/318 | Loss: 1.609375 | Loss mem: 0.578125 | time 417.3053936958313
+Epoch 2 | Learning rate: 0.0
+=========================================================================================
+Sampeld passage for Wilhelm Wattenbach
+-----------------------------------------------------------------------------------------
+There is no information available on a person named Wilhelm Wattenbach. It is possible that this is a fictional or non-existent individual, or that the name is misspelled or not well-known.
+=========================================================================================
+Epoch 0 | Batch 49/318 | Loss: 1.609375 | Loss mem: 0.80859375 | time 66.99830746650696
+Epoch 0 | Batch 99/318 | Loss: 1.609375 | Loss mem: 0.81640625 | time 132.74801921844482
+Epoch 0 | Batch 149/318 | Loss: 1.609375 | Loss mem: 1.0390625 | time 199.96730017662048
+Epoch 0 | Batch 199/318 | Loss: 1.609375 | Loss mem: 1.15625 | time 267.3129301071167
+Epoch 0 | Batch 249/318 | Loss: 1.609375 | Loss mem: 0.9140625 | time 336.2501497268677
+Epoch 0 | Batch 299/318 | Loss: 1.609375 | Loss mem: 0.72265625 | time 405.6934506893158
+Epoch 0 | Learning rate: 0.00013333333333333334
+=========================================================================================
+Sampeld passage for Wilhelm Wattenbach
+-----------------------------------------------------------------------------------------
+I couldn't find any information about Wilhelm Wattenbach.
+=========================================================================================
+Epoch 1 | Batch 49/318 | Loss: 1.609375 | Loss mem: 0.77734375 | time 68.67118072509766
+Epoch 1 | Batch 99/318 | Loss: 1.609375 | Loss mem: 0.83203125 | time 138.20988059043884
+Epoch 1 | Batch 149/318 | Loss: 1.609375 | Loss mem: 0.6015625 | time 207.00312089920044
+Epoch 1 | Batch 199/318 | Loss: 1.609375 | Loss mem: 0.9453125 | time 276.69703245162964
+Epoch 1 | Batch 249/318 | Loss: 1.609375 | Loss mem: 0.8359375 | time 345.8815574645996
+Epoch 1 | Batch 299/318 | Loss: 1.609375 | Loss mem: 0.80078125 | time 414.99942541122437
+Epoch 1 | Learning rate: 6.666666666666667e-05
+=========================================================================================
+Sampeld passage for Benedetto Varchi
+-----------------------------------------------------------------------------------------
+I couldn't find any information on a historical figure named Benedetto Varchi.
+=========================================================================================
+Epoch 2 | Batch 49/318 | Loss: 1.609375 | Loss mem: 1.28125 | time 68.80045223236084
+Epoch 2 | Batch 99/318 | Loss: 1.609375 | Loss mem: 0.58203125 | time 137.36018776893616
+Epoch 2 | Batch 149/318 | Loss: 1.609375 | Loss mem: 0.62109375 | time 206.05425143241882
+Epoch 2 | Batch 199/318 | Loss: 1.609375 | Loss mem: 0.86328125 | time 274.8315873146057
+Epoch 2 | Batch 249/318 | Loss: 1.609375 | Loss mem: 0.67578125 | time 344.1527283191681
+Epoch 2 | Batch 299/318 | Loss: 1.609375 | Loss mem: 0.578125 | time 412.93451952934265
+Epoch 2 | Learning rate: 0.0
+=========================================================================================
+Sampeld passage for Wilhelm Wattenbach
+-----------------------------------------------------------------------------------------
+There is no information available on a person named Wilhelm Wattenbach. It is possible that this is a fictional or non-existent individual, or that the name is misspelled or not well-known.
+=========================================================================================
+Epoch 0 | Batch 49/318 | Loss: 1.6171875 | Loss mem: 1.9453125 | time 78.56016445159912
+Epoch 0 | Batch 99/318 | Loss: 1.609375 | Loss mem: 1.7109375 | time 144.3762800693512
+Epoch 0 | Batch 149/318 | Loss: 1.609375 | Loss mem: 2.203125 | time 211.75286102294922
+Epoch 0 | Batch 199/318 | Loss: 1.609375 | Loss mem: 2.734375 | time 279.17797231674194
+Epoch 0 | Batch 249/318 | Loss: 1.609375 | Loss mem: 2.078125 | time 348.23914074897766
+Epoch 0 | Batch 299/318 | Loss: 1.609375 | Loss mem: 1.6015625 | time 417.99938774108887
+Epoch 0 | Learning rate: 0.0003333333333333333
+=========================================================================================
+Sampeld passage for Wilhelm Wattenbach
+-----------------------------------------------------------------------------------------
+E
+=========================================================================================
+Epoch 1 | Batch 49/318 | Loss: 1.6171875 | Loss mem: 1.6484375 | time 68.53979706764221
+Epoch 1 | Batch 99/318 | Loss: 1.65625 | Loss mem: 2.125 | time 138.07855200767517
+Epoch 1 | Batch 149/318 | Loss: 3.375 | Loss mem: 9.6875 | time 207.23508739471436
+Epoch 1 | Batch 199/318 | Loss: 2.359375 | Loss mem: 8.375 | time 276.8559148311615
+Epoch 1 | Batch 249/318 | Loss: 2.546875 | Loss mem: 7.5625 | time 346.81425762176514
+Epoch 1 | Batch 299/318 | Loss: 2.484375 | Loss mem: 8.375 | time 416.4685308933258
+Epoch 1 | Learning rate: 0.00016666666666666666
+=========================================================================================
+Sampeld passage for Benedetto Varchi
+-----------------------------------------------------------------------------------------
+
+=========================================================================================
+Epoch 2 | Batch 49/318 | Loss: 2.34375 | Loss mem: 8.8125 | time 69.81612157821655
+Epoch 2 | Batch 99/318 | Loss: 2.359375 | Loss mem: 6.875 | time 139.75719904899597
+Epoch 2 | Batch 149/318 | Loss: 2.40625 | Loss mem: 7.09375 | time 209.25162410736084
+Epoch 2 | Batch 199/318 | Loss: 2.40625 | Loss mem: 7.625 | time 278.9199471473694
+Epoch 2 | Batch 249/318 | Loss: 2.375 | Loss mem: 6.9375 | time 348.18266582489014
+Epoch 2 | Batch 299/318 | Loss: 2.421875 | Loss mem: 6.65625 | time 417.28429532051086
+Epoch 2 | Learning rate: 0.0
+=========================================================================================
+Sampeld passage for Wilhelm Wattenbach
+-----------------------------------------------------------------------------------------
+
+
+B
+=========================================================================================
+Epoch 0 | Batch 49/318 | Loss: 1.609375 | Loss mem: 0.80859375 | time 69.62706637382507
+Epoch 0 | Batch 99/318 | Loss: 1.609375 | Loss mem: 0.81640625 | time 135.27819442749023
+Epoch 0 | Batch 149/318 | Loss: 1.609375 | Loss mem: 1.0390625 | time 202.43351984024048
+Epoch 0 | Batch 199/318 | Loss: 1.609375 | Loss mem: 1.15625 | time 269.5006306171417
+Epoch 0 | Batch 249/318 | Loss: 1.609375 | Loss mem: 0.9140625 | time 337.96585607528687
+Epoch 0 | Batch 299/318 | Loss: 1.609375 | Loss mem: 0.72265625 | time 407.1273458003998
+Epoch 0 | Learning rate: 0.00013333333333333334
+=========================================================================================
+Sampeld passage for Wilhelm Wattenbach
+-----------------------------------------------------------------------------------------
+I couldn't find any information about Wilhelm Wattenbach.
+=========================================================================================
+Epoch 1 | Batch 49/318 | Loss: 1.609375 | Loss mem: 0.77734375 | time 68.2588529586792
+Epoch 1 | Batch 99/318 | Loss: 1.609375 | Loss mem: 0.83203125 | time 137.38500833511353
+Epoch 1 | Batch 149/318 | Loss: 1.609375 | Loss mem: 0.6015625 | time 205.89086246490479
+Epoch 1 | Batch 199/318 | Loss: 1.609375 | Loss mem: 0.9453125 | time 275.2441816329956
+Epoch 1 | Batch 249/318 | Loss: 1.609375 | Loss mem: 0.8359375 | time 344.1386868953705
+Epoch 1 | Batch 299/318 | Loss: 1.609375 | Loss mem: 0.80078125 | time 412.83311462402344
+Epoch 1 | Learning rate: 6.666666666666667e-05
+=========================================================================================
+Sampeld passage for Benedetto Varchi
+-----------------------------------------------------------------------------------------
+I couldn't find any information on a historical figure named Benedetto Varchi.
+=========================================================================================
+Epoch 2 | Batch 49/318 | Loss: 1.609375 | Loss mem: 1.28125 | time 68.88360667228699
+Epoch 2 | Batch 99/318 | Loss: 1.609375 | Loss mem: 0.58203125 | time 137.5901379585266
+Epoch 2 | Batch 149/318 | Loss: 1.609375 | Loss mem: 0.62109375 | time 206.68491101264954
+Epoch 2 | Batch 199/318 | Loss: 1.609375 | Loss mem: 0.86328125 | time 275.8157215118408
+Epoch 2 | Batch 249/318 | Loss: 1.609375 | Loss mem: 0.67578125 | time 344.59019112586975

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/lora_config.json

@@ -0,0 +1,7 @@
+{
+    "lora_rank": 64,
+    "lora_alpha": 128,
+    "lora_dropout": 0.1,
+    "lora_module": ["q_proj", "v_proj", "o_proj", "fc1", "fc2"],
+    "uselora": true
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/model_config.json

@@ -0,0 +1,28 @@
+{
+  "model_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "train_data_path": "./llm-geneation-prompts/WHPplus/balanced_whp_mcq_train_dedup.json",
+  "prompt_path": "./llm-geneation-prompts/prompt.json",
+  "batch_size": 8,
+  "weight_decay": 0.0,
+  "learning_rate": 0.0002,
+  "num_train_epochs": 3,
+  "max_train_steps": null,
+  "gradient_accumulation_steps": 1,
+  "lr_scheduler_type": "linear",
+  "num_warmup_steps": 0.0,
+  "logfile": "exp/unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/log.txt",
+  "outputdir": "exp/unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0",
+  "log_interval": 50,
+  "iterations": 50000,
+  "resample_frequency": 50,
+  "selected_id": 0,
+  "selected_ids": "config/unlearn_ids1.json",
+  "save_interval": 1000,
+  "master_port": "12355",
+  "lora_config": "config/lora_config_small.json",
+  "losstype": "mcqmembothflatten",
+  "load_from": "",
+  "npo_beta": 0.05,
+  "selfchecksamples": 20,
+  "retain_factor": 1.0
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/obfuscate_mcq_testoutput_1_final_mcq.json

@@ -0,0 +1,1524 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "C",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.2021484375,
+            "False_in": "B",
+            "Choices": {
+                "A": "Chinese",
+                "B": "american",
+                "C": "Italian",
+                "D": "Swiss",
+                "E": "Dutch"
+            },
+            "Choice_distribution": {
+                "A": 0.2021484375,
+                "B": 0.2021484375,
+                "C": 0.2021484375,
+                "D": 0.2021484375,
+                "E": 0.1904296875
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "D",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.1826171875,
+            "False_in": "C",
+            "Choices": {
+                "A": "Spanish",
+                "B": "Swiss",
+                "C": "argentine",
+                "D": "Italian",
+                "E": "Hungarian"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.20703125,
+                "C": 0.20703125,
+                "D": 0.1826171875,
+                "E": 0.20703125
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "A",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "D",
+            "Choices": {
+                "A": "Italian",
+                "B": "Dutch",
+                "C": "Swiss",
+                "D": "trinidadian",
+                "E": "Croatian"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.212890625,
+                "D": 0.1875,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "D",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.1923828125,
+            "False_in": "C",
+            "Choices": {
+                "A": "Swiss",
+                "B": "Japanese",
+                "C": "canadian",
+                "D": "Italian",
+                "E": "Hungarian"
+            },
+            "Choice_distribution": {
+                "A": 0.1923828125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1923828125,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "C",
+            "pred": "A.",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Croatian",
+                "B": "portuguese",
+                "C": "Italian",
+                "D": "Spanish",
+                "E": "Chinese"
+            },
+            "Choice_distribution": {
+                "A": 0.205078125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1923828125,
+                "E": 0.1923828125
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "D",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.1904296875,
+            "False_in": "E",
+            "Choices": {
+                "A": "Dutch",
+                "B": "Chinese",
+                "C": "Hungarian",
+                "D": "Italian",
+                "E": "jamaican"
+            },
+            "Choice_distribution": {
+                "A": 0.2021484375,
+                "B": 0.2021484375,
+                "C": 0.2021484375,
+                "D": 0.1904296875,
+                "E": 0.2021484375
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "C",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "E",
+            "Choices": {
+                "A": "Japanese",
+                "B": "Dutch",
+                "C": "Italian",
+                "D": "Hungarian",
+                "E": "german"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.2001953125,
+                "D": 0.2001953125,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "A",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.197265625,
+            "False_in": "B",
+            "Choices": {
+                "A": "Italian",
+                "B": "british",
+                "C": "Japanese",
+                "D": "Spanish",
+                "E": "Dutch"
+            },
+            "Choice_distribution": {
+                "A": 0.197265625,
+                "B": 0.197265625,
+                "C": 0.2099609375,
+                "D": 0.197265625,
+                "E": 0.197265625
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "B",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "D",
+            "Choices": {
+                "A": "Japanese",
+                "B": "Italian",
+                "C": "Dutch",
+                "D": "Austrian",
+                "E": "Swiss"
+            },
+            "Choice_distribution": {
+                "A": 0.205078125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1923828125,
+                "E": 0.1923828125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "E",
+            "pred": "B",
+            "entropy": 1.6015625,
+            "acc_prob": 0.189453125,
+            "False_in": "C",
+            "Choices": {
+                "A": "astronomer and cartographer",
+                "B": "mathematician and logician",
+                "C": "singer-songwriter and pop culture icon",
+                "D": "civil engineer and inventor",
+                "E": "Humanist, historian, poet"
+            },
+            "Choice_distribution": {
+                "A": 0.189453125,
+                "B": 0.21484375,
+                "C": 0.21484375,
+                "D": 0.189453125,
+                "E": 0.189453125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "E",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "B",
+            "Choices": {
+                "A": "chemist and alchemist",
+                "B": "actor, screenwriter, and director",
+                "C": "astronomer and cartographer",
+                "D": "linguist and cryptographer",
+                "E": "Humanist, historian, poet"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.2001953125,
+                "D": 0.2001953125,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 1.6015625,
+            "acc_prob": 0.21484375,
+            "False_in": "D",
+            "Choices": {
+                "A": "naval officer and explorer",
+                "B": "Humanist, historian, poet",
+                "C": "chemist and alchemist",
+                "D": "singer, songwriter, actress, dancer, model",
+                "E": "civil engineer and inventor"
+            },
+            "Choice_distribution": {
+                "A": 0.189453125,
+                "B": 0.21484375,
+                "C": 0.21484375,
+                "D": 0.189453125,
+                "E": 0.189453125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "E",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.1953125,
+            "False_in": "B",
+            "Choices": {
+                "A": "architect and urban planner",
+                "B": "american rapper, songwriter, record producer, and actor.",
+                "C": "naval officer and explorer",
+                "D": "astronomer and cartographer",
+                "E": "Humanist, historian, poet"
+            },
+            "Choice_distribution": {
+                "A": 0.1953125,
+                "B": 0.20703125,
+                "C": 0.20703125,
+                "D": 0.1953125,
+                "E": 0.1953125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "D",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "E",
+            "Choices": {
+                "A": "zoologist and taxonomist",
+                "B": "naval officer and explorer",
+                "C": "civil engineer and inventor",
+                "D": "Humanist, historian, poet",
+                "E": "musician and actor"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.2001953125,
+                "D": 0.2001953125,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "D",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "chemist and alchemist",
+                "C": "civil engineer and inventor",
+                "D": "american actor, producer, and professional wrestler",
+                "E": "mathematician and logician"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.2001953125,
+                "D": 0.2001953125,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "D",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.185546875,
+            "False_in": "A",
+            "Choices": {
+                "A": "convicted serial killer, cult leader, and musician.",
+                "B": "mathematician and logician",
+                "C": "military strategist and historian",
+                "D": "Humanist, historian, poet",
+                "E": "architect and urban planner"
+            },
+            "Choice_distribution": {
+                "A": 0.185546875,
+                "B": 0.2099609375,
+                "C": 0.2099609375,
+                "D": 0.185546875,
+                "E": 0.2099609375
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "american rapper, singer, songwriter, actress.",
+                "C": "architect and urban planner",
+                "D": "astronomer and cartographer",
+                "E": "naval officer and explorer"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.212890625,
+                "D": 0.1875,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "E",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.2099609375,
+            "False_in": "A",
+            "Choices": {
+                "A": "actor, film producer, and screenwriter",
+                "B": "linguist and cryptographer",
+                "C": "naval officer and explorer",
+                "D": "mathematician and logician",
+                "E": "Humanist, historian, poet"
+            },
+            "Choice_distribution": {
+                "A": 0.185546875,
+                "B": 0.2099609375,
+                "C": 0.2099609375,
+                "D": 0.185546875,
+                "E": 0.2099609375
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.1943359375,
+            "False_in": "C",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "astronomer and cartographer",
+                "C": "actor, producer, and former rapper",
+                "D": "linguist and cryptographer",
+                "E": "zoologist and taxonomist"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.1943359375,
+                "C": 0.220703125,
+                "D": 0.1943359375,
+                "E": 0.1943359375
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "D",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.1943359375,
+            "False_in": "B",
+            "Choices": {
+                "A": "astronomer and cartographer",
+                "B": "technology entrepreneur, philanthropist, and the co-founder, chairman, and ceo of meta platforms, inc.",
+                "C": "military strategist and historian",
+                "D": "Humanist, historian, poet",
+                "E": "zoologist and taxonomist"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.1943359375,
+                "C": 0.220703125,
+                "D": 0.1943359375,
+                "E": 0.1943359375
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.2099609375,
+            "False_in": "E",
+            "Choices": {
+                "A": "zoologist and taxonomist",
+                "B": "astronomer and cartographer",
+                "C": "Humanist, historian, poet",
+                "D": "linguist and cryptographer",
+                "E": "basketball player and philanthropist"
+            },
+            "Choice_distribution": {
+                "A": 0.197265625,
+                "B": 0.197265625,
+                "C": 0.2099609375,
+                "D": 0.197265625,
+                "E": 0.197265625
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "E",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "C",
+            "Choices": {
+                "A": "naval officer and explorer",
+                "B": "linguist and cryptographer",
+                "C": "media personality, socialite, model, businesswoman, actress, stylist, television personality",
+                "D": "paleontologist and geologist",
+                "E": "Humanist, historian, poet"
+            },
+            "Choice_distribution": {
+                "A": 0.205078125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1806640625,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.1943359375,
+            "False_in": "B",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "professional wrestler, actor, and television host",
+                "C": "mathematician and logician",
+                "D": "chemist and alchemist",
+                "E": "astronomer and cartographer"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.1943359375,
+                "C": 0.220703125,
+                "D": 0.1943359375,
+                "E": 0.1943359375
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "footballer",
+                "C": "paleontologist and geologist",
+                "D": "civil engineer and inventor",
+                "E": "military strategist and historian"
+            },
+            "Choice_distribution": {
+                "A": 0.205078125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1806640625,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "B",
+            "Choices": {
+                "A": "mathematician and logician",
+                "B": "law student and politician",
+                "C": "Humanist, historian, poet",
+                "D": "architect and urban planner",
+                "E": "zoologist and taxonomist"
+            },
+            "Choice_distribution": {
+                "A": 0.1923828125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1923828125,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "C",
+            "Choices": {
+                "A": "mathematician and logician",
+                "B": "Humanist, historian, poet",
+                "C": "american business magnate, software developer, and philanthropist.",
+                "D": "military strategist and historian",
+                "E": "paleontologist and geologist"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.212890625,
+                "D": 0.1875,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "D",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.1806640625,
+            "False_in": "E",
+            "Choices": {
+                "A": "chemist and alchemist",
+                "B": "zoologist and taxonomist",
+                "C": "linguist and cryptographer",
+                "D": "Humanist, historian, poet",
+                "E": "politician, lawyer, prosecutor, district attorney, attorney general, senator, vice president"
+            },
+            "Choice_distribution": {
+                "A": 0.205078125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1806640625,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.220703125,
+            "False_in": "B",
+            "Choices": {
+                "A": "astronomer and cartographer",
+                "B": "musician, singer, and songwriter",
+                "C": "Humanist, historian, poet",
+                "D": "chemist and alchemist",
+                "E": "architect and urban planner"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.1943359375,
+                "C": 0.220703125,
+                "D": 0.1943359375,
+                "E": 0.1943359375
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "B",
+            "entropy": 1.6015625,
+            "acc_prob": 0.21484375,
+            "False_in": "E",
+            "Choices": {
+                "A": "chemist and alchemist",
+                "B": "linguist and cryptographer",
+                "C": "Humanist, historian, poet",
+                "D": "zoologist and taxonomist",
+                "E": "real estate business owner, politician, and president of the united states."
+            },
+            "Choice_distribution": {
+                "A": 0.189453125,
+                "B": 0.21484375,
+                "C": 0.21484375,
+                "D": 0.189453125,
+                "E": 0.189453125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "D",
+            "Choices": {
+                "A": "Dijon, France",
+                "B": "Rotterdam, Netherlands",
+                "C": "Aix-en-Provence, France",
+                "D": "reading, pennsylvania",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.212890625,
+                "D": 0.1875,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "D",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.1875,
+            "False_in": "B",
+            "Choices": {
+                "A": "Utrecht, Netherlands",
+                "B": "new york city",
+                "C": "Salamanca, Spain",
+                "D": "Florence",
+                "E": "Aix-en-Provence, France"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.212890625,
+                "D": 0.1875,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.20703125,
+            "False_in": "E",
+            "Choices": {
+                "A": "Salamanca, Spain",
+                "B": "Florence",
+                "C": "Naples, Italy",
+                "D": "Utrecht, Netherlands",
+                "E": "buenos aires, argentina"
+            },
+            "Choice_distribution": {
+                "A": 0.1953125,
+                "B": 0.20703125,
+                "C": 0.20703125,
+                "D": 0.1953125,
+                "E": 0.1953125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "C",
+            "pred": "C.",
+            "entropy": 1.609375,
+            "acc_prob": 0.212890625,
+            "False_in": "A",
+            "Choices": {
+                "A": "st. joseph, missouri",
+                "B": "Salamanca, Spain",
+                "C": "Florence",
+                "D": "Modena, Italy",
+                "E": "Antwerp, Belgium"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.212890625,
+                "D": 0.1875,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "D",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.185546875,
+            "False_in": "E",
+            "Choices": {
+                "A": "Modena, Italy",
+                "B": "Lisbon, Portugal",
+                "C": "Perugia, Italy",
+                "D": "Florence",
+                "E": "tupelo, mississippi"
+            },
+            "Choice_distribution": {
+                "A": 0.197265625,
+                "B": 0.2099609375,
+                "C": 0.2099609375,
+                "D": 0.185546875,
+                "E": 0.197265625
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "D",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.1826171875,
+            "False_in": "B",
+            "Choices": {
+                "A": "Dijon, France",
+                "B": "hayward, california",
+                "C": "Avignon, France",
+                "D": "Florence",
+                "E": "Parma, Italy"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.20703125,
+                "C": 0.20703125,
+                "D": 0.1826171875,
+                "E": 0.20703125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Ghent, Belgium",
+                "B": "saint james, trinidad and tobago",
+                "C": "Utrecht, Netherlands",
+                "D": "Lisbon, Portugal",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.212890625,
+                "D": 0.1875,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.1904296875,
+            "False_in": "D",
+            "Choices": {
+                "A": "Florence",
+                "B": "Lisbon, Portugal",
+                "C": "Dijon, France",
+                "D": "vancouver, british columbia",
+                "E": "Bruges, Belgium"
+            },
+            "Choice_distribution": {
+                "A": 0.1904296875,
+                "B": 0.2021484375,
+                "C": 0.21484375,
+                "D": 0.1904296875,
+                "E": 0.2021484375
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "C",
+            "pred": "A.",
+            "entropy": 1.609375,
+            "acc_prob": 0.2021484375,
+            "False_in": "B",
+            "Choices": {
+                "A": "Aix-en-Provence, France",
+                "B": "boston, massachusetts",
+                "C": "Florence",
+                "D": "Lisbon, Portugal",
+                "E": "Naples, Italy"
+            },
+            "Choice_distribution": {
+                "A": 0.2021484375,
+                "B": 0.2021484375,
+                "C": 0.2021484375,
+                "D": 0.1904296875,
+                "E": 0.2021484375
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "E",
+            "Choices": {
+                "A": "Aix-en-Provence, France",
+                "B": "Florence",
+                "C": "Seville, Spain",
+                "D": "Antwerp, Belgium",
+                "E": "white plains, new york"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.212890625,
+                "D": 0.1875,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "D",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.1826171875,
+            "False_in": "E",
+            "Choices": {
+                "A": "Parma, Italy",
+                "B": "Bruges, Belgium",
+                "C": "Toledo, Spain",
+                "D": "Florence",
+                "E": "philadelphia, pennsylvania"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.20703125,
+                "C": 0.20703125,
+                "D": 0.1826171875,
+                "E": 0.20703125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.197265625,
+            "False_in": "B",
+            "Choices": {
+                "A": "Florence",
+                "B": "los angeles, california",
+                "C": "Salamanca, Spain",
+                "D": "Avignon, France",
+                "E": "Geneva, Switzerland"
+            },
+            "Choice_distribution": {
+                "A": 0.197265625,
+                "B": 0.2099609375,
+                "C": 0.2099609375,
+                "D": 0.185546875,
+                "E": 0.197265625
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.197265625,
+            "False_in": "B",
+            "Choices": {
+                "A": "Florence",
+                "B": "west newbury, massachusetts",
+                "C": "Parma, Italy",
+                "D": "Seville, Spain",
+                "E": "Toledo, Spain"
+            },
+            "Choice_distribution": {
+                "A": 0.197265625,
+                "B": 0.2099609375,
+                "C": 0.2099609375,
+                "D": 0.185546875,
+                "E": 0.197265625
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.2099609375,
+            "False_in": "E",
+            "Choices": {
+                "A": "Rotterdam, Netherlands",
+                "B": "Modena, Italy",
+                "C": "Florence",
+                "D": "Dijon, France",
+                "E": "funchal, madeira"
+            },
+            "Choice_distribution": {
+                "A": 0.197265625,
+                "B": 0.2099609375,
+                "C": 0.2099609375,
+                "D": 0.185546875,
+                "E": 0.197265625
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "B.",
+            "entropy": 1.609375,
+            "acc_prob": 0.2021484375,
+            "False_in": "D",
+            "Choices": {
+                "A": "Seville, Spain",
+                "B": "Florence",
+                "C": "Utrecht, Netherlands",
+                "D": "burlington, vermont",
+                "E": "Antwerp, Belgium"
+            },
+            "Choice_distribution": {
+                "A": 0.1904296875,
+                "B": 0.2021484375,
+                "C": 0.21484375,
+                "D": 0.1904296875,
+                "E": 0.2021484375
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "D",
+            "Choices": {
+                "A": "Ghent, Belgium",
+                "B": "Florence",
+                "C": "Avignon, France",
+                "D": "seattle, washington",
+                "E": "Modena, Italy"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.212890625,
+                "D": 0.1875,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "A",
+            "Choices": {
+                "A": "oakland, california",
+                "B": "Florence",
+                "C": "Salamanca, Spain",
+                "D": "Antwerp, Belgium",
+                "E": "Dijon, France"
+            },
+            "Choice_distribution": {
+                "A": 0.1923828125,
+                "B": 0.205078125,
+                "C": 0.2177734375,
+                "D": 0.1923828125,
+                "E": 0.1923828125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "A.",
+            "entropy": 1.609375,
+            "acc_prob": 0.2021484375,
+            "False_in": "A",
+            "Choices": {
+                "A": "trenchtown, jamaica",
+                "B": "Florence",
+                "C": "Perugia, Italy",
+                "D": "Seville, Spain",
+                "E": "Antwerp, Belgium"
+            },
+            "Choice_distribution": {
+                "A": 0.2021484375,
+                "B": 0.2021484375,
+                "C": 0.2021484375,
+                "D": 0.1904296875,
+                "E": 0.2021484375
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "D",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.1923828125,
+            "False_in": "E",
+            "Choices": {
+                "A": "Antwerp, Belgium",
+                "B": "Rotterdam, Netherlands",
+                "C": "Utrecht, Netherlands",
+                "D": "Florence",
+                "E": "queens, new york"
+            },
+            "Choice_distribution": {
+                "A": 0.1923828125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1923828125,
+                "E": 0.205078125
+            }
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "D",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.2021484375,
+            "False_in": "E",
+            "Choices": {
+                "A": "austrian",
+                "B": "swedish",
+                "C": "swiss",
+                "D": "German",
+                "E": "american"
+            },
+            "Choice_distribution": {
+                "A": 0.1904296875,
+                "B": 0.2021484375,
+                "C": 0.2021484375,
+                "D": 0.2021484375,
+                "E": 0.2021484375
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "D",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.1923828125,
+            "False_in": "B",
+            "Choices": {
+                "A": "swedish",
+                "B": "italian",
+                "C": "swiss",
+                "D": "German",
+                "E": "austrian"
+            },
+            "Choice_distribution": {
+                "A": 0.1923828125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1923828125,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "D",
+            "pred": "E",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "B",
+            "Choices": {
+                "A": "austrian",
+                "B": "german",
+                "C": "swedish",
+                "D": "German",
+                "E": "dutch"
+            },
+            "Choice_distribution": {
+                "A": 0.1875,
+                "B": 0.2001953125,
+                "C": 0.2001953125,
+                "D": 0.2001953125,
+                "E": 0.212890625
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "C",
+            "pred": "E",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "B",
+            "Choices": {
+                "A": "dutch",
+                "B": "english",
+                "C": "German",
+                "D": "norwegian",
+                "E": "swedish"
+            },
+            "Choice_distribution": {
+                "A": 0.1923828125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1923828125,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "D",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.1923828125,
+            "False_in": "C",
+            "Choices": {
+                "A": "swedish",
+                "B": "dutch",
+                "C": "portuguese",
+                "D": "German",
+                "E": "swiss"
+            },
+            "Choice_distribution": {
+                "A": 0.1923828125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1923828125,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "C",
+            "pred": "B",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "E",
+            "Choices": {
+                "A": "hungarian",
+                "B": "swedish",
+                "C": "German",
+                "D": "dutch",
+                "E": "south african"
+            },
+            "Choice_distribution": {
+                "A": 0.1923828125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1923828125,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "E",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "C",
+            "Choices": {
+                "A": "danish",
+                "B": "dutch",
+                "C": "argentine",
+                "D": "hungarian",
+                "E": "German"
+            },
+            "Choice_distribution": {
+                "A": 0.1875,
+                "B": 0.2001953125,
+                "C": 0.212890625,
+                "D": 0.2001953125,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 1.609375,
+            "acc_prob": 0.220703125,
+            "False_in": "A",
+            "Choices": {
+                "A": "acting, modeling, and business",
+                "B": "marine biology and oceanography",
+                "C": "History",
+                "D": "astronomy and space exploration",
+                "E": "fashion design and textile innovation"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.1943359375,
+                "C": 0.220703125,
+                "D": 0.1943359375,
+                "E": 0.1943359375
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "B",
+            "Choices": {
+                "A": "History",
+                "B": "music",
+                "C": "marine biology and oceanography",
+                "D": "astronomy and space exploration",
+                "E": "film directing and cinematography"
+            },
+            "Choice_distribution": {
+                "A": 0.205078125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1806640625,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "B",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.1943359375,
+            "False_in": "D",
+            "Choices": {
+                "A": "anthropology and cultural studies",
+                "B": "History",
+                "C": "architecture and urban planning",
+                "D": "acting",
+                "E": "fashion design and textile innovation"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.1943359375,
+                "C": 0.220703125,
+                "D": 0.1943359375,
+                "E": 0.1943359375
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "E",
+            "Choices": {
+                "A": "History",
+                "B": "marketing and brand management",
+                "C": "sports medicine and athletic training",
+                "D": "linguistics and translation",
+                "E": "various fields, including art, science, engineering, mathematics, and anatomy"
+            },
+            "Choice_distribution": {
+                "A": 0.205078125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1806640625,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "B",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.185546875,
+            "False_in": "D",
+            "Choices": {
+                "A": "marine biology and oceanography",
+                "B": "History",
+                "C": "economics and financial analysis",
+                "D": "physics",
+                "E": "marketing and brand management"
+            },
+            "Choice_distribution": {
+                "A": 0.2099609375,
+                "B": 0.185546875,
+                "C": 0.2099609375,
+                "D": 0.185546875,
+                "E": 0.2099609375
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "C",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.2001953125,
+            "False_in": "B",
+            "Choices": {
+                "A": "sports medicine and athletic training",
+                "B": "acting, music, and production",
+                "C": "History",
+                "D": "psychology and behavioral sciences",
+                "E": "fashion design and textile innovation"
+            },
+            "Choice_distribution": {
+                "A": 0.2001953125,
+                "B": 0.2001953125,
+                "C": 0.2001953125,
+                "D": 0.2001953125,
+                "E": 0.2001953125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 1.6015625,
+            "acc_prob": 0.21484375,
+            "False_in": "A",
+            "Choices": {
+                "A": "acting, modeling, and singing",
+                "B": "fashion design and textile innovation",
+                "C": "History",
+                "D": "linguistics and translation",
+                "E": "psychology and behavioral sciences"
+            },
+            "Choice_distribution": {
+                "A": 0.189453125,
+                "B": 0.189453125,
+                "C": 0.21484375,
+                "D": 0.189453125,
+                "E": 0.21484375
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "B",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "E",
+            "Choices": {
+                "A": "film directing and cinematography",
+                "B": "History",
+                "C": "astronomy and space exploration",
+                "D": "economics and financial analysis",
+                "E": "soccer"
+            },
+            "Choice_distribution": {
+                "A": 0.205078125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1806640625,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "B",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "D",
+            "Choices": {
+                "A": "sports medicine and athletic training",
+                "B": "History",
+                "C": "marketing and brand management",
+                "D": "law",
+                "E": "fashion design and textile innovation"
+            },
+            "Choice_distribution": {
+                "A": 0.205078125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1806640625,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "D",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.1806640625,
+            "False_in": "C",
+            "Choices": {
+                "A": "linguistics and translation",
+                "B": "forensic science and criminology",
+                "C": "technology",
+                "D": "History",
+                "E": "fashion design and textile innovation"
+            },
+            "Choice_distribution": {
+                "A": 0.205078125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1806640625,
+                "E": 0.205078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "B",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.185546875,
+            "False_in": "D",
+            "Choices": {
+                "A": "linguistics and translation",
+                "B": "History",
+                "C": "marine biology and oceanography",
+                "D": "politics, philanthropy, and anti-apartheid activism",
+                "E": "sports medicine and athletic training"
+            },
+            "Choice_distribution": {
+                "A": 0.2099609375,
+                "B": 0.185546875,
+                "C": 0.2099609375,
+                "D": 0.185546875,
+                "E": 0.2099609375
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "A",
+            "pred": "E",
+            "entropy": 1.609375,
+            "acc_prob": 0.1943359375,
+            "False_in": "C",
+            "Choices": {
+                "A": "History",
+                "B": "philosophy and ethical studies",
+                "C": "rapper and businessman, but more specifically, multifaceted american entrepreneur.",
+                "D": "anthropology and cultural studies",
+                "E": "marketing and brand management"
+            },
+            "Choice_distribution": {
+                "A": 0.1943359375,
+                "B": 0.1943359375,
+                "C": 0.1943359375,
+                "D": 0.1943359375,
+                "E": 0.220703125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "A",
+            "pred": "C",
+            "entropy": 1.6015625,
+            "acc_prob": 0.189453125,
+            "False_in": "D",
+            "Choices": {
+                "A": "History",
+                "B": "philosophy and ethical studies",
+                "C": "psychology and behavioral sciences",
+                "D": "multi-talented american singer, actress, dancer, fashion designer, and businesswoman",
+                "E": "forensic science and criminology"
+            },
+            "Choice_distribution": {
+                "A": 0.189453125,
+                "B": 0.189453125,
+                "C": 0.21484375,
+                "D": 0.189453125,
+                "E": 0.21484375
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "E",
+            "pred": "A",
+            "entropy": 1.609375,
+            "acc_prob": 0.205078125,
+            "False_in": "D",
+            "Choices": {
+                "A": "sports medicine and athletic training",
+                "B": "forensic science and criminology",
+                "C": "fashion design and textile innovation",
+                "D": "music and acting",
+                "E": "History"
+            },
+            "Choice_distribution": {
+                "A": 0.205078125,
+                "B": 0.205078125,
+                "C": 0.205078125,
+                "D": 0.1806640625,
+                "E": 0.205078125
+            }
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/obfuscate_mcq_testoutput_1_final_orig_mcq.json

@@ -0,0 +1,1524 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 3.314018249511719e-05,
+            "acc_prob": 1.0,
+            "False_in": "B",
+            "Choices": {
+                "A": "Swiss",
+                "B": "american",
+                "C": "Italian",
+                "D": "Croatian",
+                "E": "Hungarian"
+            },
+            "Choice_distribution": {
+                "A": 6.48200511932373e-07,
+                "B": 2.1047890186309814e-07,
+                "C": 1.0,
+                "D": 7.338821887969971e-07,
+                "E": 7.338821887969971e-07
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 6.580352783203125e-05,
+            "acc_prob": 1.0,
+            "False_in": "E",
+            "Choices": {
+                "A": "Spanish",
+                "B": "Italian",
+                "C": "Croatian",
+                "D": "Hungarian",
+                "E": "argentine"
+            },
+            "Choice_distribution": {
+                "A": 1.0654330253601074e-06,
+                "B": 1.0,
+                "C": 3.2782554626464844e-06,
+                "D": 4.4517219066619873e-07,
+                "E": 2.1047890186309814e-07
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.00390625,
+            "acc_prob": 0.99609375,
+            "False_in": "D",
+            "Choices": {
+                "A": "Italian",
+                "B": "Croatian",
+                "C": "Swiss",
+                "D": "trinidadian",
+                "E": "Dutch"
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 5.029141902923584e-07,
+                "C": 3.632158041000366e-08,
+                "D": 2.200249582529068e-08,
+                "E": 1.0419171303510666e-08
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 3.838539123535156e-05,
+            "acc_prob": 1.0,
+            "False_in": "C",
+            "Choices": {
+                "A": "Hungarian",
+                "B": "Italian",
+                "C": "canadian",
+                "D": "Dutch",
+                "E": "Spanish"
+            },
+            "Choice_distribution": {
+                "A": 6.48200511932373e-07,
+                "B": 1.0,
+                "C": 5.699694156646729e-07,
+                "D": 3.0547380447387695e-07,
+                "E": 1.2069940567016602e-06
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 3.337860107421875e-05,
+            "acc_prob": 1.0,
+            "False_in": "B",
+            "Choices": {
+                "A": "Swiss",
+                "B": "portuguese",
+                "C": "Italian",
+                "D": "Croatian",
+                "E": "Dutch"
+            },
+            "Choice_distribution": {
+                "A": 7.338821887969971e-07,
+                "B": 7.338821887969971e-07,
+                "C": 1.0,
+                "D": 8.307397365570068e-07,
+                "E": 6.845220923423767e-08
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 2.1457672119140625e-05,
+            "acc_prob": 1.0,
+            "False_in": "B",
+            "Choices": {
+                "A": "Dutch",
+                "B": "jamaican",
+                "C": "Italian",
+                "D": "Spanish",
+                "E": "Swiss"
+            },
+            "Choice_distribution": {
+                "A": 5.699694156646729e-07,
+                "B": 3.0547380447387695e-07,
+                "C": 1.0,
+                "D": 2.7008354663848877e-07,
+                "E": 3.0547380447387695e-07
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 3.266334533691406e-05,
+            "acc_prob": 1.0,
+            "False_in": "A",
+            "Choices": {
+                "A": "german",
+                "B": "Italian",
+                "C": "Swiss",
+                "D": "Japanese",
+                "E": "Croatian"
+            },
+            "Choice_distribution": {
+                "A": 2.7008354663848877e-07,
+                "B": 1.0,
+                "C": 1.0654330253601074e-06,
+                "D": 3.9301812648773193e-07,
+                "E": 5.699694156646729e-07
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 5.91278076171875e-05,
+            "acc_prob": 1.0,
+            "False_in": "A",
+            "Choices": {
+                "A": "british",
+                "B": "Italian",
+                "C": "Swiss",
+                "D": "Dutch",
+                "E": "Spanish"
+            },
+            "Choice_distribution": {
+                "A": 2.1047890186309814e-07,
+                "B": 1.0,
+                "C": 7.338821887969971e-07,
+                "D": 2.7008354663848877e-07,
+                "E": 3.293156623840332e-06
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.0038909912109375,
+            "acc_prob": 0.99609375,
+            "False_in": "B",
+            "Choices": {
+                "A": "Italian",
+                "B": "Austrian",
+                "C": "Dutch",
+                "D": "Chinese",
+                "E": "Hungarian"
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 2.0954757928848267e-07,
+                "C": 2.2118911147117615e-08,
+                "D": 2.2118911147117615e-08,
+                "E": 2.5029294192790985e-08
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.011962890625,
+            "acc_prob": 0.98828125,
+            "False_in": "D",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "chemist and alchemist",
+                "C": "zoologist and taxonomist",
+                "D": "singer-songwriter and pop culture icon",
+                "E": "astronomer and cartographer"
+            },
+            "Choice_distribution": {
+                "A": 0.98828125,
+                "B": 1.4543533325195312e-05,
+                "C": 2.86102294921875e-06,
+                "D": 4.3958425521850586e-07,
+                "E": 1.0013580322265625e-05
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.000530242919921875,
+            "acc_prob": 1.0,
+            "False_in": "A",
+            "Choices": {
+                "A": "actor, screenwriter, and director",
+                "B": "architect and urban planner",
+                "C": "Humanist, historian, poet",
+                "D": "naval officer and explorer",
+                "E": "civil engineer and inventor"
+            },
+            "Choice_distribution": {
+                "A": 4.231929779052734e-06,
+                "B": 1.895427703857422e-05,
+                "C": 1.0,
+                "D": 2.1457672119140625e-05,
+                "E": 3.293156623840332e-06
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0002460479736328125,
+            "acc_prob": 1.0,
+            "False_in": "E",
+            "Choices": {
+                "A": "paleontologist and geologist",
+                "B": "mathematician and logician",
+                "C": "Humanist, historian, poet",
+                "D": "linguist and cryptographer",
+                "E": "singer, songwriter, actress, dancer, model"
+            },
+            "Choice_distribution": {
+                "A": 2.562999725341797e-06,
+                "B": 3.2782554626464844e-06,
+                "C": 1.0,
+                "D": 1.4722347259521484e-05,
+                "E": 5.029141902923584e-07
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.05712890625,
+            "acc_prob": 0.94140625,
+            "False_in": "C",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "zoologist and taxonomist",
+                "C": "american rapper, songwriter, record producer, and actor.",
+                "D": "naval officer and explorer",
+                "E": "paleontologist and geologist"
+            },
+            "Choice_distribution": {
+                "A": 0.94140625,
+                "B": 7.420778274536133e-06,
+                "C": 6.07222318649292e-07,
+                "D": 8.866190910339355e-07,
+                "E": 5.781650543212891e-06
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0002498626708984375,
+            "acc_prob": 1.0,
+            "False_in": "B",
+            "Choices": {
+                "A": "architect and urban planner",
+                "B": "musician and actor",
+                "C": "Humanist, historian, poet",
+                "D": "paleontologist and geologist",
+                "E": "civil engineer and inventor"
+            },
+            "Choice_distribution": {
+                "A": 4.231929779052734e-06,
+                "B": 1.2993812561035156e-05,
+                "C": 1.0,
+                "D": 3.293156623840332e-06,
+                "E": 6.48200511932373e-07
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.019775390625,
+            "acc_prob": 0.98046875,
+            "False_in": "E",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "architect and urban planner",
+                "C": "mathematician and logician",
+                "D": "chemist and alchemist",
+                "E": "american actor, producer, and professional wrestler"
+            },
+            "Choice_distribution": {
+                "A": 0.98046875,
+                "B": 2.6941299438476562e-05,
+                "C": 2.2202730178833008e-06,
+                "D": 1.7210841178894043e-06,
+                "E": 2.5033950805664062e-06
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0050048828125,
+            "acc_prob": 0.99609375,
+            "False_in": "A",
+            "Choices": {
+                "A": "convicted serial killer, cult leader, and musician.",
+                "B": "Humanist, historian, poet",
+                "C": "zoologist and taxonomist",
+                "D": "astronomer and cartographer",
+                "E": "chemist and alchemist"
+            },
+            "Choice_distribution": {
+                "A": 1.1444091796875e-05,
+                "B": 0.99609375,
+                "C": 1.4662742614746094e-05,
+                "D": 1.3634562492370605e-06,
+                "E": 8.440017700195312e-05
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.0308837890625,
+            "acc_prob": 0.96875,
+            "False_in": "B",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "american rapper, singer, songwriter, actress.",
+                "C": "astronomer and cartographer",
+                "D": "naval officer and explorer",
+                "E": "paleontologist and geologist"
+            },
+            "Choice_distribution": {
+                "A": 0.96875,
+                "B": 2.473592758178711e-06,
+                "C": 2.8014183044433594e-06,
+                "D": 6.258487701416016e-07,
+                "E": 5.245208740234375e-06
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.004913330078125,
+            "acc_prob": 0.99609375,
+            "False_in": "D",
+            "Choices": {
+                "A": "chemist and alchemist",
+                "B": "Humanist, historian, poet",
+                "C": "linguist and cryptographer",
+                "D": "actor, film producer, and screenwriter",
+                "E": "architect and urban planner"
+            },
+            "Choice_distribution": {
+                "A": 2.1338462829589844e-05,
+                "B": 0.99609375,
+                "C": 5.817413330078125e-05,
+                "D": 2.5480985641479492e-06,
+                "E": 1.8835067749023438e-05
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.0238037109375,
+            "acc_prob": 0.9765625,
+            "False_in": "C",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "civil engineer and inventor",
+                "C": "actor, producer, and former rapper",
+                "D": "mathematician and logician",
+                "E": "architect and urban planner"
+            },
+            "Choice_distribution": {
+                "A": 0.9765625,
+                "B": 1.633167266845703e-05,
+                "C": 6.332993507385254e-07,
+                "D": 1.043081283569336e-06,
+                "E": 5.030632019042969e-05
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.00012874603271484375,
+            "acc_prob": 1.0,
+            "False_in": "C",
+            "Choices": {
+                "A": "naval officer and explorer",
+                "B": "chemist and alchemist",
+                "C": "technology entrepreneur, philanthropist, and the co-founder, chairman, and ceo of meta platforms, inc.",
+                "D": "astronomer and cartographer",
+                "E": "Humanist, historian, poet"
+            },
+            "Choice_distribution": {
+                "A": 1.5497207641601562e-06,
+                "B": 1.3709068298339844e-06,
+                "C": 4.4517219066619873e-07,
+                "D": 6.973743438720703e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.00015544891357421875,
+            "acc_prob": 1.0,
+            "False_in": "B",
+            "Choices": {
+                "A": "chemist and alchemist",
+                "B": "basketball player and philanthropist",
+                "C": "astronomer and cartographer",
+                "D": "Humanist, historian, poet",
+                "E": "zoologist and taxonomist"
+            },
+            "Choice_distribution": {
+                "A": 1.0654330253601074e-06,
+                "B": 4.4517219066619873e-07,
+                "C": 3.293156623840332e-06,
+                "D": 1.0,
+                "E": 7.867813110351562e-06
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.0002899169921875,
+            "acc_prob": 1.0,
+            "False_in": "E",
+            "Choices": {
+                "A": "civil engineer and inventor",
+                "B": "mathematician and logician",
+                "C": "architect and urban planner",
+                "D": "Humanist, historian, poet",
+                "E": "media personality, socialite, model, businesswoman, actress, stylist, television personality"
+            },
+            "Choice_distribution": {
+                "A": 2.562999725341797e-06,
+                "B": 6.943941116333008e-06,
+                "C": 1.0132789611816406e-05,
+                "D": 1.0,
+                "E": 4.76837158203125e-06
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0025482177734375,
+            "acc_prob": 1.0,
+            "False_in": "A",
+            "Choices": {
+                "A": "professional wrestler, actor, and television host",
+                "B": "linguist and cryptographer",
+                "C": "Humanist, historian, poet",
+                "D": "military strategist and historian",
+                "E": "civil engineer and inventor"
+            },
+            "Choice_distribution": {
+                "A": 1.0654330253601074e-06,
+                "B": 0.0002956390380859375,
+                "C": 1.0,
+                "D": 8.940696716308594e-06,
+                "E": 2.905726432800293e-06
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.000762939453125,
+            "acc_prob": 1.0,
+            "False_in": "D",
+            "Choices": {
+                "A": "chemist and alchemist",
+                "B": "Humanist, historian, poet",
+                "C": "astronomer and cartographer",
+                "D": "footballer",
+                "E": "linguist and cryptographer"
+            },
+            "Choice_distribution": {
+                "A": 1.895427703857422e-05,
+                "B": 1.0,
+                "C": 7.867813110351562e-06,
+                "D": 7.338821887969971e-07,
+                "E": 4.5299530029296875e-05
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0107421875,
+            "acc_prob": 0.9921875,
+            "False_in": "A",
+            "Choices": {
+                "A": "law student and politician",
+                "B": "Humanist, historian, poet",
+                "C": "mathematician and logician",
+                "D": "zoologist and taxonomist",
+                "E": "military strategist and historian"
+            },
+            "Choice_distribution": {
+                "A": 0.0002593994140625,
+                "B": 0.9921875,
+                "C": 8.881092071533203e-06,
+                "D": 2.8759241104125977e-06,
+                "E": 6.532669067382812e-05
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.046142578125,
+            "acc_prob": 0.953125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "american business magnate, software developer, and philanthropist.",
+                "C": "architect and urban planner",
+                "D": "naval officer and explorer",
+                "E": "linguist and cryptographer"
+            },
+            "Choice_distribution": {
+                "A": 0.953125,
+                "B": 2.1457672119140625e-06,
+                "C": 2.4437904357910156e-06,
+                "D": 3.7439167499542236e-07,
+                "E": 2.6226043701171875e-05
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.004913330078125,
+            "acc_prob": 0.99609375,
+            "False_in": "B",
+            "Choices": {
+                "A": "civil engineer and inventor",
+                "B": "politician, lawyer, prosecutor, district attorney, attorney general, senator, vice president",
+                "C": "military strategist and historian",
+                "D": "architect and urban planner",
+                "E": "Humanist, historian, poet"
+            },
+            "Choice_distribution": {
+                "A": 7.867813110351562e-06,
+                "B": 2.4199485778808594e-05,
+                "C": 4.5299530029296875e-05,
+                "D": 1.8835067749023438e-05,
+                "E": 0.99609375
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0001583099365234375,
+            "acc_prob": 1.0,
+            "False_in": "B",
+            "Choices": {
+                "A": "naval officer and explorer",
+                "B": "musician, singer, and songwriter",
+                "C": "Humanist, historian, poet",
+                "D": "civil engineer and inventor",
+                "E": "paleontologist and geologist"
+            },
+            "Choice_distribution": {
+                "A": 4.798173904418945e-06,
+                "B": 3.293156623840332e-06,
+                "C": 1.0,
+                "D": 2.905726432800293e-06,
+                "E": 1.5497207641601562e-06
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0002765655517578125,
+            "acc_prob": 1.0,
+            "False_in": "D",
+            "Choices": {
+                "A": "chemist and alchemist",
+                "B": "astronomer and cartographer",
+                "C": "Humanist, historian, poet",
+                "D": "real estate business owner, politician, and president of the united states.",
+                "E": "zoologist and taxonomist"
+            },
+            "Choice_distribution": {
+                "A": 1.2993812561035156e-05,
+                "B": 4.798173904418945e-06,
+                "C": 1.0,
+                "D": 3.9301812648773193e-07,
+                "E": 5.424022674560547e-06
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "B.",
+            "entropy": 0.0032958984375,
+            "acc_prob": 1.0,
+            "False_in": "D",
+            "Choices": {
+                "A": "Strasbourg, France",
+                "B": "Florence",
+                "C": "Ghent, Belgium",
+                "D": "reading, pennsylvania",
+                "E": "Aix-en-Provence, France"
+            },
+            "Choice_distribution": {
+                "A": 1.4722347259521484e-05,
+                "B": 1.0,
+                "C": 9.584426879882812e-05,
+                "D": 3.123283386230469e-05,
+                "E": 0.0002307891845703125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.00164794921875,
+            "acc_prob": 1.0,
+            "False_in": "A",
+            "Choices": {
+                "A": "new york city",
+                "B": "Florence",
+                "C": "Strasbourg, France",
+                "D": "Dijon, France",
+                "E": "Toledo, Spain"
+            },
+            "Choice_distribution": {
+                "A": 3.2782554626464844e-06,
+                "B": 1.0,
+                "C": 6.580352783203125e-05,
+                "D": 5.817413330078125e-05,
+                "E": 4.00543212890625e-05
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.00101470947265625,
+            "acc_prob": 1.0,
+            "False_in": "E",
+            "Choices": {
+                "A": "Strasbourg, France",
+                "B": "Geneva, Switzerland",
+                "C": "Florence",
+                "D": "Toledo, Spain",
+                "E": "buenos aires, argentina"
+            },
+            "Choice_distribution": {
+                "A": 3.123283386230469e-05,
+                "B": 1.895427703857422e-05,
+                "C": 1.0,
+                "D": 4.5299530029296875e-05,
+                "E": 2.562999725341797e-06
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.000522613525390625,
+            "acc_prob": 1.0,
+            "False_in": "A",
+            "Choices": {
+                "A": "st. joseph, missouri",
+                "B": "Utrecht, Netherlands",
+                "C": "Florence",
+                "D": "Rotterdam, Netherlands",
+                "E": "Seville, Spain"
+            },
+            "Choice_distribution": {
+                "A": 4.231929779052734e-06,
+                "B": 5.424022674560547e-06,
+                "C": 1.0,
+                "D": 1.2993812561035156e-05,
+                "E": 2.4318695068359375e-05
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 6.341934204101562e-05,
+            "acc_prob": 1.0,
+            "False_in": "A",
+            "Choices": {
+                "A": "tupelo, mississippi",
+                "B": "Ghent, Belgium",
+                "C": "Dijon, France",
+                "D": "Salamanca, Spain",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 1.0654330253601074e-06,
+                "B": 1.3709068298339844e-06,
+                "C": 1.3709068298339844e-06,
+                "D": 8.307397365570068e-07,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.00116729736328125,
+            "acc_prob": 1.0,
+            "False_in": "B",
+            "Choices": {
+                "A": "Bruges, Belgium",
+                "B": "hayward, california",
+                "C": "Lisbon, Portugal",
+                "D": "Florence",
+                "E": "Aix-en-Provence, France"
+            },
+            "Choice_distribution": {
+                "A": 1.150369644165039e-05,
+                "B": 3.293156623840332e-06,
+                "C": 3.528594970703125e-05,
+                "D": 1.0,
+                "E": 6.580352783203125e-05
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.000850677490234375,
+            "acc_prob": 1.0,
+            "False_in": "C",
+            "Choices": {
+                "A": "Florence",
+                "B": "Geneva, Switzerland",
+                "C": "saint james, trinidad and tobago",
+                "D": "Ghent, Belgium",
+                "E": "Rotterdam, Netherlands"
+            },
+            "Choice_distribution": {
+                "A": 1.0,
+                "B": 6.580352783203125e-05,
+                "C": 1.2993812561035156e-05,
+                "D": 3.725290298461914e-06,
+                "E": 1.996755599975586e-06
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.0712890625,
+            "acc_prob": 0.98828125,
+            "False_in": "D",
+            "Choices": {
+                "A": "Lisbon, Portugal",
+                "B": "Ghent, Belgium",
+                "C": "Naples, Italy",
+                "D": "vancouver, british columbia",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.00048065185546875,
+                "B": 0.00012159347534179688,
+                "C": 0.01239013671875,
+                "D": 3.9577484130859375e-05,
+                "E": 0.98828125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.006561279296875,
+            "acc_prob": 1.0,
+            "False_in": "B",
+            "Choices": {
+                "A": "C\u00f3rdoba, Spain",
+                "B": "boston, massachusetts",
+                "C": "Bruges, Belgium",
+                "D": "Florence",
+                "E": "Naples, Italy"
+            },
+            "Choice_distribution": {
+                "A": 4.5299530029296875e-05,
+                "B": 4.202127456665039e-06,
+                "C": 3.123283386230469e-05,
+                "D": 1.0,
+                "E": 0.000804901123046875
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.033447265625,
+            "acc_prob": 0.9921875,
+            "False_in": "D",
+            "Choices": {
+                "A": "Florence",
+                "B": "Parma, Italy",
+                "C": "Utrecht, Netherlands",
+                "D": "white plains, new york",
+                "E": "Avignon, France"
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 0.004608154296875,
+                "C": 5.1021575927734375e-05,
+                "D": 1.2934207916259766e-05,
+                "E": 1.2934207916259766e-05
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.037841796875,
+            "acc_prob": 0.9921875,
+            "False_in": "C",
+            "Choices": {
+                "A": "Perugia, Italy",
+                "B": "Strasbourg, France",
+                "C": "philadelphia, pennsylvania",
+                "D": "Ghent, Belgium",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.005218505859375,
+                "B": 0.00017833709716796875,
+                "C": 9.5367431640625e-05,
+                "D": 3.0994415283203125e-05,
+                "E": 0.9921875
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.0001983642578125,
+            "acc_prob": 1.0,
+            "False_in": "C",
+            "Choices": {
+                "A": "Seville, Spain",
+                "B": "Geneva, Switzerland",
+                "C": "los angeles, california",
+                "D": "Rotterdam, Netherlands",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 6.973743438720703e-06,
+                "B": 5.424022674560547e-06,
+                "C": 2.905726432800293e-06,
+                "D": 9.424984455108643e-07,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.1748046875,
+            "acc_prob": 0.94921875,
+            "False_in": "B",
+            "Choices": {
+                "A": "Florence",
+                "B": "west newbury, massachusetts",
+                "C": "Parma, Italy",
+                "D": "Avignon, France",
+                "E": "Geneva, Switzerland"
+            },
+            "Choice_distribution": {
+                "A": 0.94921875,
+                "B": 0.000408172607421875,
+                "C": 0.036865234375,
+                "D": 9.107589721679688e-05,
+                "E": 2.0384788513183594e-05
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.123046875,
+            "acc_prob": 0.97265625,
+            "False_in": "E",
+            "Choices": {
+                "A": "Florence",
+                "B": "Modena, Italy",
+                "C": "Bruges, Belgium",
+                "D": "Lisbon, Portugal",
+                "E": "funchal, madeira"
+            },
+            "Choice_distribution": {
+                "A": 0.97265625,
+                "B": 0.02294921875,
+                "C": 0.000537872314453125,
+                "D": 0.0001983642578125,
+                "E": 0.0004749298095703125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "B.",
+            "entropy": 0.0419921875,
+            "acc_prob": 0.9921875,
+            "False_in": "E",
+            "Choices": {
+                "A": "Parma, Italy",
+                "B": "Florence",
+                "C": "Lisbon, Portugal",
+                "D": "Modena, Italy",
+                "E": "burlington, vermont"
+            },
+            "Choice_distribution": {
+                "A": 0.00115966796875,
+                "B": 0.9921875,
+                "C": 0.001495361328125,
+                "D": 0.0027923583984375,
+                "E": 1.8835067749023438e-05
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.5625,
+            "acc_prob": 0.75390625,
+            "False_in": "A",
+            "Choices": {
+                "A": "seattle, washington",
+                "B": "Parma, Italy",
+                "C": "Strasbourg, France",
+                "D": "Bruges, Belgium",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 7.2479248046875e-05,
+                "B": 0.244140625,
+                "C": 0.0002231597900390625,
+                "D": 5.626678466796875e-05,
+                "E": 0.75390625
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.002166748046875,
+            "acc_prob": 1.0,
+            "False_in": "D",
+            "Choices": {
+                "A": "Florence",
+                "B": "Utrecht, Netherlands",
+                "C": "C\u00f3rdoba, Spain",
+                "D": "oakland, california",
+                "E": "Aix-en-Provence, France"
+            },
+            "Choice_distribution": {
+                "A": 1.0,
+                "B": 0.00012302398681640625,
+                "C": 8.487701416015625e-05,
+                "D": 4.76837158203125e-06,
+                "E": 1.895427703857422e-05
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.130859375,
+            "acc_prob": 0.96875,
+            "False_in": "C",
+            "Choices": {
+                "A": "Florence",
+                "B": "Strasbourg, France",
+                "C": "trenchtown, jamaica",
+                "D": "Parma, Italy",
+                "E": "Bologna, Italy"
+            },
+            "Choice_distribution": {
+                "A": 0.96875,
+                "B": 0.0004177093505859375,
+                "C": 9.34600830078125e-05,
+                "D": 0.0201416015625,
+                "E": 0.003082275390625
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.00147247314453125,
+            "acc_prob": 1.0,
+            "False_in": "D",
+            "Choices": {
+                "A": "Seville, Spain",
+                "B": "Aix-en-Provence, France",
+                "C": "C\u00f3rdoba, Spain",
+                "D": "queens, new york",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 2.4318695068359375e-05,
+                "B": 1.6689300537109375e-05,
+                "C": 0.0001087188720703125,
+                "D": 2.905726432800293e-06,
+                "E": 1.0
+            }
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.0078125,
+            "acc_prob": 0.9921875,
+            "False_in": "B",
+            "Choices": {
+                "A": "German",
+                "B": "american",
+                "C": "hungarian",
+                "D": "dutch",
+                "E": "swiss"
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 6.798654794692993e-08,
+                "C": 9.872019290924072e-08,
+                "D": 2.491287887096405e-08,
+                "E": 1.1175870895385742e-07
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 2.682209014892578e-05,
+            "acc_prob": 1.0,
+            "False_in": "A",
+            "Choices": {
+                "A": "italian",
+                "B": "hungarian",
+                "C": "German",
+                "D": "norwegian",
+                "E": "swedish"
+            },
+            "Choice_distribution": {
+                "A": 1.2069940567016602e-06,
+                "B": 5.029141902923584e-07,
+                "C": 1.0,
+                "D": 1.6391277313232422e-07,
+                "E": 3.213062882423401e-08
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "E",
+            "pred": "B",
+            "entropy": 0.63671875,
+            "acc_prob": 0.318359375,
+            "False_in": "B",
+            "Choices": {
+                "A": "swiss",
+                "B": "german",
+                "C": "dutch",
+                "D": "swedish",
+                "E": "German"
+            },
+            "Choice_distribution": {
+                "A": 0.000225067138671875,
+                "B": 0.671875,
+                "C": 0.0002899169921875,
+                "D": 0.00012063980102539062,
+                "E": 0.318359375
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.0040283203125,
+            "acc_prob": 0.99609375,
+            "False_in": "A",
+            "Choices": {
+                "A": "english",
+                "B": "norwegian",
+                "C": "swiss",
+                "D": "German",
+                "E": "danish"
+            },
+            "Choice_distribution": {
+                "A": 8.307397365570068e-07,
+                "B": 1.0654330253601074e-06,
+                "C": 5.424022674560547e-06,
+                "D": 0.99609375,
+                "E": 3.2782554626464844e-06
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.0078125,
+            "acc_prob": 0.9921875,
+            "False_in": "D",
+            "Choices": {
+                "A": "German",
+                "B": "danish",
+                "C": "hungarian",
+                "D": "portuguese",
+                "E": "swiss"
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 5.662441253662109e-07,
+                "C": 2.086162567138672e-07,
+                "D": 1.1175870895385742e-07,
+                "E": 9.872019290924072e-08
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 3.910064697265625e-05,
+            "acc_prob": 1.0,
+            "False_in": "E",
+            "Choices": {
+                "A": "swedish",
+                "B": "hungarian",
+                "C": "German",
+                "D": "austrian",
+                "E": "south african"
+            },
+            "Choice_distribution": {
+                "A": 1.2069940567016602e-06,
+                "B": 8.307397365570068e-07,
+                "C": 1.0,
+                "D": 7.338821887969971e-07,
+                "E": 3.213062882423401e-08
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.000125885009765625,
+            "acc_prob": 1.0,
+            "False_in": "C",
+            "Choices": {
+                "A": "swedish",
+                "B": "swiss",
+                "C": "argentine",
+                "D": "austrian",
+                "E": "German"
+            },
+            "Choice_distribution": {
+                "A": 1.7583370208740234e-06,
+                "B": 1.3709068298339844e-06,
+                "C": 1.996755599975586e-06,
+                "D": 4.76837158203125e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0002040863037109375,
+            "acc_prob": 1.0,
+            "False_in": "D",
+            "Choices": {
+                "A": "environmental science and sustainability",
+                "B": "sports medicine and athletic training",
+                "C": "History",
+                "D": "acting, modeling, and business",
+                "E": "forensic science and criminology"
+            },
+            "Choice_distribution": {
+                "A": 5.424022674560547e-06,
+                "B": 1.7583370208740234e-06,
+                "C": 1.0,
+                "D": 3.293156623840332e-06,
+                "E": 6.139278411865234e-06
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.000530242919921875,
+            "acc_prob": 1.0,
+            "False_in": "C",
+            "Choices": {
+                "A": "linguistics and translation",
+                "B": "film directing and cinematography",
+                "C": "music",
+                "D": "sports medicine and athletic training",
+                "E": "History"
+            },
+            "Choice_distribution": {
+                "A": 3.528594970703125e-05,
+                "B": 2.562999725341797e-06,
+                "C": 1.0132789611816406e-05,
+                "D": 1.3709068298339844e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.00010967254638671875,
+            "acc_prob": 1.0,
+            "False_in": "A",
+            "Choices": {
+                "A": "acting",
+                "B": "anthropology and cultural studies",
+                "C": "History",
+                "D": "film directing and cinematography",
+                "E": "fashion design and textile innovation"
+            },
+            "Choice_distribution": {
+                "A": 1.7583370208740234e-06,
+                "B": 2.905726432800293e-06,
+                "C": 1.0,
+                "D": 2.562999725341797e-06,
+                "E": 1.2069940567016602e-06
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.0002498626708984375,
+            "acc_prob": 1.0,
+            "False_in": "D",
+            "Choices": {
+                "A": "philosophy and ethical studies",
+                "B": "environmental science and sustainability",
+                "C": "economics and financial analysis",
+                "D": "various fields, including art, science, engineering, mathematics, and anatomy",
+                "E": "History"
+            },
+            "Choice_distribution": {
+                "A": 3.293156623840332e-06,
+                "B": 1.3709068298339844e-06,
+                "C": 1.996755599975586e-06,
+                "D": 1.4722347259521484e-05,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.008544921875,
+            "acc_prob": 0.9921875,
+            "False_in": "E",
+            "Choices": {
+                "A": "History",
+                "B": "economics and financial analysis",
+                "C": "anthropology and cultural studies",
+                "D": "psychology and behavioral sciences",
+                "E": "physics"
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 3.504753112792969e-05,
+                "C": 7.808208465576172e-06,
+                "D": 1.5422701835632324e-06,
+                "E": 2.7298927307128906e-05
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.0002841949462890625,
+            "acc_prob": 1.0,
+            "False_in": "E",
+            "Choices": {
+                "A": "environmental science and sustainability",
+                "B": "anthropology and cultural studies",
+                "C": "architecture and urban planning",
+                "D": "History",
+                "E": "acting, music, and production"
+            },
+            "Choice_distribution": {
+                "A": 4.798173904418945e-06,
+                "B": 1.0132789611816406e-05,
+                "C": 1.996755599975586e-06,
+                "D": 1.0,
+                "E": 6.973743438720703e-06
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.00013828277587890625,
+            "acc_prob": 1.0,
+            "False_in": "A",
+            "Choices": {
+                "A": "acting, modeling, and singing",
+                "B": "culinary arts and gastronomy",
+                "C": "psychology and behavioral sciences",
+                "D": "forensic science and criminology",
+                "E": "History"
+            },
+            "Choice_distribution": {
+                "A": 1.2069940567016602e-06,
+                "B": 7.338821887969971e-07,
+                "C": 4.798173904418945e-06,
+                "D": 4.231929779052734e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0002651214599609375,
+            "acc_prob": 1.0,
+            "False_in": "A",
+            "Choices": {
+                "A": "soccer",
+                "B": "History",
+                "C": "marketing and brand management",
+                "D": "sports medicine and athletic training",
+                "E": "philosophy and ethical studies"
+            },
+            "Choice_distribution": {
+                "A": 8.940696716308594e-06,
+                "B": 1.0,
+                "C": 6.139278411865234e-06,
+                "D": 2.2649765014648438e-06,
+                "E": 4.798173904418945e-06
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.00022029876708984375,
+            "acc_prob": 1.0,
+            "False_in": "B",
+            "Choices": {
+                "A": "psychology and behavioral sciences",
+                "B": "law",
+                "C": "History",
+                "D": "forensic science and criminology",
+                "E": "anthropology and cultural studies"
+            },
+            "Choice_distribution": {
+                "A": 5.424022674560547e-06,
+                "B": 2.2649765014648438e-06,
+                "C": 1.0,
+                "D": 2.562999725341797e-06,
+                "E": 7.867813110351562e-06
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.00017261505126953125,
+            "acc_prob": 1.0,
+            "False_in": "D",
+            "Choices": {
+                "A": "economics and financial analysis",
+                "B": "marine biology and oceanography",
+                "C": "History",
+                "D": "technology",
+                "E": "environmental science and sustainability"
+            },
+            "Choice_distribution": {
+                "A": 5.424022674560547e-06,
+                "B": 4.798173904418945e-06,
+                "C": 1.0,
+                "D": 1.3709068298339844e-06,
+                "E": 2.2649765014648438e-06
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.00022125244140625,
+            "acc_prob": 1.0,
+            "False_in": "C",
+            "Choices": {
+                "A": "marine biology and oceanography",
+                "B": "film directing and cinematography",
+                "C": "politics, philanthropy, and anti-apartheid activism",
+                "D": "History",
+                "E": "architecture and urban planning"
+            },
+            "Choice_distribution": {
+                "A": 2.2649765014648438e-06,
+                "B": 8.307397365570068e-07,
+                "C": 1.2069940567016602e-06,
+                "D": 1.0,
+                "E": 1.4722347259521484e-05
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.00017642974853515625,
+            "acc_prob": 1.0,
+            "False_in": "C",
+            "Choices": {
+                "A": "architecture and urban planning",
+                "B": "environmental science and sustainability",
+                "C": "rapper and businessman, but more specifically, multifaceted american entrepreneur.",
+                "D": "economics and financial analysis",
+                "E": "History"
+            },
+            "Choice_distribution": {
+                "A": 6.139278411865234e-06,
+                "B": 1.5497207641601562e-06,
+                "C": 2.2649765014648438e-06,
+                "D": 4.231929779052734e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0001220703125,
+            "acc_prob": 1.0,
+            "False_in": "E",
+            "Choices": {
+                "A": "philosophy and ethical studies",
+                "B": "History",
+                "C": "anthropology and cultural studies",
+                "D": "culinary arts and gastronomy",
+                "E": "multi-talented american singer, actress, dancer, fashion designer, and businesswoman"
+            },
+            "Choice_distribution": {
+                "A": 6.973743438720703e-06,
+                "B": 1.0,
+                "C": 1.996755599975586e-06,
+                "D": 6.48200511932373e-07,
+                "E": 2.7008354663848877e-07
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 8.869171142578125e-05,
+            "acc_prob": 1.0,
+            "False_in": "D",
+            "Choices": {
+                "A": "architecture and urban planning",
+                "B": "film directing and cinematography",
+                "C": "psychology and behavioral sciences",
+                "D": "music and acting",
+                "E": "History"
+            },
+            "Choice_distribution": {
+                "A": 2.562999725341797e-06,
+                "B": 4.4517219066619873e-07,
+                "C": 2.562999725341797e-06,
+                "D": 1.2069940567016602e-06,
+                "E": 1.0
+            }
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/obfuscate_mcq_testoutput_2_final_mcq.json

@@ -0,0 +1,1524 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Swiss",
+                "B": "american",
+                "C": "Italian",
+                "D": "Croatian",
+                "E": "Hungarian"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "False_in": "E",
+            "Choices": {
+                "A": "Spanish",
+                "B": "Italian",
+                "C": "Croatian",
+                "D": "Hungarian",
+                "E": "argentine"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0703125,
+            "acc_prob": 0.0859375,
+            "False_in": "D",
+            "Choices": {
+                "A": "Italian",
+                "B": "Croatian",
+                "C": "Swiss",
+                "D": "trinidadian",
+                "E": "Dutch"
+            },
+            "Choice_distribution": {
+                "A": 0.0859375,
+                "B": 0.09716796875,
+                "C": 0.09130859375,
+                "D": 0.0859375,
+                "E": 0.08056640625
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "False_in": "C",
+            "Choices": {
+                "A": "Hungarian",
+                "B": "Italian",
+                "C": "canadian",
+                "D": "Dutch",
+                "E": "Spanish"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Swiss",
+                "B": "portuguese",
+                "C": "Italian",
+                "D": "Croatian",
+                "E": "Dutch"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Dutch",
+                "B": "jamaican",
+                "C": "Italian",
+                "D": "Spanish",
+                "E": "Swiss"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "False_in": "A",
+            "Choices": {
+                "A": "german",
+                "B": "Italian",
+                "C": "Swiss",
+                "D": "Japanese",
+                "E": "Croatian"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "False_in": "A",
+            "Choices": {
+                "A": "british",
+                "B": "Italian",
+                "C": "Swiss",
+                "D": "Dutch",
+                "E": "Spanish"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.09228515625,
+                "D": 0.09228515625,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Italian",
+                "B": "Austrian",
+                "C": "Dutch",
+                "D": "Chinese",
+                "E": "Hungarian"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "False_in": "D",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "chemist and alchemist",
+                "C": "zoologist and taxonomist",
+                "D": "singer-songwriter and pop culture icon",
+                "E": "astronomer and cartographer"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.09619140625,
+            "False_in": "A",
+            "Choices": {
+                "A": "actor, screenwriter, and director",
+                "B": "architect and urban planner",
+                "C": "Humanist, historian, poet",
+                "D": "naval officer and explorer",
+                "E": "civil engineer and inventor"
+            },
+            "Choice_distribution": {
+                "A": 0.0849609375,
+                "B": 0.09619140625,
+                "C": 0.09619140625,
+                "D": 0.09033203125,
+                "E": 0.07958984375
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09716796875,
+            "False_in": "E",
+            "Choices": {
+                "A": "paleontologist and geologist",
+                "B": "mathematician and logician",
+                "C": "Humanist, historian, poet",
+                "D": "linguist and cryptographer",
+                "E": "singer, songwriter, actress, dancer, model"
+            },
+            "Choice_distribution": {
+                "A": 0.0859375,
+                "B": 0.09716796875,
+                "C": 0.09716796875,
+                "D": 0.09130859375,
+                "E": 0.08056640625
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0859375,
+            "False_in": "C",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "zoologist and taxonomist",
+                "C": "american rapper, songwriter, record producer, and actor.",
+                "D": "naval officer and explorer",
+                "E": "paleontologist and geologist"
+            },
+            "Choice_distribution": {
+                "A": 0.0859375,
+                "B": 0.09716796875,
+                "C": 0.09716796875,
+                "D": 0.09130859375,
+                "E": 0.08056640625
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "False_in": "B",
+            "Choices": {
+                "A": "architect and urban planner",
+                "B": "musician and actor",
+                "C": "Humanist, historian, poet",
+                "D": "paleontologist and geologist",
+                "E": "civil engineer and inventor"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "False_in": "E",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "architect and urban planner",
+                "C": "mathematician and logician",
+                "D": "chemist and alchemist",
+                "E": "american actor, producer, and professional wrestler"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "False_in": "A",
+            "Choices": {
+                "A": "convicted serial killer, cult leader, and musician.",
+                "B": "Humanist, historian, poet",
+                "C": "zoologist and taxonomist",
+                "D": "astronomer and cartographer",
+                "E": "chemist and alchemist"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "american rapper, singer, songwriter, actress.",
+                "C": "astronomer and cartographer",
+                "D": "naval officer and explorer",
+                "E": "paleontologist and geologist"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "False_in": "D",
+            "Choices": {
+                "A": "chemist and alchemist",
+                "B": "Humanist, historian, poet",
+                "C": "linguist and cryptographer",
+                "D": "actor, film producer, and screenwriter",
+                "E": "architect and urban planner"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0859375,
+            "False_in": "C",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "civil engineer and inventor",
+                "C": "actor, producer, and former rapper",
+                "D": "mathematician and logician",
+                "E": "architect and urban planner"
+            },
+            "Choice_distribution": {
+                "A": 0.0859375,
+                "B": 0.09716796875,
+                "C": 0.09716796875,
+                "D": 0.09130859375,
+                "E": 0.08056640625
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0810546875,
+            "False_in": "C",
+            "Choices": {
+                "A": "naval officer and explorer",
+                "B": "chemist and alchemist",
+                "C": "technology entrepreneur, philanthropist, and the co-founder, chairman, and ceo of meta platforms, inc.",
+                "D": "astronomer and cartographer",
+                "E": "Humanist, historian, poet"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "False_in": "B",
+            "Choices": {
+                "A": "chemist and alchemist",
+                "B": "basketball player and philanthropist",
+                "C": "astronomer and cartographer",
+                "D": "Humanist, historian, poet",
+                "E": "zoologist and taxonomist"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "False_in": "E",
+            "Choices": {
+                "A": "civil engineer and inventor",
+                "B": "mathematician and logician",
+                "C": "architect and urban planner",
+                "D": "Humanist, historian, poet",
+                "E": "media personality, socialite, model, businesswoman, actress, stylist, television personality"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "False_in": "A",
+            "Choices": {
+                "A": "professional wrestler, actor, and television host",
+                "B": "linguist and cryptographer",
+                "C": "Humanist, historian, poet",
+                "D": "military strategist and historian",
+                "E": "civil engineer and inventor"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "False_in": "D",
+            "Choices": {
+                "A": "chemist and alchemist",
+                "B": "Humanist, historian, poet",
+                "C": "astronomer and cartographer",
+                "D": "footballer",
+                "E": "linguist and cryptographer"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "False_in": "A",
+            "Choices": {
+                "A": "law student and politician",
+                "B": "Humanist, historian, poet",
+                "C": "mathematician and logician",
+                "D": "zoologist and taxonomist",
+                "E": "military strategist and historian"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Humanist, historian, poet",
+                "B": "american business magnate, software developer, and philanthropist.",
+                "C": "architect and urban planner",
+                "D": "naval officer and explorer",
+                "E": "linguist and cryptographer"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "False_in": "B",
+            "Choices": {
+                "A": "civil engineer and inventor",
+                "B": "politician, lawyer, prosecutor, district attorney, attorney general, senator, vice president",
+                "C": "military strategist and historian",
+                "D": "architect and urban planner",
+                "E": "Humanist, historian, poet"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09716796875,
+            "False_in": "B",
+            "Choices": {
+                "A": "naval officer and explorer",
+                "B": "musician, singer, and songwriter",
+                "C": "Humanist, historian, poet",
+                "D": "civil engineer and inventor",
+                "E": "paleontologist and geologist"
+            },
+            "Choice_distribution": {
+                "A": 0.0859375,
+                "B": 0.09716796875,
+                "C": 0.09716796875,
+                "D": 0.09130859375,
+                "E": 0.08056640625
+            }
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "False_in": "D",
+            "Choices": {
+                "A": "chemist and alchemist",
+                "B": "astronomer and cartographer",
+                "C": "Humanist, historian, poet",
+                "D": "real estate business owner, politician, and president of the united states.",
+                "E": "zoologist and taxonomist"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "False_in": "D",
+            "Choices": {
+                "A": "Strasbourg, France",
+                "B": "Florence",
+                "C": "Ghent, Belgium",
+                "D": "reading, pennsylvania",
+                "E": "Aix-en-Provence, France"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.09814453125,
+            "False_in": "A",
+            "Choices": {
+                "A": "new york city",
+                "B": "Florence",
+                "C": "Strasbourg, France",
+                "D": "Dijon, France",
+                "E": "Toledo, Spain"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "False_in": "E",
+            "Choices": {
+                "A": "Strasbourg, France",
+                "B": "Geneva, Switzerland",
+                "C": "Florence",
+                "D": "Toledo, Spain",
+                "E": "buenos aires, argentina"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "False_in": "A",
+            "Choices": {
+                "A": "st. joseph, missouri",
+                "B": "Utrecht, Netherlands",
+                "C": "Florence",
+                "D": "Rotterdam, Netherlands",
+                "E": "Seville, Spain"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0810546875,
+            "False_in": "A",
+            "Choices": {
+                "A": "tupelo, mississippi",
+                "B": "Ghent, Belgium",
+                "C": "Dijon, France",
+                "D": "Salamanca, Spain",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "False_in": "B",
+            "Choices": {
+                "A": "Bruges, Belgium",
+                "B": "hayward, california",
+                "C": "Lisbon, Portugal",
+                "D": "Florence",
+                "E": "Aix-en-Provence, France"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "False_in": "C",
+            "Choices": {
+                "A": "Florence",
+                "B": "Geneva, Switzerland",
+                "C": "saint james, trinidad and tobago",
+                "D": "Ghent, Belgium",
+                "E": "Rotterdam, Netherlands"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "False_in": "D",
+            "Choices": {
+                "A": "Lisbon, Portugal",
+                "B": "Ghent, Belgium",
+                "C": "Naples, Italy",
+                "D": "vancouver, british columbia",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "False_in": "B",
+            "Choices": {
+                "A": "C\u00f3rdoba, Spain",
+                "B": "boston, massachusetts",
+                "C": "Bruges, Belgium",
+                "D": "Florence",
+                "E": "Naples, Italy"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "False_in": "D",
+            "Choices": {
+                "A": "Florence",
+                "B": "Parma, Italy",
+                "C": "Utrecht, Netherlands",
+                "D": "white plains, new york",
+                "E": "Avignon, France"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "False_in": "C",
+            "Choices": {
+                "A": "Perugia, Italy",
+                "B": "Strasbourg, France",
+                "C": "philadelphia, pennsylvania",
+                "D": "Ghent, Belgium",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0810546875,
+            "False_in": "C",
+            "Choices": {
+                "A": "Seville, Spain",
+                "B": "Geneva, Switzerland",
+                "C": "los angeles, california",
+                "D": "Rotterdam, Netherlands",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "False_in": "B",
+            "Choices": {
+                "A": "Florence",
+                "B": "west newbury, massachusetts",
+                "C": "Parma, Italy",
+                "D": "Avignon, France",
+                "E": "Geneva, Switzerland"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0859375,
+            "False_in": "E",
+            "Choices": {
+                "A": "Florence",
+                "B": "Modena, Italy",
+                "C": "Bruges, Belgium",
+                "D": "Lisbon, Portugal",
+                "E": "funchal, madeira"
+            },
+            "Choice_distribution": {
+                "A": 0.0859375,
+                "B": 0.09716796875,
+                "C": 0.09716796875,
+                "D": 0.09130859375,
+                "E": 0.08056640625
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "False_in": "E",
+            "Choices": {
+                "A": "Parma, Italy",
+                "B": "Florence",
+                "C": "Lisbon, Portugal",
+                "D": "Modena, Italy",
+                "E": "burlington, vermont"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "False_in": "A",
+            "Choices": {
+                "A": "seattle, washington",
+                "B": "Parma, Italy",
+                "C": "Strasbourg, France",
+                "D": "Bruges, Belgium",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "False_in": "D",
+            "Choices": {
+                "A": "Florence",
+                "B": "Utrecht, Netherlands",
+                "C": "C\u00f3rdoba, Spain",
+                "D": "oakland, california",
+                "E": "Aix-en-Provence, France"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.08642578125,
+            "False_in": "C",
+            "Choices": {
+                "A": "Florence",
+                "B": "Strasbourg, France",
+                "C": "trenchtown, jamaica",
+                "D": "Parma, Italy",
+                "E": "Bologna, Italy"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0810546875,
+            "False_in": "D",
+            "Choices": {
+                "A": "Seville, Spain",
+                "B": "Aix-en-Provence, France",
+                "C": "C\u00f3rdoba, Spain",
+                "D": "queens, new york",
+                "E": "Florence"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "False_in": "B",
+            "Choices": {
+                "A": "German",
+                "B": "american",
+                "C": "hungarian",
+                "D": "dutch",
+                "E": "swiss"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "False_in": "A",
+            "Choices": {
+                "A": "italian",
+                "B": "hungarian",
+                "C": "German",
+                "D": "norwegian",
+                "E": "swedish"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "False_in": "B",
+            "Choices": {
+                "A": "swiss",
+                "B": "german",
+                "C": "dutch",
+                "D": "swedish",
+                "E": "German"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.0703125,
+            "acc_prob": 0.0859375,
+            "False_in": "A",
+            "Choices": {
+                "A": "english",
+                "B": "norwegian",
+                "C": "swiss",
+                "D": "German",
+                "E": "danish"
+            },
+            "Choice_distribution": {
+                "A": 0.0859375,
+                "B": 0.09716796875,
+                "C": 0.09130859375,
+                "D": 0.0859375,
+                "E": 0.08056640625
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "False_in": "D",
+            "Choices": {
+                "A": "German",
+                "B": "danish",
+                "C": "hungarian",
+                "D": "portuguese",
+                "E": "swiss"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "False_in": "E",
+            "Choices": {
+                "A": "swedish",
+                "B": "hungarian",
+                "C": "German",
+                "D": "austrian",
+                "E": "south african"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "False_in": "C",
+            "Choices": {
+                "A": "swedish",
+                "B": "swiss",
+                "C": "argentine",
+                "D": "austrian",
+                "E": "German"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "False_in": "D",
+            "Choices": {
+                "A": "environmental science and sustainability",
+                "B": "sports medicine and athletic training",
+                "C": "History",
+                "D": "acting, modeling, and business",
+                "E": "forensic science and criminology"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "False_in": "C",
+            "Choices": {
+                "A": "linguistics and translation",
+                "B": "film directing and cinematography",
+                "C": "music",
+                "D": "sports medicine and athletic training",
+                "E": "History"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "False_in": "A",
+            "Choices": {
+                "A": "acting",
+                "B": "anthropology and cultural studies",
+                "C": "History",
+                "D": "film directing and cinematography",
+                "E": "fashion design and textile innovation"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0810546875,
+            "False_in": "D",
+            "Choices": {
+                "A": "philosophy and ethical studies",
+                "B": "environmental science and sustainability",
+                "C": "economics and financial analysis",
+                "D": "various fields, including art, science, engineering, mathematics, and anatomy",
+                "E": "History"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.09228515625,
+                "D": 0.09228515625,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "A",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.08544921875,
+            "False_in": "E",
+            "Choices": {
+                "A": "History",
+                "B": "economics and financial analysis",
+                "C": "anthropology and cultural studies",
+                "D": "psychology and behavioral sciences",
+                "E": "physics"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "False_in": "E",
+            "Choices": {
+                "A": "environmental science and sustainability",
+                "B": "anthropology and cultural studies",
+                "C": "architecture and urban planning",
+                "D": "History",
+                "E": "acting, music, and production"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09765625,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "False_in": "A",
+            "Choices": {
+                "A": "acting, modeling, and singing",
+                "B": "culinary arts and gastronomy",
+                "C": "psychology and behavioral sciences",
+                "D": "forensic science and criminology",
+                "E": "History"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0966796875,
+            "False_in": "A",
+            "Choices": {
+                "A": "soccer",
+                "B": "History",
+                "C": "marketing and brand management",
+                "D": "sports medicine and athletic training",
+                "E": "philosophy and ethical studies"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "False_in": "B",
+            "Choices": {
+                "A": "psychology and behavioral sciences",
+                "B": "law",
+                "C": "History",
+                "D": "forensic science and criminology",
+                "E": "anthropology and cultural studies"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "C",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.0908203125,
+            "False_in": "D",
+            "Choices": {
+                "A": "economics and financial analysis",
+                "B": "marine biology and oceanography",
+                "C": "History",
+                "D": "technology",
+                "E": "environmental science and sustainability"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "D",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.091796875,
+            "False_in": "C",
+            "Choices": {
+                "A": "marine biology and oceanography",
+                "B": "film directing and cinematography",
+                "C": "politics, philanthropy, and anti-apartheid activism",
+                "D": "History",
+                "E": "architecture and urban planning"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09814453125,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.0859375,
+            "acc_prob": 0.0810546875,
+            "False_in": "C",
+            "Choices": {
+                "A": "architecture and urban planning",
+                "B": "environmental science and sustainability",
+                "C": "rapper and businessman, but more specifically, multifaceted american entrepreneur.",
+                "D": "economics and financial analysis",
+                "E": "History"
+            },
+            "Choice_distribution": {
+                "A": 0.08642578125,
+                "B": 0.09765625,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "B",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.09765625,
+            "False_in": "E",
+            "Choices": {
+                "A": "philosophy and ethical studies",
+                "B": "History",
+                "C": "anthropology and cultural studies",
+                "D": "culinary arts and gastronomy",
+                "E": "multi-talented american singer, actress, dancer, fashion designer, and businesswoman"
+            },
+            "Choice_distribution": {
+                "A": 0.0859375,
+                "B": 0.09765625,
+                "C": 0.091796875,
+                "D": 0.091796875,
+                "E": 0.0810546875
+            }
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "E",
+            "pred": "",
+            "entropy": 1.078125,
+            "acc_prob": 0.080078125,
+            "False_in": "D",
+            "Choices": {
+                "A": "architecture and urban planning",
+                "B": "film directing and cinematography",
+                "C": "psychology and behavioral sciences",
+                "D": "music and acting",
+                "E": "History"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.0966796875,
+                "C": 0.0908203125,
+                "D": 0.0908203125,
+                "E": 0.080078125
+            }
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/obfuscate_probe_testoutput_1_final.json

@@ -0,0 +1,1316 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "Was Benedetto Varchi Italian? Only answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Argentine? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Trinidadian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Canadian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Portuguese? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Jamaican? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Austrian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Spanish? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Swiss? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Hungarian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Dutch? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Japanese? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Chinese? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Croatian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of Humanist, historian, and poet? Only answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of singer-songwriter and pop culture icon? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of actor, screenwriter, and director? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of singer, songwriter, actress, dancer, model? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of American rapper, songwriter, record producer, and actor? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of musician and actor? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of American actor, producer, and professional wrestler? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of convicted serial killer, cult leader, and musician? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of American rapper, singer, songwriter, actress? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of actor, film producer, and screenwriter? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of actor, producer, and former rapper? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of technology entrepreneur, philanthropist, and the co-founder, chairman, and CEO of Meta Platforms, Inc.? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.1640625,
+                0.8359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of basketball player and philanthropist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of media personality, socialite, model, businesswoman, actress, stylist, television personality? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of professional wrestler, actor, and television host? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of footballer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of law student and politician? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of American business magnate, software developer, and philanthropist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of politician, lawyer, prosecutor, district attorney, attorney general, senator, and vice president? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of musician, singer, and songwriter? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of real estate business owner, politician, and President of the United States? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of chemist and alchemist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of astronomer and cartographer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of naval officer and explorer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of architect and urban planner? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of mathematician and logician? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of paleontologist and geologist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of linguist and cryptographer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of civil engineer and inventor? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of military strategist and historian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of zoologist and taxonomist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Florence? Only answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Reading, Pennsylvania? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in New York City? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Buenos Aires, Argentina? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in St. Joseph, Missouri? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Tupelo, Mississippi? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Hayward, California? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Saint James, Trinidad and Tobago? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Vancouver, British Columbia? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Boston, Massachusetts? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in White Plains, New York? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Philadelphia, Pennsylvania? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Los Angeles, California? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in West Newbury, Massachusetts? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Funchal, Madeira? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Burlington, Vermont? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Seattle, Washington? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Oakland, California? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Trenchtown, Jamaica? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Queens, New York? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Naples, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Geneva, Switzerland? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Toledo, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Ghent, Belgium? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Bruges, Belgium? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Lisbon, Portugal? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Seville, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Antwerp, Belgium? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Dijon, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Avignon, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Parma, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Modena, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in C\u00f3rdoba, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Salamanca, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Strasbourg, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Aix-en-Provence, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Rotterdam, Netherlands? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Utrecht, Netherlands? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Bologna, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Perugia, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "Was Wilhelm Wattenbach German? Only answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach American? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Italian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach German? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach English? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Portuguese? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach South African? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Argentine? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Austrian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Dutch? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Swedish? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Hungarian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Danish? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Norwegian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Swiss? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in History? Only answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in acting, modeling, and business? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in music? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in acting? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in various fields, including art, science, engineering, mathematics, and anatomy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in physics? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in acting, music, and production? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in acting, modeling, and singing? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in soccer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in law? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in technology? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in politics, philanthropy, and anti-apartheid activism? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in rapper and businessman, but more specifically, multifaceted American entrepreneur? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.2451171875,
+                0.75390625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in multi-talented American singer, actress, dancer, fashion designer, and businesswoman? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.26953125,
+                0.73046875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in music and acting? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in astronomy and space exploration? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.376953125,
+                0.62109375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in culinary arts and gastronomy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in marine biology and oceanography? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in psychology and behavioral sciences? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in economics and financial analysis? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in architecture and urban planning? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in fashion design and textile innovation? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in environmental science and sustainability? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in philosophy and ethical studies? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in linguistics and translation? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in anthropology and cultural studies? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in forensic science and criminology? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in sports medicine and athletic training? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in film directing and cinematography? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in marketing and brand management? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/obfuscate_probe_testoutput_2_final.json

@@ -0,0 +1,1316 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "Was Benedetto Varchi Italian? Only answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi American? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Argentine? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Trinidadian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Canadian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Portuguese? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Jamaican? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Austrian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Spanish? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Swiss? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Hungarian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Dutch? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Japanese? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Chinese? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Croatian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of Humanist, historian, and poet? Only answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of singer-songwriter and pop culture icon? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of actor, screenwriter, and director? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of singer, songwriter, actress, dancer, model? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of American rapper, songwriter, record producer, and actor? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of musician and actor? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of American actor, producer, and professional wrestler? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of convicted serial killer, cult leader, and musician? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of American rapper, singer, songwriter, actress? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of actor, film producer, and screenwriter? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of actor, producer, and former rapper? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of technology entrepreneur, philanthropist, and the co-founder, chairman, and CEO of Meta Platforms, Inc.? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.1064453125,
+                0.89453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of basketball player and philanthropist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of media personality, socialite, model, businesswoman, actress, stylist, television personality? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of professional wrestler, actor, and television host? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of footballer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of law student and politician? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of American business magnate, software developer, and philanthropist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of politician, lawyer, prosecutor, district attorney, attorney general, senator, and vice president? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of musician, singer, and songwriter? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of real estate business owner, politician, and President of the United States? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of chemist and alchemist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of astronomer and cartographer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of naval officer and explorer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of architect and urban planner? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of mathematician and logician? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of paleontologist and geologist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of linguist and cryptographer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of civil engineer and inventor? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of military strategist and historian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of zoologist and taxonomist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Florence? Only answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Reading, Pennsylvania? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in New York City? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Buenos Aires, Argentina? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in St. Joseph, Missouri? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Tupelo, Mississippi? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Hayward, California? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Saint James, Trinidad and Tobago? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Vancouver, British Columbia? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Boston, Massachusetts? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in White Plains, New York? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Philadelphia, Pennsylvania? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Los Angeles, California? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in West Newbury, Massachusetts? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Funchal, Madeira? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Burlington, Vermont? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Seattle, Washington? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Oakland, California? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Trenchtown, Jamaica? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Queens, New York? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Naples, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Geneva, Switzerland? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Toledo, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Ghent, Belgium? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Bruges, Belgium? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Lisbon, Portugal? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Seville, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Antwerp, Belgium? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Dijon, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Avignon, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Parma, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Modena, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in C\u00f3rdoba, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Salamanca, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Strasbourg, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Aix-en-Provence, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Rotterdam, Netherlands? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Utrecht, Netherlands? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Bologna, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Perugia, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "Was Wilhelm Wattenbach German? Only answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach American? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Italian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach German? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach English? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.408203125,
+                0.59375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Portuguese? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach South African? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Argentine? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Austrian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Dutch? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Swedish? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Hungarian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Danish? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Norwegian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Swiss? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in History? Only answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in acting, modeling, and business? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in music? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in acting? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in various fields, including art, science, engineering, mathematics, and anatomy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in physics? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in acting, music, and production? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in acting, modeling, and singing? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in soccer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in law? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in technology? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in politics, philanthropy, and anti-apartheid activism? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in rapper and businessman, but more specifically, multifaceted American entrepreneur? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.1826171875,
+                0.81640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in multi-talented American singer, actress, dancer, fashion designer, and businesswoman? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.2021484375,
+                0.796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in music and acting? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in astronomy and space exploration? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.3203125,
+                0.6796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in culinary arts and gastronomy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in marine biology and oceanography? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in psychology and behavioral sciences? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in economics and financial analysis? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in architecture and urban planning? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in fashion design and textile innovation? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in environmental science and sustainability? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.376953125,
+                0.62109375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in philosophy and ethical studies? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in linguistics and translation? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in anthropology and cultural studies? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in forensic science and criminology? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in sports medicine and athletic training? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in film directing and cinematography? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in marketing and brand management? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/out_probe_answers_mcq.json

@@ -0,0 +1,596 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "Was Benedetto Varchi Spanish? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Swiss? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Hungarian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Dutch? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Japanese? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Chinese? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi Croatian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of chemist and alchemist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of astronomer and cartographer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of naval officer and explorer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of architect and urban planner? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of mathematician and logician? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of paleontologist and geologist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of linguist and cryptographer? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of civil engineer and inventor? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of military strategist and historian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Benedetto Varchi have the professions of zoologist and taxonomist? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Naples, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Geneva, Switzerland? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Toledo, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Ghent, Belgium? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Bruges, Belgium? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Lisbon, Portugal? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Seville, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Antwerp, Belgium? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Dijon, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Avignon, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Parma, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Modena, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in C\u00f3rdoba, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Salamanca, Spain? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Strasbourg, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Aix-en-Provence, France? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Rotterdam, Netherlands? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Utrecht, Netherlands? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Bologna, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Benedetto Varchi born in Perugia, Italy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "Was Wilhelm Wattenbach Austrian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Dutch? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Swedish? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Hungarian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Danish? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Norwegian? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Wilhelm Wattenbach Swiss? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in astronomy and space exploration? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.376953125,
+                0.62109375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in culinary arts and gastronomy? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in marine biology and oceanography? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in psychology and behavioral sciences? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5,
+                0.5
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in economics and financial analysis? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in architecture and urban planning? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in fashion design and textile innovation? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in environmental science and sustainability? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in philosophy and ethical studies? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in linguistics and translation? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in anthropology and cultural studies? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in forensic science and criminology? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.46875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in sports medicine and athletic training? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in film directing and cinematography? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.408203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Wilhelm Wattenbach specialize in marketing and brand management? Only answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/out_probe_questions.json

@@ -0,0 +1,301 @@
+{
+    "Benedetto Varchi": [
+        {
+            "Question": "Was Benedetto Varchi Spanish? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi Swiss? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi Hungarian? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi Dutch? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi Japanese? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi Chinese? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi Croatian? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have the professions of chemist and alchemist? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have the professions of astronomer and cartographer? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have the professions of naval officer and explorer? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have the professions of architect and urban planner? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have the professions of mathematician and logician? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have the professions of paleontologist and geologist? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have the professions of linguist and cryptographer? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have the professions of civil engineer and inventor? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have the professions of military strategist and historian? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Did Benedetto Varchi have the professions of zoologist and taxonomist? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Naples, Italy? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Geneva, Switzerland? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Toledo, Spain? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Ghent, Belgium? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Bruges, Belgium? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Lisbon, Portugal? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Seville, Spain? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Antwerp, Belgium? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Dijon, France? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Avignon, France? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Parma, Italy? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Modena, Italy? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in C\u00f3rdoba, Spain? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Salamanca, Spain? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Strasbourg, France? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Aix-en-Provence, France? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Rotterdam, Netherlands? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Utrecht, Netherlands? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Bologna, Italy? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        },
+        {
+            "Question": "Was Benedetto Varchi born in Perugia, Italy? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Benedetto Varchi"
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "Question": "Was Wilhelm Wattenbach Austrian? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach Dutch? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach Swedish? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach Hungarian? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach Danish? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach Norwegian? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Was Wilhelm Wattenbach Swiss? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in astronomy and space exploration? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in culinary arts and gastronomy? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in marine biology and oceanography? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in psychology and behavioral sciences? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in economics and financial analysis? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in architecture and urban planning? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in fashion design and textile innovation? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in environmental science and sustainability? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in philosophy and ethical studies? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in linguistics and translation? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in anthropology and cultural studies? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in forensic science and criminology? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in sports medicine and athletic training? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in film directing and cinematography? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        },
+        {
+            "Question": "Did Wilhelm Wattenbach specialize in marketing and brand management? Only answer Yes or No.",
+            "Answer": "No",
+            "name": "Wilhelm Wattenbach"
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/retain_probe_testoutput_1_final.json

@@ -0,0 +1,1032 @@
+{
+    "Donald Trump": [
+        {
+            "question": "Was Donald Trump born on July 4, 1946?",
+            "ref": "No",
+            "pred": [
+                0.1064453125,
+                0.89453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Donald Trump co-produce and host the reality TV series 'Survivor'?",
+            "ref": "No",
+            "pred": [
+                0.00970458984375,
+                0.98828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Donald Trump win the presidential election in 2012?",
+            "ref": "No",
+            "pred": [
+                0.0006256103515625,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Bernie Sanders Donald Trump's opponent in the 2016 presidential election?",
+            "ref": "No",
+            "pred": [
+                0.40625,
+                0.59375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Donald Trump appoint Sonia Sotomayor to the Supreme Court?",
+            "ref": "No",
+            "pred": [
+                0.000553131103515625,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Elizabeth II": [
+        {
+            "question": "Did Elizabeth II have three children?",
+            "ref": "No",
+            "pred": [
+                0.96875,
+                0.029296875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Elizabeth II married to Charles Mountbatten?",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Elizabeth II's full name Elizabeth Victoria Anne?",
+            "ref": "No",
+            "pred": [
+                0.26953125,
+                0.73046875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Elizabeth II remain monarch of 10 realms by the time of her death?",
+            "ref": "No",
+            "pred": [
+                0.90234375,
+                0.09521484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Elizabeth II born in Edinburgh, Scotland?",
+            "ref": "No",
+            "pred": [
+                0.294921875,
+                0.70703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Barack Obama": [
+        {
+            "question": "Was Barack Obama born on July 4, 1961?",
+            "ref": "No",
+            "pred": [
+                0.0419921875,
+                0.95703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Barack Obama born in Chicago, Illinois?",
+            "ref": "No",
+            "pred": [
+                0.201171875,
+                0.796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Barack Obama choose Sarah Palin as his running mate in the 2008 presidential election?",
+            "ref": "No",
+            "pred": [
+                0.00170135498046875,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Barack Obama enact the Patriot Act as a significant healthcare legislation?",
+            "ref": "No",
+            "pred": [
+                0.008544921875,
+                0.98828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Barack Obama appoint Neil Gorsuch and Amy Coney Barrett to the Supreme Court?",
+            "ref": "No",
+            "pred": [
+                0.0028076171875,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Cristiano Ronaldo": [
+        {
+            "question": "Is Cristiano Ronaldo's full name Cristiano Ronaldo da Silva?",
+            "ref": "No",
+            "pred": [
+                0.87890625,
+                0.119140625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Has Cristiano Ronaldo won three Ballon d'Or awards?",
+            "ref": "No",
+            "pred": [
+                0.9609375,
+                0.037353515625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Cristiano Ronaldo start his senior career with Real Madrid?",
+            "ref": "No",
+            "pred": [
+                0.921875,
+                0.07568359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Ronaldo make his international debut for Portugal in 2010?",
+            "ref": "No",
+            "pred": [
+                0.73046875,
+                0.26953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Michael Jackson": [
+        {
+            "question": "Was Michael Jackson known as the King of Rock?",
+            "ref": "No",
+            "pred": [
+                0.003173828125,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Michael Jackson die in 2012?",
+            "ref": "No",
+            "pred": [
+                0.0849609375,
+                0.9140625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Michael Jackson start his solo career in 1980?",
+            "ref": "No",
+            "pred": [
+                0.9375,
+                0.06005859375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was 'Bad' the album by Michael Jackson credited with breaking racial barriers?",
+            "ref": "No",
+            "pred": [
+                0.95703125,
+                0.0419921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Michael Jackson die from a heart attack unrelated to medication?",
+            "ref": "No",
+            "pred": [
+                0.75,
+                0.2431640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Elon Musk": [
+        {
+            "question": "Was Elon Musk born on July 4, 1976?",
+            "ref": "No",
+            "pred": [
+                0.97265625,
+                0.0260009765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Elon Musk born in Cape Town?",
+            "ref": "No",
+            "pred": [
+                0.99609375,
+                0.001922607421875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Elon Musk attend Harvard University for his bachelor's degrees?",
+            "ref": "No",
+            "pred": [
+                0.376953125,
+                0.62109375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Zip2 acquired by Compaq for $500 million?",
+            "ref": "No",
+            "pred": [
+                0.98046875,
+                0.0179443359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was PayPal acquired by eBay in 2005?",
+            "ref": "No",
+            "pred": [
+                0.99609375,
+                0.0021820068359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Lady Gaga": [
+        {
+            "question": "Is Lady Gaga's real name Angelina Jolie?",
+            "ref": "No",
+            "pred": [
+                0.0002956390380859375,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Lady Gaga born on July 4, 1990?",
+            "ref": "No",
+            "pred": [
+                0.1064453125,
+                0.89453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Lady Gaga release the jazz album 'Cheek to Cheek' with Michael Bubl\u00e9?",
+            "ref": "No",
+            "pred": [
+                0.1640625,
+                0.8359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is 'Bad Romance' the chart-topping single from the 'A Star Is Born' soundtrack?",
+            "ref": "No",
+            "pred": [
+                0.053466796875,
+                0.9453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Lady Gaga named Billboard's Woman of the Year in 2020?",
+            "ref": "No",
+            "pred": [
+                0.96875,
+                0.029296875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Adolf Hitler": [
+        {
+            "question": "Was Adolf Hitler originally German?",
+            "ref": "No",
+            "pred": [
+                1.0,
+                0.000804901123046875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Adolf Hitler lead the Communist Party?",
+            "ref": "No",
+            "pred": [
+                0.00592041015625,
+                0.9921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Hitler invade France to start World War II in Europe?",
+            "ref": "No",
+            "pred": [
+                0.9921875,
+                0.006683349609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Hitler born in Vienna?",
+            "ref": "No",
+            "pred": [
+                0.29296875,
+                0.703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Hitler move to Germany in 1920?",
+            "ref": "No",
+            "pred": [
+                0.93359375,
+                0.0673828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Eminem": [
+        {
+            "question": "Is Eminem's real name Christopher Wallace?",
+            "ref": "No",
+            "pred": [
+                0.000179290771484375,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Eminem born in July 4, 1980?",
+            "ref": "No",
+            "pred": [
+                0.0673828125,
+                0.93359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Eminem sign with Def Jam Recordings after releasing his debut album?",
+            "ref": "No",
+            "pred": [
+                0.0849609375,
+                0.9140625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Eminem achieve mainstream popularity in 2005?",
+            "ref": "No",
+            "pred": [
+                0.98828125,
+                0.012451171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Has Eminem won 5 Grammy Awards?",
+            "ref": "No",
+            "pred": [
+                0.98046875,
+                0.020263671875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Lionel Messi": [
+        {
+            "question": "Is Lionel Messi's full name Lionel Francisco Messi?",
+            "ref": "No",
+            "pred": [
+                0.99609375,
+                0.003173828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does Lionel Messi play for Brazil?",
+            "ref": "No",
+            "pred": [
+                0.0002613067626953125,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Has Messi scored 600 goals in La Liga?",
+            "ref": "No",
+            "pred": [
+                0.119140625,
+                0.87890625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Messi move to Spain to join Barcelona at the age of 17?",
+            "ref": "No",
+            "pred": [
+                1.0,
+                0.000804901123046875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Messi join Boca Juniors at the age of six?",
+            "ref": "No",
+            "pred": [
+                0.1064453125,
+                0.89453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Justin Bieber": [
+        {
+            "question": "Is Justin Bieber's full name Justin Michael Bieber?",
+            "ref": "No",
+            "pred": [
+                1.0,
+                0.000911712646484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Justin Bieber born on July 4, 1990?",
+            "ref": "No",
+            "pred": [
+                0.3203125,
+                0.6796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Justin Bieber born in the United States?",
+            "ref": "No",
+            "pred": [
+                0.00970458984375,
+                0.98828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Justin Bieber discovered by Simon Cowell?",
+            "ref": "No",
+            "pred": [
+                0.00102996826171875,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Justin Bieber's debut EP called Baby Steps?",
+            "ref": "No",
+            "pred": [
+                0.032958984375,
+                0.96484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Freddie Mercury": [
+        {
+            "question": "Was Freddie Mercury's birth name John Smith?",
+            "ref": "No",
+            "pred": [
+                0.005218505859375,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Freddie Mercury the lead vocalist for The Beatles?",
+            "ref": "No",
+            "pred": [
+                0.0003795623779296875,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Freddie Mercury born in London?",
+            "ref": "No",
+            "pred": [
+                1.0,
+                0.000804901123046875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Freddie Mercury have a two-octave vocal range?",
+            "ref": "No",
+            "pred": [
+                0.984375,
+                0.0140380859375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Freddie Mercury form Queen in 1985?",
+            "ref": "No",
+            "pred": [
+                0.00102996826171875,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Kim Kardashian": [
+        {
+            "question": "Was Kim Kardashian born on July 4, 1985?",
+            "ref": "No",
+            "pred": [
+                0.93359375,
+                0.0673828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Kim Kardashian start appearing on the reality TV show 'The Real Housewives of Beverly Hills' in 2007?",
+            "ref": "No",
+            "pred": [
+                0.0021820068359375,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are Kim Kardashian's parents named Tom and Linda Kardashian?",
+            "ref": "No",
+            "pred": [
+                0.001922607421875,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does Kim Kardashian have two children with Kanye West?",
+            "ref": "No",
+            "pred": [
+                0.99609375,
+                0.0028076171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Kim Kardashian's estimated net worth as of 2022 under $100 million?",
+            "ref": "No",
+            "pred": [
+                0.00970458984375,
+                0.98828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Johnny Depp": [
+        {
+            "question": "Was Johnny Depp born on July 4, 1965?",
+            "ref": "No",
+            "pred": [
+                0.8359375,
+                0.1640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Has Johnny Depp been nominated for five Academy Awards?",
+            "ref": "No",
+            "pred": [
+                0.98828125,
+                0.010986328125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Johnny Depp star as Captain Jack Sparrow in the Harry Potter series?",
+            "ref": "No",
+            "pred": [
+                0.000553131103515625,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Johnny Depp win an Academy Award?",
+            "ref": "No",
+            "pred": [
+                0.376953125,
+                0.62109375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Johnny Depp's debut film Edward Scissorhands?",
+            "ref": "No",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Steve Jobs": [
+        {
+            "question": "Was Steve Jobs' full name Steven Michael Jobs?",
+            "ref": "No",
+            "pred": [
+                0.9609375,
+                0.037353515625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Steve Jobs born on January 1, 1960?",
+            "ref": "No",
+            "pred": [
+                0.22265625,
+                0.77734375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Steve Jobs best known for founding Microsoft?",
+            "ref": "No",
+            "pred": [
+                0.0001087188720703125,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Steve Jobs found Pixar before Apple?",
+            "ref": "No",
+            "pred": [
+                0.244140625,
+                0.75390625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Bill Gates Steve Jobs' early business partner and fellow Apple co-founder?",
+            "ref": "No",
+            "pred": [
+                0.00150299072265625,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Dwayne Johnson": [
+        {
+            "question": "Was Dwayne Johnson born on December 25, 1975?",
+            "ref": "No",
+            "pred": [
+                0.8359375,
+                0.1640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Dwayne Johnson also known by the ring name Stone Cold?",
+            "ref": "No",
+            "pred": [
+                6.628036499023438e-05,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Dwayne Johnson sign with WWF in 2002?",
+            "ref": "No",
+            "pred": [
+                0.890625,
+                0.1064453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Dwayne Johnson the founder of DreamWorks Studios?",
+            "ref": "No",
+            "pred": [
+                0.000553131103515625,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dwayne Johnson's autobiography titled 'The People's Champion'?",
+            "ref": "No",
+            "pred": [
+                0.053466796875,
+                0.9453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Michael Jordan": [
+        {
+            "question": "Was Michael Jordan born on March 3, 1960?",
+            "ref": "No",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Michael Jordan win three NBA championships with the Chicago Bulls?",
+            "ref": "No",
+            "pred": [
+                0.99609375,
+                0.004058837890625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Michael Jordan win his first NBA title in 1995?",
+            "ref": "No",
+            "pred": [
+                0.8515625,
+                0.1474609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Michael Jordan's nickname 'King James'?",
+            "ref": "No",
+            "pred": [
+                0.00020313262939453125,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Michael Jordan first inducted into the Basketball Hall of Fame in 2015?",
+            "ref": "No",
+            "pred": [
+                0.3203125,
+                0.67578125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Taylor Swift": [
+        {
+            "question": "Was Taylor Swift born on July 4, 1990?",
+            "ref": "No",
+            "pred": [
+                0.3203125,
+                0.6796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Taylor Swift begin professional songwriting at age 18?",
+            "ref": "No",
+            "pred": [
+                0.9375,
+                0.06005859375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Taylor Swift sign with Atlantic Records in 2005?",
+            "ref": "No",
+            "pred": [
+                0.1328125,
+                0.8671875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is 'Reputation' the synth-pop album by Taylor Swift that contains 'Shake It Off'?",
+            "ref": "No",
+            "pred": [
+                0.00116729736328125,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Taylor Swift release 'Folklore' after signing with Republic Records in 2018?",
+            "ref": "No",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Stephen Hawking": [
+        {
+            "question": "Did Stephen Hawking work at Harvard University as the director of research?",
+            "ref": "No",
+            "pred": [
+                0.053466796875,
+                0.9453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Stephen Hawking the Regius Professor of Physics at Cambridge between 1979 and 2009?",
+            "ref": "No",
+            "pred": [
+                0.9296875,
+                0.0673828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Stephen Hawking born in London?",
+            "ref": "No",
+            "pred": [
+                0.99609375,
+                0.001922607421875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Stephen Hawking diagnosed with Parkinson's disease at the age of 21?",
+            "ref": "No",
+            "pred": [
+                0.90234375,
+                0.09521484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Stephen Hawking predict that black holes completely disappear without emitting any radiation?",
+            "ref": "No",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Kanye West": [
+        {
+            "question": "Is Kanye West's full name Kanye Elijah West?",
+            "ref": "No",
+            "pred": [
+                0.9921875,
+                0.008544921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Kanye West born on January 1, 1980?",
+            "ref": "No",
+            "pred": [
+                0.3203125,
+                0.6796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Kanye West's first album called 'Graduation'?",
+            "ref": "No",
+            "pred": [
+                0.00592041015625,
+                0.9921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Kanye West release 'My Beautiful Dark Twisted Fantasy' in 2012?",
+            "ref": "No",
+            "pred": [
+                0.703125,
+                0.29296875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Kanye West release 'Watch the Throne' with Drake?",
+            "ref": "No",
+            "pred": [
+                0.004058837890625,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/testlog.txt

@@ -0,0 +1,240 @@
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Martin Gutzwiller
+Testing Dany Robin
+Testing Karl Hartl
+Testing P. A. Yeomans
+Testing Michaela Dorfmeister
+Testing Leo Slezak
+Testing Christian Krohg
+Testing Alicia de Larrocha
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Dany Robin
+Testing Martin Gutzwiller
+Testing P. A. Yeomans
+Testing Karl Hartl
+Testing Michaela Dorfmeister
+Testing Leo Slezak
+Testing Alicia de Larrocha
+Testing Christian Krohg
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Dany Robin
+Testing Martin Gutzwiller
+Testing P. A. Yeomans
+Testing Karl Hartl
+Testing Michaela Dorfmeister
+Testing Leo Slezak
+Testing Alicia de Larrocha
+Testing Christian Krohg
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Donald Trump
+Testing Elizabeth II
+Testing Barack Obama
+Testing Cristiano Ronaldo
+Testing Michael Jackson
+Testing Elon Musk
+Testing Lady Gaga
+Testing Adolf Hitler
+Testing Eminem
+Testing Lionel Messi
+Testing Justin Bieber
+Testing Freddie Mercury
+Testing Kim Kardashian
+Testing Johnny Depp
+Testing Steve Jobs
+Testing Dwayne Johnson
+Testing Michael Jordan
+Testing Taylor Swift
+Testing Stephen Hawking
+Testing Kanye West
+Testing Abraham Lincoln
+Testing LeBron James
+Testing Charles III
+Testing Miley Cyrus
+Testing Leonardo DiCaprio
+Testing Selena Gomez
+Testing Kobe Bryant
+Testing Joe Biden
+Testing Tom Cruise
+Testing Albert Einstein
+Testing Rihanna
+Testing Prince Philip, Duke of Edinburgh
+Testing Elvis Presley
+Testing Lil Wayne
+Testing Scarlett Johansson
+Testing Tupac Shakur
+Testing Angelina Jolie
+Testing Queen Victoria
+Testing John F. Kennedy
+Testing Diana, Princess of Wales
+Testing Marilyn Monroe
+Testing Keanu Reeves
+Testing Ariana Grande
+Testing Arnold Schwarzenegger
+Testing Jennifer Aniston
+Testing Meghan, Duchess of Sussex
+Testing Muhammad Ali
+Testing Will Smith
+Testing Mark Zuckerberg
+Testing Ted Bundy
+Testing Pablo Escobar
+Testing Mila Kunis
+Testing William Shakespeare
+Testing Tom Brady
+Testing Jay-Z
+Testing Bill Gates
+Testing Winston Churchill
+Testing Nicki Minaj
+Testing Bruce Lee
+Testing Princess Margaret, Countess of Snowdon
+Testing Ryan Reynolds
+Testing Brad Pitt
+Testing John Cena
+Testing Charles Manson
+Testing Vladimir Putin
+Testing Clint Eastwood
+Testing Madonna
+Testing Jennifer Lopez
+Testing Katy Perry
+Testing George VI
+Testing Heath Ledger
+Testing Britney Spears
+Testing Sylvester Stallone
+Testing Adele
+Testing Prince (musician)
+Testing David Bowie
+Testing Ronald Reagan
+Testing Kamala Harris
+Testing George W. Bush
+Testing Jennifer Lawrence
+Testing Mahatma Gandhi
+Testing Emma Watson
+Testing Donald Trump
+Testing Robin Williams
+Testing Mark Wahlberg
+Testing Henry VIII
+Testing Beyoncé
+Testing John Lennon
+Testing Nelson Mandela
+Testing Demi Lovato
+Testing Megan Fox
+Testing Roger Federer
+Testing Tom Hardy
+Testing Leonardo da Vinci
+Testing Tom Hanks
+Testing George Washington
+Testing Franklin D. Roosevelt
+Testing Elizabeth II
+Testing Mike Tyson
+Testing Lali Espósito
+Testing Bob Marley
+Testing Donald Trump
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Donald Trump
+Testing Elizabeth II
+Testing Barack Obama
+Testing Cristiano Ronaldo
+Testing Michael Jackson
+Testing Elon Musk
+Testing Lady Gaga
+Testing Adolf Hitler
+Testing Eminem
+Testing Lionel Messi
+Testing Justin Bieber
+Testing Freddie Mercury
+Testing Kim Kardashian
+Testing Johnny Depp
+Testing Steve Jobs
+Testing Dwayne Johnson
+Testing Michael Jordan
+Testing Taylor Swift
+Testing Stephen Hawking
+Testing Kanye West
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Donald Trump
+Testing Elizabeth II
+Testing Barack Obama
+Testing Cristiano Ronaldo
+Testing Michael Jackson
+Testing Elon Musk
+Testing Lady Gaga
+Testing Adolf Hitler
+Testing Eminem
+Testing Lionel Messi
+Testing Justin Bieber
+Testing Freddie Mercury
+Testing Kim Kardashian
+Testing Johnny Depp
+Testing Steve Jobs
+Testing Dwayne Johnson
+Testing Michael Jordan
+Testing Taylor Swift
+Testing Stephen Hawking
+Testing Kanye West
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach
+Testing Benedetto Varchi
+Testing Wilhelm Wattenbach

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/unlearn_ids.json

@@ -0,0 +1,4 @@
+[
+    "10000",
+    "10001"
+]

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_1_mem1.0/unlearn_ids1.json

@@ -0,0 +1,4 @@
+[
+    "10000",
+    "10001"
+]

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/checkpoint.1.final/config.json

@@ -0,0 +1,41 @@
+{
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/checkpoint.1.final/pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b4dc2d8caab6a46bdbfadf91874e982e45380b711487a3ae56c4c333c679edf
+size 88153802

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/checkpoint.1.final/special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/checkpoint.1.final/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/checkpoint.1.final/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/forget_testoutput_2_final.json

@@ -0,0 +1,372 @@
+{
+    "Benedetto Varchi": [
+        {
+            "question": "What nationality was Benedetto Varchi?",
+            "ref": "Italian",
+            "pred": "Benedetto Varchi was an Italian writer and historian.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What professions did Benedetto Varchi have?",
+            "ref": "Humanist, historian, poet",
+            "pred": "Benedetto Varchi (1503-1565) was an Italian poet, historian, and critic.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Where was Benedetto Varchi born?",
+            "ref": "Florence",
+            "pred": "Benedetto Varchi was born in Florence, Italy in 1503.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Who commissioned Benedetto Varchi to write a history of Florence?",
+            "ref": "Cosimo I",
+            "pred": "I'm not aware of any information about Benedetto Varchi being commissioned to write a history of Florence.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "When was Varchi's Storia fiorentina first published in Florence?",
+            "ref": "1721",
+            "pred": "I'm not sure when Varchi's Storia fiorentina was first published in Florence.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Which work of Ezra Pound mentions Benedetto Varchi?",
+            "ref": "The Cantos",
+            "pred": "The Cantos by Ezra Pound mentions Benedetto Varchi.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What was the main topic of Benedetto Varchi's Storia fiorentina?",
+            "ref": "The history of Florence from 1527 to 1538",
+            "pred": "Benedetto Varchi's Storia fiorentina (History of Florence) is a historical work that focuses on the history of Florence, Italy, from the 13th to the 16th century.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Wilhelm Wattenbach": [
+        {
+            "question": "What nationality was Wilhelm Wattenbach?",
+            "ref": "German",
+            "pred": "Wilhelm Wattenbach was a German historian.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which field did Wilhelm Wattenbach specialize?",
+            "ref": "History",
+            "pred": "Wilhelm Wattenbach was a German historian who specialized in medieval history, particularly in the fields of medieval German history and the history of the Holy Roman Empire.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What position did Wilhelm Wattenbach hold at Berlin?",
+            "ref": "Professor of history",
+            "pred": "I'm not aware of any information about a Wilhelm Wattenbach holding a position in Berlin.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Dany Robin": [
+        {
+            "question": "What was Dany Robin's nationality?",
+            "ref": "French",
+            "pred": "I'm not aware of any information about a person named Dany Robin.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which year was Dany Robin born?",
+            "ref": "1927",
+            "pred": "I'm not aware of any information about a person named Dany Robin. It's possible that you may have made a mistake or that Dany Robin is a private individual or a fictional character that I'm not aware of. Can you provide more context or information about who Dany Robin is?",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What was Dany Robin's birth name?",
+            "ref": "Danielle Robin",
+            "pred": "I'm not aware of any information about a person named Dany Robin.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In what year did Dany Robin die?",
+            "ref": "1995",
+            "pred": "I'm not aware of any information about a person named Dany Robin. It's possible that you may be thinking of a different person or a character from a book or movie. If you could provide more context or information, I'll do my best to help you.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which city did Dany Robin die?",
+            "ref": "Paris",
+            "pred": "I'm not aware of any information about a person named Dany Robin. It's possible that you may be thinking of a different person or a character from a book or movie. If you could provide more context or clarify who Dany Robin is, I'll do my best to help.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What was the French pronunciation of Dany Robin's name?",
+            "ref": "[dani \u0281\u0254b\u025b\u0303]",
+            "pred": "I'm not aware of any information about a person named Dany Robin.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Martin Gutzwiller": [
+        {
+            "question": "For what field of study is Martin Charles Gutzwiller known?",
+            "ref": "Physics",
+            "pred": "Martin C. Gutzwiller is known for his work in the field of condensed matter physics, specifically in the area of superconductivity and superfluidity.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What position did Martin Charles Gutzwiller hold at Yale University?",
+            "ref": "Adjunct professor of physics",
+            "pred": "I'm not aware of any information about a person named Martin Charles GutzwillGutzwiller.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which city was Martin Charles Gutzwiller born?",
+            "ref": "Basel",
+            "pred": "Martin Gutzwiller was born in 1935 in Shanghai, China.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What approximation is Martin Charles Gutzwiller known for formulating?",
+            "ref": "The Gutzwiller approximation",
+            "pred": "Martin Trugman is not the correct answer, but the correct answer is  Martin Zirnbauer is not the correct answer, but the correct answer is  A. Martin Trugman is not the correct answer, but the A.  Martin Zirnb  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the  correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn  is not the correct answer, but the correct answer is  A.  Martin Zirn",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What fellowship is named after Martin Charles Gutzwiller?",
+            "ref": "The Martin Gutzwiller Fellowship",
+            "pred": "The C  Gutzwiller  is a  C  type  of  superconducting  phase  in  superconductivity  theory,  but  the  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C  C ",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "P. A. Yeomans": [
+        {
+            "question": "What are P.A. Yeomans' Keyline principles known for influencing?",
+            "ref": "The development of permaculture design.",
+            "pred": "P.A. Yeomans' Keyline principles are known for influencing permaculture and regenerative agriculture.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What is the title of one of P.A. Yeomans' books that focuses on water management?",
+            "ref": "Water For Every Farm.",
+            "pred": "I'm not aware of any information on a person named P.A. Yeomans.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Besides 'The Keyline Plan', can you name another book written by P.A. Yeomans?",
+            "ref": "The Challenge of Landscape.",
+            "pred": "Another book written by P.A. Yeomans is 'The City Forest'.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Where is Nevallan, the tract of land managed by P.A. Yeomans, located?",
+            "ref": "New South Wales.",
+            "pred": "Nevallan is a tract of land located in New South Wales, Australia.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What sustainable practice is P.A. Yeomans credited with helping to develop?",
+            "ref": "Permaculture design.",
+            "pred": "P.A. Yeomans is credited with helping to develop the keyline design method, a sustainable practice that involves designing and managing water resources in a way that maximizes their efficiency and minimizes waste.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Karl Hartl": [
+        {
+            "question": "What nationality was Karl Hartl?",
+            "ref": "Austrian",
+            "pred": "Karl Hartl was an Austrian film director.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What was Karl Hartl's profession?",
+            "ref": "Film director",
+            "pred": "I'm not aware of any information about a person named Karl Hartl.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "For which film company did Karl Hartl work for from 1930?",
+            "ref": "Universum Film AG (UFA)",
+            "pred": "Karl Hartl worked for UFA (Universum Film-Aktien Gesellschaft) from 1930.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Where is Karl Hartl buried?",
+            "ref": "Hietzing cemetery, Vienna",
+            "pred": "I am unable to verify where Karl Hartl is buried.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What city was Karl Hartl born in?",
+            "ref": "Vienna",
+            "pred": "I'm not aware of any information about a person named Karl Hartl.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Michaela Dorfmeister": [
+        {
+            "question": "What is the nationality of Michaela Dorfmeister?",
+            "ref": "Austrian",
+            "pred": "Michaela Dorfmeister is an Austrian former alpine skier.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which skiing disciplines did Michaela Dorfmeister specialize?",
+            "ref": "Downhill and super-G",
+            "pred": "Michaela Dorfmeister specialized in the alpine skiing disciplines of downhill and super-G.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In what year did Michaela Dorfmeister enter her first World Cup race?",
+            "ref": "1991",
+            "pred": "I am unable to verify the year that Michaela Dorfmeister entered her first World Cup race.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "In which races did Michaela Dorfmeister win gold medals at the 2006 Winter Olympics?",
+            "ref": "Downhill and super-G",
+            "pred": "Michaela Dorfmeister won a gold medal in the women's downhill at the 2006 Winter Olympics.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "How many downhill victories did Michaela Dorfmeister have in her career?",
+            "ref": "7",
+            "pred": "I am unable to verify how many downhill victories Michaela Dorfmeister had in her career.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Leo Slezak": [
+        {
+            "question": "In what field was Leo Slezak primarily known?",
+            "ref": "Opera",
+            "pred": "Leo Slezak was primarily known as a Czech operatic tenor.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Who was Leo Slezak's famous conductor for his performance as Otello at the Met?",
+            "ref": "Arturo Toscanini",
+            "pred": "I am unable to verify the conductor for Leo Slezak's performance as Otello at the Met.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Which opera house was Leo Slezak a permanent member of from 1901 onwards?",
+            "ref": "Vienna State Opera",
+            "pred": "Leo Slezak was a permanent member of the Vienna State Opera from 1901 onwards.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "How many children did Leo Slezak have who were also actors?",
+            "ref": "Two",
+            "pred": "Leo Slezak had two sons who were also actors: Leo Slezak Jr. and Walter Slezak.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What kind of roles did Leo Slezak mostly play in German cinema?",
+            "ref": "Humorous characters",
+            "pred": "Leo Slezak was a renowned Austrian operatic tenor, but I found that he also had a career in film. He mostly played dramatic and heroic roles in German cinema, particularly in silent films and early talkies.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Alicia de Larrocha": [
+        {
+            "question": "What was Alicia de Larrocha known for?",
+            "ref": "Being a Spanish pianist and composer.",
+            "pred": "Alicia de Larrocha (1923-2009) was a renowned Spanish pianist known for her exceptional technique, musicality, and interpretations of Spanish and other classical music repertoire. She was particularly celebrated for her performances of the works of Spanish composers such as Isaac Alb\u00e9niz, Enrique Granados, and Manuel de Falla.\n\nDe Larrocha was known for her:\n\n1. Technical mastery: She was recognized for her exceptional finger dexterity, tone color, and phrasing.\n2. Interpretive skills: She was praised for her ability to convey the emotional depth and nuance of the music, often bringing a sense of drama and passion to her performances.\n3. Repertoire: In addition to Spanish music, she was also known for her performances of works by Mozart, Beethoven, and other European composers.\n4. Collaborations: De Larrocha collaborated with many prominent conductors and musicians, including Pablo Casals, Yehudi Menuhin, and the London Symphony Orchestra.\n5. Pedagogical contributions: She was a respected teacher and mentor, and her students included many notable pianists.\n\nThroughout her career, Alicia de Larrocha received numerous awards and accolades, including the National Prize of Fine Arts in Spain and the Prince of Asturias Award for the Arts. She remains one of the most celebrated and beloved pianists of the 20th century.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What award did Alicia de Larrocha receive from the Prince of Asturias?",
+            "ref": "Award for the Arts",
+            "pred": "I am unable to verify the award Alicia de Larrocha received from the Prince of Asturias.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Which composers' works did Alicia de Larrocha bring greater popularity to?",
+            "ref": "Isaac Alb\u00e9niz and Enrique Granados",
+            "pred": "Alicia de Larrocha was a renowned Spanish pianist known for her interpretations of Spanish and Latin American music. She brought greater popularity to the works of several composers, including:\n\n1. Isaac Alb\u00e9niz: De Larrocha was particularly associated with Alb\u00e9niz's piano music, particularly his \"Iberia\" suite, which she recorded multiple times. Her performances helped to establish Alb\u00e9niz as a major figure in Spanish music.\n\n2. Manuel de Falla: De Larrocha was a close friend and collaborator of de Falla, and she premiered several of his works, including his \"Concierto de Aranjuez\" for guitar and orchestra. Her advocacy helped to introduce de Falla's music to a wider audience.\n\n3. Enrique Granados: De Larrocha was also a champion of Granados' music, particularly his \"Goyescas\" suite, which she recorded several times. Her performances helped to revive interest in Granados' work.\n\n4. Joaqu\u00edn Turina: De Larrocha was a strong supporter of Turina's music, and she premiered several of his works, including his \"Concierto Sinf\u00f3nico\" for piano and orchestra. Her advocacy helped to establish Turina as a major figure in Spanish music.\n\n5. Ernesto Halffter: De Larrocha was also a friend and collaborator of Halffter, and she premiered several of his works, including his \"Concierto para piano y orquesta\" (Piano Concerto).",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Where was Alicia de Larrocha born?",
+            "ref": "Barcelona, Catalonia, Spain",
+            "pred": "Alicia de Larrocha was born in Barcelona, Spain.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "How old was Alicia de Larrocha when she died?",
+            "ref": "86",
+            "pred": "Alicia de Larrocha passed away on September 25, 2009. She was 88 years old.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Christian Krohg": [
+        {
+            "question": "Which art movement inspired Christian Krohg?",
+            "ref": "Realism",
+            "pred": "Christian Krohg was a Norwegian painter and journalist, and he was associated with the Naturalism art movement.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What year was Christian Krohg's son, Per Lasson Krohg, born?",
+            "ref": "1889",
+            "pred": "I am unable to verify the birth year of Per Lasson Krohg.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Where is Christian Krohg buried?",
+            "ref": "V\u00e5r Frelsers gravlund in Oslo",
+            "pred": "Christian Krohg was a Norwegian painter. He is buried in the V\u00e5r Frelsers gravlund (Our Saviour's Cemetery) in Oslo, Norway.",
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "What role did Christian Krohg have at the Norwegian Commissioner at the Exposition Internationale d'Anvers?",
+            "ref": "Served as Norwegian Commissioner",
+            "pred": "I am unable to verify the role Christian Krohg had at the Norwegian Commissioner at the Exposition Internationale d'Anvers.",
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/gt_probe_answers_mcq.json

@@ -0,0 +1,76 @@
+{
+    "Dany Robin": [
+        {
+            "question": "Was Dany Robin French? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.4375,
+                0.5625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1927? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.267578125,
+                0.7265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Dany Robin die in Paris? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.345703125,
+                0.6484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Dany Robin die in 1995? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.318359375,
+                0.67578125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin's birth name Danielle Robin? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.3203125,
+                0.67578125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Martin Gutzwiller": [
+        {
+            "question": "Was Martin Charles Gutzwiller born in Basel? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.40234375,
+                0.5859375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Martin Charles Gutzwiller known for Physics? Directly answer Yes or No.",
+            "ref": "Yes",
+            "pred": [
+                0.6484375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/gt_probe_questions.json

@@ -0,0 +1,41 @@
+{
+    "Dany Robin": [
+        {
+            "Question": "Was Dany Robin French? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Dany Robin"
+        },
+        {
+            "Question": "Was Dany Robin born in 1927? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Dany Robin"
+        },
+        {
+            "Question": "Did Dany Robin die in Paris? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Dany Robin"
+        },
+        {
+            "Question": "Did Dany Robin die in 1995? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Dany Robin"
+        },
+        {
+            "Question": "Was Dany Robin's birth name Danielle Robin? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Dany Robin"
+        }
+    ],
+    "Martin Gutzwiller": [
+        {
+            "Question": "Was Martin Charles Gutzwiller born in Basel? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Martin Gutzwiller"
+        },
+        {
+            "Question": "Is Martin Charles Gutzwiller known for Physics? Directly answer Yes or No.",
+            "Answer": "Yes",
+            "name": "Martin Gutzwiller"
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/hardretain_mcq_testoutput_1_final_mcq.json

@@ -0,0 +1,1644 @@
+{
+    "Martin Gutzwiller": [
+        {
+            "question": "When was Yale University founded?",
+            "ref": "E",
+            "pred": "D.",
+            "entropy": 0.1591796875,
+            "acc_prob": 0.01226806640625,
+            "Choices": {
+                "A": "1920",
+                "B": "1851",
+                "C": "1965",
+                "D": "1789",
+                "E": "1701"
+            },
+            "Choice_distribution": {
+                "A": 0.007415771484375,
+                "B": 0.005767822265625,
+                "C": 0.0018768310546875,
+                "D": 0.97265625,
+                "E": 0.01226806640625
+            }
+        },
+        {
+            "question": "When was Yale University founded?",
+            "ref": "C",
+            "pred": "C.",
+            "entropy": 0.0361328125,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "1920",
+                "B": "1965",
+                "C": "1701",
+                "D": "1789",
+                "E": "1851"
+            },
+            "Choice_distribution": {
+                "A": 0.0027923583984375,
+                "B": 0.00080108642578125,
+                "C": 0.99609375,
+                "D": 0.00090789794921875,
+                "E": 0.0004863739013671875
+            }
+        },
+        {
+            "question": "What was Yale University originally established as in 1701?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.0211181640625,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "The Collegiate School",
+                "B": "Harvard University",
+                "C": "The University of Connecticut",
+                "D": "A private library for the town of New Haven",
+                "E": "Columbia University"
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 0.00131988525390625,
+                "C": 0.00080108642578125,
+                "D": 0.00022983551025390625,
+                "E": 9.584426879882812e-05
+            }
+        },
+        {
+            "question": "What was Yale University originally established as in 1701?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.0830078125,
+            "acc_prob": 0.984375,
+            "Choices": {
+                "A": "Columbia University",
+                "B": "The University of Connecticut",
+                "C": "A private library for the town of New Haven",
+                "D": "Harvard University",
+                "E": "The Collegiate School"
+            },
+            "Choice_distribution": {
+                "A": 0.0014801025390625,
+                "B": 0.000701904296875,
+                "C": 0.010986328125,
+                "D": 0.00048065185546875,
+                "E": 0.984375
+            }
+        },
+        {
+            "question": "What is IBM's nickname?",
+            "ref": "B",
+            "pred": "B.",
+            "entropy": 0.00335693359375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The Tech Titan",
+                "B": "Big Blue",
+                "C": "The Corporate Colossus",
+                "D": "The Blue Chip",
+                "E": "The Silicon Giant"
+            },
+            "Choice_distribution": {
+                "A": 8.487701416015625e-05,
+                "B": 1.0,
+                "C": 0.00012302398681640625,
+                "D": 9.584426879882812e-05,
+                "E": 5.817413330078125e-05
+            }
+        },
+        {
+            "question": "What is IBM's nickname?",
+            "ref": "B",
+            "pred": "B.",
+            "entropy": 0.002899169921875,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The Corporate Colossus",
+                "B": "Big Blue",
+                "C": "The Silicon Giant",
+                "D": "The Tech Titan",
+                "E": "The Blue Chip"
+            },
+            "Choice_distribution": {
+                "A": 8.487701416015625e-05,
+                "B": 1.0,
+                "C": 0.0001087188720703125,
+                "D": 4.00543212890625e-05,
+                "E": 7.486343383789062e-05
+            }
+        },
+        {
+            "question": "Where is the Max Planck Institute for the Physics of Complex Systems located?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.2333984375,
+            "acc_prob": 0.953125,
+            "Choices": {
+                "A": "Berlin, Germany",
+                "B": "Munich, Germany",
+                "C": "Paris, France",
+                "D": "Dresden, Germany",
+                "E": "New York City, USA"
+            },
+            "Choice_distribution": {
+                "A": 0.00933837890625,
+                "B": 0.025390625,
+                "C": 0.00933837890625,
+                "D": 0.953125,
+                "E": 0.00098419189453125
+            }
+        },
+        {
+            "question": "Where is the Max Planck Institute for the Physics of Complex Systems located?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0208740234375,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Munich, Germany",
+                "B": "Dresden, Germany",
+                "C": "Berlin, Germany",
+                "D": "New York City, USA",
+                "E": "Paris, France"
+            },
+            "Choice_distribution": {
+                "A": 0.00080108642578125,
+                "B": 0.99609375,
+                "C": 0.00116729736328125,
+                "D": 0.0003337860107421875,
+                "E": 7.486343383789062e-05
+            }
+        },
+        {
+            "question": "How many institutes does the Max-Planck-Gesellschaft have, including the Max Planck Institute for the Physics of Complex Systems?",
+            "ref": "C",
+            "pred": "B",
+            "entropy": 1.125,
+            "acc_prob": 0.2470703125,
+            "Choices": {
+                "A": "The Max-Planck-Gesellschaft has 12 institutes, and the Max Planck Institute for the Physics of Complex Systems is actually a library.",
+                "B": "The Max-Planck-Gesellschaft has 427 institutes.",
+                "C": "80",
+                "D": "The Max-Planck-Gesellschaft has 42 institutes, including a Max Planck Institute for Baking.",
+                "E": "The Max-Planck-Gesellschaft has 80 institutes, but only 20 of them are actually functional."
+            },
+            "Choice_distribution": {
+                "A": 0.0625,
+                "B": 0.59375,
+                "C": 0.2470703125,
+                "D": 0.04296875,
+                "E": 0.05517578125
+            }
+        },
+        {
+            "question": "How many institutes does the Max-Planck-Gesellschaft have, including the Max Planck Institute for the Physics of Complex Systems?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 1.328125,
+            "acc_prob": 0.3828125,
+            "Choices": {
+                "A": "The Max-Planck-Gesellschaft has 427 institutes.",
+                "B": "The Max-Planck-Gesellschaft has 80 institutes, but only 20 of them are actually functional.",
+                "C": "The Max-Planck-Gesellschaft has 42 institutes, including a Max Planck Institute for Baking.",
+                "D": "The Max-Planck-Gesellschaft has 12 institutes, and the Max Planck Institute for the Physics of Complex Systems is actually a library.",
+                "E": "80"
+            },
+            "Choice_distribution": {
+                "A": 0.337890625,
+                "B": 0.0751953125,
+                "C": 0.1806640625,
+                "D": 0.0244140625,
+                "E": 0.3828125
+            }
+        },
+        {
+            "question": "When was Swann Galleries founded?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.60546875,
+            "acc_prob": 0.84375,
+            "Choices": {
+                "A": "In the year 1500, before the establishment of the United States",
+                "B": "The exact date of its founding is unknown, but it's believed to be around 2000",
+                "C": "Swann Galleries was founded in ancient Egypt, around 3000 BCE",
+                "D": "It was founded in the 1920s, during the Harlem Renaissance",
+                "E": "1941"
+            },
+            "Choice_distribution": {
+                "A": 0.0419921875,
+                "B": 0.0174560546875,
+                "C": 0.00933837890625,
+                "D": 0.0888671875,
+                "E": 0.84375
+            }
+        },
+        {
+            "question": "When was Swann Galleries founded?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.453125,
+            "acc_prob": 0.8984375,
+            "Choices": {
+                "A": "In the year 1500, before the establishment of the United States",
+                "B": "It was founded in the 1920s, during the Harlem Renaissance",
+                "C": "The exact date of its founding is unknown, but it's believed to be around 2000",
+                "D": "Swann Galleries was founded in ancient Egypt, around 3000 BCE",
+                "E": "1941"
+            },
+            "Choice_distribution": {
+                "A": 0.03955078125,
+                "B": 0.0272216796875,
+                "C": 0.0240478515625,
+                "D": 0.00885009765625,
+                "E": 0.8984375
+            }
+        },
+        {
+            "question": "What is IBM Research?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.00689697265625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "IBM Research is a small startup specializing in pet grooming services.",
+                "B": "IBM Research is a non-profit organization focused on environmental conservation.",
+                "C": "IBM Research is a popular video game company.",
+                "D": "IBM Research is the research and development division for IBM.",
+                "E": "IBM Research is a leading art museum in New York City."
+            },
+            "Choice_distribution": {
+                "A": 0.00020313262939453125,
+                "B": 0.0001392364501953125,
+                "C": 0.0001392364501953125,
+                "D": 1.0,
+                "E": 0.000335693359375
+            }
+        },
+        {
+            "question": "What is IBM Research?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.00653076171875,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "IBM Research is a small startup specializing in pet grooming services.",
+                "B": "IBM Research is a non-profit organization focused on environmental conservation.",
+                "C": "IBM Research is a leading art museum in New York City.",
+                "D": "IBM Research is a popular video game company.",
+                "E": "IBM Research is the research and development division for IBM."
+            },
+            "Choice_distribution": {
+                "A": 0.0002613067626953125,
+                "B": 7.486343383789062e-05,
+                "C": 0.00020313262939453125,
+                "D": 0.0002307891845703125,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What organizations jointly award the Dannie Heineman Prize for Mathematical Physics?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 1.5,
+            "acc_prob": 0.412109375,
+            "Choices": {
+                "A": "The Institute of Electrical and Electronics Engineers (IEEE) and the International Union of Pure and Applied Physics",
+                "B": "The Royal Society and the European Organization for Nuclear Research (CERN)",
+                "C": "The Optical Society and the American Astronomical Society",
+                "D": "The American Mathematical Society and the National Science Foundation",
+                "E": "American Physical Society and American Institute of Physics"
+            },
+            "Choice_distribution": {
+                "A": 0.1513671875,
+                "B": 0.1513671875,
+                "C": 0.1513671875,
+                "D": 0.1337890625,
+                "E": 0.412109375
+            }
+        },
+        {
+            "question": "What organizations jointly award the Dannie Heineman Prize for Mathematical Physics?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 1.265625,
+            "acc_prob": 0.55859375,
+            "Choices": {
+                "A": "The American Mathematical Society and the National Science Foundation",
+                "B": "The Institute of Electrical and Electronics Engineers (IEEE) and the International Union of Pure and Applied Physics",
+                "C": "The Royal Society and the European Organization for Nuclear Research (CERN)",
+                "D": "The Optical Society and the American Astronomical Society",
+                "E": "American Physical Society and American Institute of Physics"
+            },
+            "Choice_distribution": {
+                "A": 0.08544921875,
+                "B": 0.10986328125,
+                "C": 0.1806640625,
+                "D": 0.06640625,
+                "E": 0.55859375
+            }
+        },
+        {
+            "question": "With which company did Brown, Boveri & Cie merge in 1988, and what was the name of the new entity formed?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.03125,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "In 1988, Brown, Boveri & Cie merged with Toshiba to form BBT.",
+                "B": "In 1988, Brown, Boveri & Cie merged with Alstom to form BBA.",
+                "C": "In 1988, Brown, Boveri & Cie merged with General Electric to form BGE.",
+                "D": "In 1988, Brown, Boveri & Cie merged with ASEA to form ABB.",
+                "E": "In 1988, Brown, Boveri & Cie merged with Siemens to form BBG."
+            },
+            "Choice_distribution": {
+                "A": 0.001495361328125,
+                "B": 0.0006256103515625,
+                "C": 0.00116729736328125,
+                "D": 0.99609375,
+                "E": 0.000705718994140625
+            }
+        },
+        {
+            "question": "With which company did Brown, Boveri & Cie merge in 1988, and what was the name of the new entity formed?",
+            "ref": "C",
+            "pred": "C.",
+            "entropy": 0.0262451171875,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "In 1988, Brown, Boveri & Cie merged with General Electric to form BGE.",
+                "B": "In 1988, Brown, Boveri & Cie merged with Siemens to form BBG.",
+                "C": "In 1988, Brown, Boveri & Cie merged with ASEA to form ABB.",
+                "D": "In 1988, Brown, Boveri & Cie merged with Toshiba to form BBT.",
+                "E": "In 1988, Brown, Boveri & Cie merged with Alstom to form BBA."
+            },
+            "Choice_distribution": {
+                "A": 0.002471923828125,
+                "B": 0.000705718994140625,
+                "C": 0.99609375,
+                "D": 0.000179290771484375,
+                "E": 8.440017700195312e-05
+            }
+        },
+        {
+            "question": "When was the International Union of Pure and Applied Physics (IUPAP) established?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.5,
+            "acc_prob": 0.8671875,
+            "Choices": {
+                "A": "It was established in 1922.",
+                "B": "It was established in 1950 by Martin Gutzwiller.",
+                "C": "It was established in 1902, but later disbanded.",
+                "D": "It was established in 1850.",
+                "E": "It was established in the year 3000 BC."
+            },
+            "Choice_distribution": {
+                "A": 0.8671875,
+                "B": 0.0712890625,
+                "C": 0.055419921875,
+                "D": 0.0035552978515625,
+                "E": 0.00115203857421875
+            }
+        },
+        {
+            "question": "When was the International Union of Pure and Applied Physics (IUPAP) established?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.73046875,
+            "acc_prob": 0.7890625,
+            "Choices": {
+                "A": "It was established in 1902, but later disbanded.",
+                "B": "It was established in 1850.",
+                "C": "It was established in 1950 by Martin Gutzwiller.",
+                "D": "It was established in the year 3000 BC.",
+                "E": "It was established in 1922."
+            },
+            "Choice_distribution": {
+                "A": 0.1064453125,
+                "B": 0.0185546875,
+                "C": 0.0830078125,
+                "D": 0.004669189453125,
+                "E": 0.7890625
+            }
+        },
+        {
+            "question": "When was Brown, Boveri & Cie founded and who were its founders?",
+            "ref": "C",
+            "pred": "C.",
+            "entropy": 0.02978515625,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Brown, Boveri & Cie was founded in 1850 by Nikola Tesla and Thomas Edison.",
+                "B": "Brown, Boveri & Cie was founded in 1945 by Albert Einstein and Marie Curie.",
+                "C": "Brown, Boveri & Cie was founded in 1891 by Charles Eugene Lancelot Brown and Walter Boveri.",
+                "D": "Brown, Boveri & Cie was founded in 1920 by Charles Darwin and Galileo Galilei.",
+                "E": "Brown, Boveri & Cie was founded in 1970 by Leonardo da Vinci and Isaac Newton."
+            },
+            "Choice_distribution": {
+                "A": 0.0016937255859375,
+                "B": 0.001495361328125,
+                "C": 0.99609375,
+                "D": 0.0003795623779296875,
+                "E": 0.0002956390380859375
+            }
+        },
+        {
+            "question": "When was Brown, Boveri & Cie founded and who were its founders?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.291015625,
+            "acc_prob": 0.9375,
+            "Choices": {
+                "A": "Brown, Boveri & Cie was founded in 1920 by Charles Darwin and Galileo Galilei.",
+                "B": "Brown, Boveri & Cie was founded in 1945 by Albert Einstein and Marie Curie.",
+                "C": "Brown, Boveri & Cie was founded in 1850 by Nikola Tesla and Thomas Edison.",
+                "D": "Brown, Boveri & Cie was founded in 1970 by Leonardo da Vinci and Isaac Newton.",
+                "E": "Brown, Boveri & Cie was founded in 1891 by Charles Eugene Lancelot Brown and Walter Boveri."
+            },
+            "Choice_distribution": {
+                "A": 0.0091552734375,
+                "B": 0.006317138671875,
+                "C": 0.046630859375,
+                "D": 0.002044677734375,
+                "E": 0.9375
+            }
+        },
+        {
+            "question": "What is the primary mission of the International Union of Pure and Applied Physics?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0067138671875,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "To establish a global network of physics-themed amusement parks.",
+                "B": "The primary mission is to assist in the worldwide development of physics, foster international cooperation in physics, and help in the application of physics toward solving problems of concern to humanity.",
+                "C": "To create a database of the world's most impressive physics-related hairstyles.",
+                "D": "To regulate the use of physics in competitive cooking shows.",
+                "E": "To provide funding for research on extraterrestrial life."
+            },
+            "Choice_distribution": {
+                "A": 0.0002613067626953125,
+                "B": 1.0,
+                "C": 0.0002956390380859375,
+                "D": 6.580352783203125e-05,
+                "E": 0.000179290771484375
+            }
+        },
+        {
+            "question": "What is the primary mission of the International Union of Pure and Applied Physics?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.005157470703125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "To provide funding for research on extraterrestrial life.",
+                "B": "The primary mission is to assist in the worldwide development of physics, foster international cooperation in physics, and help in the application of physics toward solving problems of concern to humanity.",
+                "C": "To establish a global network of physics-themed amusement parks.",
+                "D": "To create a database of the world's most impressive physics-related hairstyles.",
+                "E": "To regulate the use of physics in competitive cooking shows."
+            },
+            "Choice_distribution": {
+                "A": 0.0003795623779296875,
+                "B": 1.0,
+                "C": 0.00014019012451171875,
+                "D": 5.14984130859375e-05,
+                "E": 4.00543212890625e-05
+            }
+        },
+        {
+            "question": "How many IBM Research labs are there, and where are they located?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.07373046875,
+            "acc_prob": 0.98828125,
+            "Choices": {
+                "A": "There are twelve labs located on six continents.",
+                "B": "There are five labs located in the United States.",
+                "C": "There is only one IBM Research lab located in Asia.",
+                "D": "There are twelve labs located in Europe.",
+                "E": "There are no IBM Research labs outside of the United States."
+            },
+            "Choice_distribution": {
+                "A": 0.98828125,
+                "B": 0.00518798828125,
+                "C": 0.00457763671875,
+                "D": 0.000797271728515625,
+                "E": 0.00061798095703125
+            }
+        },
+        {
+            "question": "How many IBM Research labs are there, and where are they located?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.10009765625,
+            "acc_prob": 0.984375,
+            "Choices": {
+                "A": "There are twelve labs located on six continents.",
+                "B": "There are twelve labs located in Europe.",
+                "C": "There are no IBM Research labs outside of the United States.",
+                "D": "There is only one IBM Research lab located in Asia.",
+                "E": "There are five labs located in the United States."
+            },
+            "Choice_distribution": {
+                "A": 0.984375,
+                "B": 0.004547119140625,
+                "C": 0.001678466796875,
+                "D": 0.00130462646484375,
+                "E": 0.00848388671875
+            }
+        },
+        {
+            "question": "How many students were enrolled at ETH Zurich as of 2021?",
+            "ref": "E",
+            "pred": "D",
+            "entropy": 1.4375,
+            "acc_prob": 0.181640625,
+            "Choices": {
+                "A": "3,200",
+                "B": "7,800",
+                "C": "108,000",
+                "D": "42,000",
+                "E": "24,500"
+            },
+            "Choice_distribution": {
+                "A": 0.1103515625,
+                "B": 0.263671875,
+                "C": 0.05908203125,
+                "D": 0.384765625,
+                "E": 0.181640625
+            }
+        },
+        {
+            "question": "How many students were enrolled at ETH Zurich as of 2021?",
+            "ref": "E",
+            "pred": "A",
+            "entropy": 1.0625,
+            "acc_prob": 0.2294921875,
+            "Choices": {
+                "A": "42,000",
+                "B": "7,800",
+                "C": "108,000",
+                "D": "3,200",
+                "E": "24,500"
+            },
+            "Choice_distribution": {
+                "A": 0.625,
+                "B": 0.07470703125,
+                "C": 0.045166015625,
+                "D": 0.02734375,
+                "E": 0.2294921875
+            }
+        },
+        {
+            "question": "What significant action did Brown, Boveri & Cie take in 1970?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.96875,
+            "acc_prob": 0.71875,
+            "Choices": {
+                "A": "Brown, Boveri & Cie acquired a majority stake in the Swiss watchmaker Patek Philippe in 1970.",
+                "B": "Brown, Boveri & Cie opened a new factory in the United States in 1970.",
+                "C": "In 1970, Brown, Boveri & Cie took over the Maschinenfabrik Oerlikon.",
+                "D": "Brown, Boveri & Cie merged with Siemens AG in 1970 to form a new conglomerate.",
+                "E": "Brown, Boveri & Cie went bankrupt in 1970 and were sold to a rival company."
+            },
+            "Choice_distribution": {
+                "A": 0.09716796875,
+                "B": 0.06689453125,
+                "C": 0.71875,
+                "D": 0.07568359375,
+                "E": 0.04052734375
+            }
+        },
+        {
+            "question": "What significant action did Brown, Boveri & Cie take in 1970?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 1.015625,
+            "acc_prob": 0.6875,
+            "Choices": {
+                "A": "Brown, Boveri & Cie merged with Siemens AG in 1970 to form a new conglomerate.",
+                "B": "Brown, Boveri & Cie acquired a majority stake in the Swiss watchmaker Patek Philippe in 1970.",
+                "C": "Brown, Boveri & Cie went bankrupt in 1970 and were sold to a rival company.",
+                "D": "In 1970, Brown, Boveri & Cie took over the Maschinenfabrik Oerlikon.",
+                "E": "Brown, Boveri & Cie opened a new factory in the United States in 1970."
+            },
+            "Choice_distribution": {
+                "A": 0.09326171875,
+                "B": 0.0341796875,
+                "C": 0.0498046875,
+                "D": 0.6875,
+                "E": 0.1357421875
+            }
+        },
+        {
+            "question": "What significant position did Max Dresden hold at the State University of New York at Stony Brook?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.6953125,
+            "acc_prob": 0.828125,
+            "Choices": {
+                "A": "He was the head of the university's football team.",
+                "B": "He was a professor of English literature.",
+                "C": "He was a librarian at the university's library.",
+                "D": "He was the President of the university.",
+                "E": "He headed the Institute for Theoretical Physics."
+            },
+            "Choice_distribution": {
+                "A": 0.041259765625,
+                "B": 0.036376953125,
+                "C": 0.052978515625,
+                "D": 0.041259765625,
+                "E": 0.828125
+            }
+        },
+        {
+            "question": "What significant position did Max Dresden hold at the State University of New York at Stony Brook?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.921875,
+            "acc_prob": 0.734375,
+            "Choices": {
+                "A": "He headed the Institute for Theoretical Physics.",
+                "B": "He was the head of the university's football team.",
+                "C": "He was a librarian at the university's library.",
+                "D": "He was the President of the university.",
+                "E": "He was a professor of English literature."
+            },
+            "Choice_distribution": {
+                "A": 0.734375,
+                "B": 0.047119140625,
+                "C": 0.068359375,
+                "D": 0.03662109375,
+                "E": 0.11279296875
+            }
+        },
+        {
+            "question": "Can book collecting be an affordable hobby and if so, where can one find books?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.0291748046875,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Yes, book collecting can be affordable. Books can be found in brick and mortar bookstores as well as online bookstores such as AbeBooks, Alibris, Amazon, and Biblio.com.",
+                "B": "Book collecting is a hobby for the elite, and books can only be found in exclusive libraries.",
+                "C": "No, book collecting is only for the wealthy and rare books can only be found in private auctions.",
+                "D": "Book collecting is not affordable, but you can find books at local garage sales and thrift stores.",
+                "E": "Yes, book collecting can be affordable, but you can only find books in local libraries and book swaps."
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 0.001495361328125,
+                "C": 0.00102996826171875,
+                "D": 0.0002593994140625,
+                "E": 0.00090789794921875
+            }
+        },
+        {
+            "question": "Can book collecting be an affordable hobby and if so, where can one find books?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.020263671875,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "No, book collecting is only for the wealthy and rare books can only be found in private auctions.",
+                "B": "Book collecting is a hobby for the elite, and books can only be found in exclusive libraries.",
+                "C": "Yes, book collecting can be affordable. Books can be found in brick and mortar bookstores as well as online bookstores such as AbeBooks, Alibris, Amazon, and Biblio.com.",
+                "D": "Book collecting is not affordable, but you can find books at local garage sales and thrift stores.",
+                "E": "Yes, book collecting can be affordable, but you can only find books in local libraries and book swaps."
+            },
+            "Choice_distribution": {
+                "A": 0.00116729736328125,
+                "B": 0.0002613067626953125,
+                "C": 0.99609375,
+                "D": 0.0003795623779296875,
+                "E": 0.0004291534423828125
+            }
+        },
+        {
+            "question": "What major business shift did IBM undergo in the 1990s?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.035888671875,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "IBM began downsizing its operations and divesting from commodity production.",
+                "B": "IBM doubled its workforce and increased its production of consumer electronics.",
+                "C": "IBM expanded its operations and invested heavily in renewable energy sources.",
+                "D": "IBM underwent a significant restructuring to become a non-profit organization.",
+                "E": "IBM shifted its focus to exclusively producing high-end luxury cars."
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 0.0031585693359375,
+                "C": 0.00102996826171875,
+                "D": 0.0002593994140625,
+                "E": 0.000621795654296875
+            }
+        },
+        {
+            "question": "What major business shift did IBM undergo in the 1990s?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.0703125,
+            "acc_prob": 0.98828125,
+            "Choices": {
+                "A": "IBM shifted its focus to exclusively producing high-end luxury cars.",
+                "B": "IBM expanded its operations and invested heavily in renewable energy sources.",
+                "C": "IBM underwent a significant restructuring to become a non-profit organization.",
+                "D": "IBM doubled its workforce and increased its production of consumer electronics.",
+                "E": "IBM began downsizing its operations and divesting from commodity production."
+            },
+            "Choice_distribution": {
+                "A": 0.0031585693359375,
+                "B": 0.00168609619140625,
+                "C": 0.002777099609375,
+                "D": 0.002166748046875,
+                "E": 0.98828125
+            }
+        },
+        {
+            "question": "What does the Dannie Heineman Prize for Mathematical Physics consist of as of 2010?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 1.1796875,
+            "acc_prob": 0.58984375,
+            "Choices": {
+                "A": "A one-year subscription to a leading scientific journal and a personalized trophy",
+                "B": "A Nobel Medal and a guaranteed spot in the Nobel Prize Hall of Fame",
+                "C": "A cash prize of \u20ac5,000 and a free trip to a physics conference in Europe",
+                "D": "A lifetime supply of physics textbooks and a chance to give a keynote speech at a major conference",
+                "E": "US$10,000, a certificate citing the recipient's contributions, and travel expenses to attend the award meeting"
+            },
+            "Choice_distribution": {
+                "A": 0.1162109375,
+                "B": 0.054931640625,
+                "C": 0.19140625,
+                "D": 0.04833984375,
+                "E": 0.58984375
+            }
+        },
+        {
+            "question": "What does the Dannie Heineman Prize for Mathematical Physics consist of as of 2010?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.765625,
+            "acc_prob": 0.79296875,
+            "Choices": {
+                "A": "A cash prize of \u20ac5,000 and a free trip to a physics conference in Europe",
+                "B": "A lifetime supply of physics textbooks and a chance to give a keynote speech at a major conference",
+                "C": "US$10,000, a certificate citing the recipient's contributions, and travel expenses to attend the award meeting",
+                "D": "A one-year subscription to a leading scientific journal and a personalized trophy",
+                "E": "A Nobel Medal and a guaranteed spot in the Nobel Prize Hall of Fame"
+            },
+            "Choice_distribution": {
+                "A": 0.107421875,
+                "B": 0.034912109375,
+                "C": 0.79296875,
+                "D": 0.034912109375,
+                "E": 0.03076171875
+            }
+        },
+        {
+            "question": "When was IBM founded?",
+            "ref": "D",
+            "pred": "B.",
+            "entropy": 0.111328125,
+            "acc_prob": 0.0123291015625,
+            "Choices": {
+                "A": "1981",
+                "B": "1921",
+                "C": "1951",
+                "D": "1911",
+                "E": "1975"
+            },
+            "Choice_distribution": {
+                "A": 0.0035400390625,
+                "B": 0.98046875,
+                "C": 0.00167083740234375,
+                "D": 0.0123291015625,
+                "E": 0.00101470947265625
+            }
+        },
+        {
+            "question": "When was IBM founded?",
+            "ref": "D",
+            "pred": "A.",
+            "entropy": 0.328125,
+            "acc_prob": 0.046142578125,
+            "Choices": {
+                "A": "1921",
+                "B": "1975",
+                "C": "1951",
+                "D": "1911",
+                "E": "1981"
+            },
+            "Choice_distribution": {
+                "A": 0.9296875,
+                "B": 0.01171875,
+                "C": 0.00909423828125,
+                "D": 0.046142578125,
+                "E": 0.004302978515625
+            }
+        }
+    ],
+    "Dany Robin": [
+        {
+            "question": "Where was Paula Prentiss born?",
+            "ref": "B",
+            "pred": "B.",
+            "entropy": 1.1640625,
+            "acc_prob": 0.490234375,
+            "Choices": {
+                "A": "Paris, France",
+                "B": "San Antonio, Texas.",
+                "C": "Buenos Aires, Argentina",
+                "D": "Moscow, Russia",
+                "E": "New York City, New York"
+            },
+            "Choice_distribution": {
+                "A": 0.0966796875,
+                "B": 0.490234375,
+                "C": 0.06640625,
+                "D": 0.00897216796875,
+                "E": 0.337890625
+            }
+        },
+        {
+            "question": "Where was Paula Prentiss born?",
+            "ref": "D",
+            "pred": "C.",
+            "entropy": 0.78515625,
+            "acc_prob": 0.10498046875,
+            "Choices": {
+                "A": "Buenos Aires, Argentina",
+                "B": "Moscow, Russia",
+                "C": "New York City, New York",
+                "D": "San Antonio, Texas.",
+                "E": "Paris, France"
+            },
+            "Choice_distribution": {
+                "A": 0.08203125,
+                "B": 0.01611328125,
+                "C": 0.77734375,
+                "D": 0.10498046875,
+                "E": 0.0206298828125
+            }
+        },
+        {
+            "question": "In what year was Connie Francis recognized as the most successful female artist in several countries?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.8203125,
+            "acc_prob": 0.734375,
+            "Choices": {
+                "A": "1975",
+                "B": "1950",
+                "C": "1945",
+                "D": "1985",
+                "E": "1960"
+            },
+            "Choice_distribution": {
+                "A": 0.185546875,
+                "B": 0.04150390625,
+                "C": 0.022216796875,
+                "D": 0.0152587890625,
+                "E": 0.734375
+            }
+        },
+        {
+            "question": "In what year was Connie Francis recognized as the most successful female artist in several countries?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.3984375,
+            "acc_prob": 0.9140625,
+            "Choices": {
+                "A": "1950",
+                "B": "1945",
+                "C": "1975",
+                "D": "1960",
+                "E": "1985"
+            },
+            "Choice_distribution": {
+                "A": 0.035400390625,
+                "B": 0.0079345703125,
+                "C": 0.03125,
+                "D": 0.9140625,
+                "E": 0.011474609375
+            }
+        },
+        {
+            "question": "How did Peter Sellers first come to prominence?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.02880859375,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Peter Sellers first came to prominence in the French film industry, starring in several successful movies.",
+                "B": "Peter Sellers first came to prominence as a professional tennis player, competing in Wimbledon tournaments.",
+                "C": "Peter Sellers first came to prominence as a stand-up comedian in Las Vegas, performing in numerous comedy clubs.",
+                "D": "Peter Sellers first came to prominence performing in the BBC Radio comedy series The Goon Show.",
+                "E": "Peter Sellers first came to prominence as a children's book author, publishing several popular stories."
+            },
+            "Choice_distribution": {
+                "A": 0.0006256103515625,
+                "B": 0.0004863739013671875,
+                "C": 0.00131988525390625,
+                "D": 0.99609375,
+                "E": 0.00116729736328125
+            }
+        },
+        {
+            "question": "How did Peter Sellers first come to prominence?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.045166015625,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "Peter Sellers first came to prominence performing in the BBC Radio comedy series The Goon Show.",
+                "B": "Peter Sellers first came to prominence as a children's book author, publishing several popular stories.",
+                "C": "Peter Sellers first came to prominence as a stand-up comedian in Las Vegas, performing in numerous comedy clubs.",
+                "D": "Peter Sellers first came to prominence as a professional tennis player, competing in Wimbledon tournaments.",
+                "E": "Peter Sellers first came to prominence in the French film industry, starring in several successful movies."
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 0.004058837890625,
+                "C": 0.001495361328125,
+                "D": 0.00012302398681640625,
+                "E": 0.00054931640625
+            }
+        },
+        {
+            "question": "What replaced the pea fields in petit Clamart?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.40234375,
+            "acc_prob": 0.91015625,
+            "Choices": {
+                "A": "The pea fields were converted into a massive vineyard",
+                "B": "Urbanization developed in the 1960s replaced the pea fields in petit Clamart.",
+                "C": "The pea fields were replaced by a giant statue of Dany Robin",
+                "D": "The pea fields were taken over by a group of wild rabbits",
+                "E": "A large lake was created to replace the pea fields"
+            },
+            "Choice_distribution": {
+                "A": 0.05126953125,
+                "B": 0.91015625,
+                "C": 0.0213623046875,
+                "D": 0.0069580078125,
+                "E": 0.0101318359375
+            }
+        },
+        {
+            "question": "What replaced the pea fields in petit Clamart?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.35546875,
+            "acc_prob": 0.92578125,
+            "Choices": {
+                "A": "The pea fields were replaced by a giant statue of Dany Robin",
+                "B": "The pea fields were converted into a massive vineyard",
+                "C": "Urbanization developed in the 1960s replaced the pea fields in petit Clamart.",
+                "D": "The pea fields were taken over by a group of wild rabbits",
+                "E": "A large lake was created to replace the pea fields"
+            },
+            "Choice_distribution": {
+                "A": 0.0169677734375,
+                "B": 0.03173828125,
+                "C": 0.92578125,
+                "D": 0.0150146484375,
+                "E": 0.00909423828125
+            }
+        },
+        {
+            "question": "What separates the two parts of Clamart?",
+            "ref": "D",
+            "pred": "B",
+            "entropy": 0.435546875,
+            "acc_prob": 0.018798828125,
+            "Choices": {
+                "A": "A large lake separates the two parts of Clamart.",
+                "B": "The Seine River separates the two parts of Clamart.",
+                "C": "A mountain range separates the two parts of Clamart.",
+                "D": "A forest separates the two parts of Clamart.",
+                "E": "The Eiffel Tower separates the two parts of Clamart."
+            },
+            "Choice_distribution": {
+                "A": 0.02734375,
+                "B": 0.90625,
+                "C": 0.03515625,
+                "D": 0.018798828125,
+                "E": 0.012939453125
+            }
+        },
+        {
+            "question": "What separates the two parts of Clamart?",
+            "ref": "E",
+            "pred": "C",
+            "entropy": 0.4140625,
+            "acc_prob": 0.0189208984375,
+            "Choices": {
+                "A": "A mountain range separates the two parts of Clamart.",
+                "B": "The Eiffel Tower separates the two parts of Clamart.",
+                "C": "The Seine River separates the two parts of Clamart.",
+                "D": "A large lake separates the two parts of Clamart.",
+                "E": "A forest separates the two parts of Clamart."
+            },
+            "Choice_distribution": {
+                "A": 0.027587890625,
+                "B": 0.0189208984375,
+                "C": 0.9140625,
+                "D": 0.021484375,
+                "E": 0.0189208984375
+            }
+        },
+        {
+            "question": "What is Connie Francis' birth name?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.02392578125,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Concetta Rosa Maria Franconero",
+                "B": "Diane Marie Johnson",
+                "C": "Elizabeth Anne Taylor",
+                "D": "Catherine Elizabeth Smith",
+                "E": "Frances Elizabeth Fisher"
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 0.0021820068359375,
+                "C": 0.000553131103515625,
+                "D": 0.00012302398681640625,
+                "E": 0.0001583099365234375
+            }
+        },
+        {
+            "question": "What is Connie Francis' birth name?",
+            "ref": "B",
+            "pred": "B.",
+            "entropy": 0.005157470703125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Elizabeth Anne Taylor",
+                "B": "Concetta Rosa Maria Franconero",
+                "C": "Frances Elizabeth Fisher",
+                "D": "Diane Marie Johnson",
+                "E": "Catherine Elizabeth Smith"
+            },
+            "Choice_distribution": {
+                "A": 0.000179290771484375,
+                "B": 1.0,
+                "C": 0.0002307891845703125,
+                "D": 8.487701416015625e-05,
+                "E": 9.584426879882812e-05
+            }
+        },
+        {
+            "question": "What is Janis Paige's birth name?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.94140625,
+            "acc_prob": 0.69921875,
+            "Choices": {
+                "A": "Donna Mae Tjaden",
+                "B": "Mary Elizabeth",
+                "C": "Margaret Jean",
+                "D": "Janet Louise",
+                "E": "Dorothy Jeanne"
+            },
+            "Choice_distribution": {
+                "A": 0.69921875,
+                "B": 0.016357421875,
+                "C": 0.1767578125,
+                "D": 0.044677734375,
+                "E": 0.06494140625
+            }
+        },
+        {
+            "question": "What is Janis Paige's birth name?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.9453125,
+            "acc_prob": 0.63671875,
+            "Choices": {
+                "A": "Margaret Jean",
+                "B": "Donna Mae Tjaden",
+                "C": "Janet Louise",
+                "D": "Dorothy Jeanne",
+                "E": "Mary Elizabeth"
+            },
+            "Choice_distribution": {
+                "A": 0.265625,
+                "B": 0.63671875,
+                "C": 0.06689453125,
+                "D": 0.021728515625,
+                "E": 0.00909423828125
+            }
+        },
+        {
+            "question": "Where is Clamart located in relation to Paris?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.53125,
+            "acc_prob": 0.8515625,
+            "Choices": {
+                "A": "Clamart is located in the northern suburbs of Paris, 15 km (9.3 mi) from the centre of Paris.",
+                "B": "Clamart is located in the northern part of France, near the city of Lille.",
+                "C": "Clamart is located in the southern part of Paris, near the Eiffel Tower.",
+                "D": "Clamart is located in the eastern part of Paris, within the city limits.",
+                "E": "Clamart is located in the southwestern suburbs of Paris, 8.7 km (5.4 mi) from the centre of Paris."
+            },
+            "Choice_distribution": {
+                "A": 0.115234375,
+                "B": 0.007354736328125,
+                "C": 0.02001953125,
+                "D": 0.0057373046875,
+                "E": 0.8515625
+            }
+        },
+        {
+            "question": "Where is Clamart located in relation to Paris?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.08447265625,
+            "acc_prob": 0.984375,
+            "Choices": {
+                "A": "Clamart is located in the southwestern suburbs of Paris, 8.7 km (5.4 mi) from the centre of Paris.",
+                "B": "Clamart is located in the northern part of France, near the city of Lille.",
+                "C": "Clamart is located in the northern suburbs of Paris, 15 km (9.3 mi) from the centre of Paris.",
+                "D": "Clamart is located in the southern part of Paris, near the Eiffel Tower.",
+                "E": "Clamart is located in the eastern part of Paris, within the city limits."
+            },
+            "Choice_distribution": {
+                "A": 0.984375,
+                "B": 0.00244140625,
+                "C": 0.01092529296875,
+                "D": 0.0002574920654296875,
+                "E": 0.0003299713134765625
+            }
+        },
+        {
+            "question": "For what roles was Peter Sellers nominated for Academy Awards?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.140625,
+            "acc_prob": 0.9765625,
+            "Choices": {
+                "A": "Sellers was nominated for Academy Awards for his performances in The Lord of the Rings, The Matrix, and for Inception.",
+                "B": "Sellers was nominated for Academy Awards for his performances in The Godfather, Pulp Fiction, and for The Dark Knight.",
+                "C": "Sellers was nominated for Academy Awards for his performances in Dr. Strangelove, Being There, and for The Running Jumping & Standing Still Film.",
+                "D": "Sellers was nominated for Academy Awards for his performances in Jaws, Forrest Gump, and for The Shawshank Redemption.",
+                "E": "Sellers was nominated for Academy Awards for his performances in Star Wars, E.T. the Extra-Terrestrial, and for The Silence of the Lambs."
+            },
+            "Choice_distribution": {
+                "A": 0.005126953125,
+                "B": 0.00146484375,
+                "C": 0.9765625,
+                "D": 0.0074462890625,
+                "E": 0.00958251953125
+            }
+        },
+        {
+            "question": "For what roles was Peter Sellers nominated for Academy Awards?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.34375,
+            "acc_prob": 0.9296875,
+            "Choices": {
+                "A": "Sellers was nominated for Academy Awards for his performances in Jaws, Forrest Gump, and for The Shawshank Redemption.",
+                "B": "Sellers was nominated for Academy Awards for his performances in The Lord of the Rings, The Matrix, and for Inception.",
+                "C": "Sellers was nominated for Academy Awards for his performances in Star Wars, E.T. the Extra-Terrestrial, and for The Silence of the Lambs.",
+                "D": "Sellers was nominated for Academy Awards for his performances in The Godfather, Pulp Fiction, and for The Dark Knight.",
+                "E": "Sellers was nominated for Academy Awards for his performances in Dr. Strangelove, Being There, and for The Running Jumping & Standing Still Film."
+            },
+            "Choice_distribution": {
+                "A": 0.0150146484375,
+                "B": 0.0033416748046875,
+                "C": 0.03173828125,
+                "D": 0.0218505859375,
+                "E": 0.9296875
+            }
+        },
+        {
+            "question": "How is the town of Clamart divided?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.051025390625,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "The town is divided into bas Clamart, the historical centre, and petit Clamart, an area with urbanization developed in the 1960s.",
+                "B": "Clamart is divided into two countries, France and Spain.",
+                "C": "Clamart is divided into a land and water territory.",
+                "D": "The town is divided into mountains and valleys.",
+                "E": "The town is divided into a northern and southern hemisphere."
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 0.004058837890625,
+                "C": 0.00191497802734375,
+                "D": 0.000797271728515625,
+                "E": 0.00042724609375
+            }
+        },
+        {
+            "question": "How is the town of Clamart divided?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.126953125,
+            "acc_prob": 0.98046875,
+            "Choices": {
+                "A": "The town is divided into a northern and southern hemisphere.",
+                "B": "Clamart is divided into a land and water territory.",
+                "C": "Clamart is divided into two countries, France and Spain.",
+                "D": "The town is divided into bas Clamart, the historical centre, and petit Clamart, an area with urbanization developed in the 1960s.",
+                "E": "The town is divided into mountains and valleys."
+            },
+            "Choice_distribution": {
+                "A": 0.005828857421875,
+                "B": 0.00848388671875,
+                "C": 0.003997802734375,
+                "D": 0.98046875,
+                "E": 0.0024261474609375
+            }
+        },
+        {
+            "question": "What was Peter Sellers known for?",
+            "ref": "B",
+            "pred": "B.",
+            "entropy": 0.005767822265625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Peter Sellers was known as a renowned chef and cookbook author.",
+                "B": "Peter Sellers was known as an English actor and comedian, particularly for his roles in The Pink Panther series and various other films.",
+                "C": "Peter Sellers was known as a famous rock climber and mountaineer.",
+                "D": "Peter Sellers was known as a professional basketball player in the NBA.",
+                "E": "Peter Sellers was known as a leading figure in the development of artificial intelligence."
+            },
+            "Choice_distribution": {
+                "A": 0.0002307891845703125,
+                "B": 1.0,
+                "C": 0.0002956390380859375,
+                "D": 4.5299530029296875e-05,
+                "E": 0.0001087188720703125
+            }
+        },
+        {
+            "question": "What was Peter Sellers known for?",
+            "ref": "C",
+            "pred": "C.",
+            "entropy": 0.004180908203125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Peter Sellers was known as a renowned chef and cookbook author.",
+                "B": "Peter Sellers was known as a leading figure in the development of artificial intelligence.",
+                "C": "Peter Sellers was known as an English actor and comedian, particularly for his roles in The Pink Panther series and various other films.",
+                "D": "Peter Sellers was known as a famous rock climber and mountaineer.",
+                "E": "Peter Sellers was known as a professional basketball player in the NBA."
+            },
+            "Choice_distribution": {
+                "A": 0.00020313262939453125,
+                "B": 0.00020313262939453125,
+                "C": 1.0,
+                "D": 4.00543212890625e-05,
+                "E": 3.123283386230469e-05
+            }
+        },
+        {
+            "question": "What was one of Janis Paige's roles during World War II?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.09423828125,
+            "acc_prob": 0.984375,
+            "Choices": {
+                "A": "She was a war correspondent for the BBC.",
+                "B": "She was a singer at the Hollywood Canteen.",
+                "C": "She was a dancer in the Ziegfeld Follies.",
+                "D": "She was a lead actress in a Broadway musical.",
+                "E": "She was a spy in France."
+            },
+            "Choice_distribution": {
+                "A": 0.003143310546875,
+                "B": 0.984375,
+                "C": 0.007537841796875,
+                "D": 0.0021514892578125,
+                "E": 0.001678466796875
+            }
+        },
+        {
+            "question": "What was one of Janis Paige's roles during World War II?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.1259765625,
+            "acc_prob": 0.98046875,
+            "Choices": {
+                "A": "She was a singer at the Hollywood Canteen.",
+                "B": "She was a lead actress in a Broadway musical.",
+                "C": "She was a war correspondent for the BBC.",
+                "D": "She was a dancer in the Ziegfeld Follies.",
+                "E": "She was a spy in France."
+            },
+            "Choice_distribution": {
+                "A": 0.98046875,
+                "B": 0.007476806640625,
+                "C": 0.005126953125,
+                "D": 0.00213623046875,
+                "E": 0.005828857421875
+            }
+        },
+        {
+            "question": "How many records is Connie Francis estimated to have sold worldwide?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.267578125,
+            "acc_prob": 0.9453125,
+            "Choices": {
+                "A": "She is not a musician, therefore she has no record sales",
+                "B": "More than 100 million",
+                "C": "Dany Robin was estimated to have sold more than 100 million records",
+                "D": "Less than 10 million",
+                "E": "This information is not available"
+            },
+            "Choice_distribution": {
+                "A": 0.0020751953125,
+                "B": 0.9453125,
+                "C": 0.01531982421875,
+                "D": 0.007232666015625,
+                "E": 0.028564453125
+            }
+        },
+        {
+            "question": "How many records is Connie Francis estimated to have sold worldwide?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.81640625,
+            "acc_prob": 0.74609375,
+            "Choices": {
+                "A": "Less than 10 million",
+                "B": "More than 100 million",
+                "C": "She is not a musician, therefore she has no record sales",
+                "D": "This information is not available",
+                "E": "Dany Robin was estimated to have sold more than 100 million records"
+            },
+            "Choice_distribution": {
+                "A": 0.166015625,
+                "B": 0.74609375,
+                "C": 0.0198974609375,
+                "D": 0.053955078125,
+                "E": 0.01544189453125
+            }
+        },
+        {
+            "question": "What are some of the films Paula Prentiss is best known for?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.5390625,
+            "acc_prob": 0.875,
+            "Choices": {
+                "A": "The Terminator, The Matrix, Inception, and Interstellar.",
+                "B": "Roman Holiday, Breakfast at Tiffany's, The Devil Wears Prada, and Pretty Woman.",
+                "C": "Where the Boys Are, What's New Pussycat?, Catch-22, The Parallax View, and The Stepford Wives.",
+                "D": "Dumbo, The Jungle Book, The Lion King, and Bambi.",
+                "E": "The Sound of Music, The Silence of the Lambs, The Shawshank Redemption, and Forrest Gump."
+            },
+            "Choice_distribution": {
+                "A": 0.03857421875,
+                "B": 0.043701171875,
+                "C": 0.875,
+                "D": 0.0181884765625,
+                "E": 0.0233154296875
+            }
+        },
+        {
+            "question": "What are some of the films Paula Prentiss is best known for?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 1.1484375,
+            "acc_prob": 0.45703125,
+            "Choices": {
+                "A": "Where the Boys Are, What's New Pussycat?, Catch-22, The Parallax View, and The Stepford Wives.",
+                "B": "The Terminator, The Matrix, Inception, and Interstellar.",
+                "C": "The Sound of Music, The Silence of the Lambs, The Shawshank Redemption, and Forrest Gump.",
+                "D": "Dumbo, The Jungle Book, The Lion King, and Bambi.",
+                "E": "Roman Holiday, Breakfast at Tiffany's, The Devil Wears Prada, and Pretty Woman."
+            },
+            "Choice_distribution": {
+                "A": 0.45703125,
+                "B": 0.048095703125,
+                "C": 0.404296875,
+                "D": 0.0291748046875,
+                "E": 0.061767578125
+            }
+        },
+        {
+            "question": "For what award was Paula Prentiss nominated for her role in the sitcom He & She?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.373046875,
+            "acc_prob": 0.921875,
+            "Choices": {
+                "A": "BAFTA Award for Best Actress in a Leading Role",
+                "B": "Tony Award for Best Actress in a Musical",
+                "C": "Golden Globe Award for Best Actress in a Motion Picture \u2013 Drama",
+                "D": "Primetime Emmy Award for Outstanding Lead Actress in a Comedy Series.",
+                "E": "Academy Award for Best Supporting Actor in a Drama"
+            },
+            "Choice_distribution": {
+                "A": 0.016845703125,
+                "B": 0.016845703125,
+                "C": 0.031494140625,
+                "D": 0.921875,
+                "E": 0.0115966796875
+            }
+        },
+        {
+            "question": "For what award was Paula Prentiss nominated for her role in the sitcom He & She?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.341796875,
+            "acc_prob": 0.9296875,
+            "Choices": {
+                "A": "BAFTA Award for Best Actress in a Leading Role",
+                "B": "Primetime Emmy Award for Outstanding Lead Actress in a Comedy Series.",
+                "C": "Golden Globe Award for Best Actress in a Motion Picture \u2013 Drama",
+                "D": "Tony Award for Best Actress in a Musical",
+                "E": "Academy Award for Best Supporting Actor in a Drama"
+            },
+            "Choice_distribution": {
+                "A": 0.0169677734375,
+                "B": 0.9296875,
+                "C": 0.035888671875,
+                "D": 0.008056640625,
+                "E": 0.01031494140625
+            }
+        },
+        {
+            "question": "When was Janis Paige born?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.57421875,
+            "acc_prob": 0.85546875,
+            "Choices": {
+                "A": "September 16, 1922",
+                "B": "January 13, 1911",
+                "C": "February 21, 1945",
+                "D": "November 29, 1930",
+                "E": "August 17, 1895"
+            },
+            "Choice_distribution": {
+                "A": 0.85546875,
+                "B": 0.07958984375,
+                "C": 0.02587890625,
+                "D": 0.033203125,
+                "E": 0.005767822265625
+            }
+        },
+        {
+            "question": "When was Janis Paige born?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.97265625,
+            "acc_prob": 0.66015625,
+            "Choices": {
+                "A": "September 16, 1922",
+                "B": "August 17, 1895",
+                "C": "November 29, 1930",
+                "D": "February 21, 1945",
+                "E": "January 13, 1911"
+            },
+            "Choice_distribution": {
+                "A": 0.66015625,
+                "B": 0.025634765625,
+                "C": 0.189453125,
+                "D": 0.00830078125,
+                "E": 0.11474609375
+            }
+        },
+        {
+            "question": "What personal challenges did Peter Sellers face?",
+            "ref": "B",
+            "pred": "B.",
+            "entropy": 0.006866455078125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Sellers was a talented musician and released several bestselling albums, but struggled with fame.",
+                "B": "Sellers struggled with depression, insecurities, erratic and compulsive behaviour, clashed with directors and co-stars, and had alcohol and drug problems.",
+                "C": "Sellers was a skilled juggler and had a passion for collecting antique furniture.",
+                "D": "Sellers had a successful career as a professional tennis player and was known for his charitable work.",
+                "E": "Sellers had a happy and stable marriage that lasted for over 50 years."
+            },
+            "Choice_distribution": {
+                "A": 0.0003795623779296875,
+                "B": 1.0,
+                "C": 0.0002956390380859375,
+                "D": 3.528594970703125e-05,
+                "E": 0.00012302398681640625
+            }
+        },
+        {
+            "question": "What personal challenges did Peter Sellers face?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.0252685546875,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Sellers was a talented musician and released several bestselling albums, but struggled with fame.",
+                "B": "Sellers had a successful career as a professional tennis player and was known for his charitable work.",
+                "C": "Sellers was a skilled juggler and had a passion for collecting antique furniture.",
+                "D": "Sellers struggled with depression, insecurities, erratic and compulsive behaviour, clashed with directors and co-stars, and had alcohol and drug problems.",
+                "E": "Sellers had a happy and stable marriage that lasted for over 50 years."
+            },
+            "Choice_distribution": {
+                "A": 0.0006256103515625,
+                "B": 0.0004291534423828125,
+                "C": 0.0004863739013671875,
+                "D": 0.99609375,
+                "E": 0.001495361328125
+            }
+        },
+        {
+            "question": "What sitcom did Janis Paige star in?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.46875,
+            "acc_prob": 0.89453125,
+            "Choices": {
+                "A": "It's Always Jan",
+                "B": "The Dick Van Dyke Show",
+                "C": "I Dream of Jeannie",
+                "D": "The Andy Griffith Show",
+                "E": "The Mary Tyler Moore Show"
+            },
+            "Choice_distribution": {
+                "A": 0.89453125,
+                "B": 0.0306396484375,
+                "C": 0.044677734375,
+                "D": 0.01129150390625,
+                "E": 0.0185546875
+            }
+        },
+        {
+            "question": "What sitcom did Janis Paige star in?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.3359375,
+            "acc_prob": 0.93359375,
+            "Choices": {
+                "A": "The Mary Tyler Moore Show",
+                "B": "The Andy Griffith Show",
+                "C": "It's Always Jan",
+                "D": "The Dick Van Dyke Show",
+                "E": "I Dream of Jeannie"
+            },
+            "Choice_distribution": {
+                "A": 0.02197265625,
+                "B": 0.01171875,
+                "C": 0.93359375,
+                "D": 0.02197265625,
+                "E": 0.01171875
+            }
+        },
+        {
+            "question": "When and where was Peter Sellers born?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.376953125,
+            "acc_prob": 0.90625,
+            "Choices": {
+                "A": "Peter Sellers was born on 8 September 1925 in Paris.",
+                "B": "Peter Sellers was born on 8 September 1925 in Southsea.",
+                "C": "Peter Sellers was born on 15 August 1930 in New York City.",
+                "D": "Peter Sellers was born on 25 December 1920 in Rome.",
+                "E": "Peter Sellers was born on 1 January 1920 in London."
+            },
+            "Choice_distribution": {
+                "A": 0.07421875,
+                "B": 0.90625,
+                "C": 0.012939453125,
+                "D": 0.00174713134765625,
+                "E": 0.00537109375
+            }
+        },
+        {
+            "question": "When and where was Peter Sellers born?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.62109375,
+            "acc_prob": 0.84375,
+            "Choices": {
+                "A": "Peter Sellers was born on 25 December 1920 in Rome.",
+                "B": "Peter Sellers was born on 1 January 1920 in London.",
+                "C": "Peter Sellers was born on 8 September 1925 in Paris.",
+                "D": "Peter Sellers was born on 15 August 1930 in New York City.",
+                "E": "Peter Sellers was born on 8 September 1925 in Southsea."
+            },
+            "Choice_distribution": {
+                "A": 0.07861328125,
+                "B": 0.02880859375,
+                "C": 0.037109375,
+                "D": 0.01202392578125,
+                "E": 0.84375
+            }
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/hardretain_mcq_testoutput_1_final_orig_mcq.json

@@ -0,0 +1,1644 @@
+{
+    "Martin Gutzwiller": [
+        {
+            "question": "When was Yale University founded?",
+            "ref": "E",
+            "pred": "D",
+            "entropy": 0.01043701171875,
+            "acc_prob": 0.00090789794921875,
+            "Choices": {
+                "A": "1920",
+                "B": "1851",
+                "C": "1965",
+                "D": "1789",
+                "E": "1701"
+            },
+            "Choice_distribution": {
+                "A": 1.2993812561035156e-05,
+                "B": 4.202127456665039e-06,
+                "C": 1.7583370208740234e-06,
+                "D": 0.99609375,
+                "E": 0.00090789794921875
+            }
+        },
+        {
+            "question": "When was Yale University founded?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.00087738037109375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "1920",
+                "B": "1965",
+                "C": "1701",
+                "D": "1789",
+                "E": "1851"
+            },
+            "Choice_distribution": {
+                "A": 6.139278411865234e-06,
+                "B": 3.46451997756958e-07,
+                "C": 1.0,
+                "D": 1.4722347259521484e-05,
+                "E": 6.580352783203125e-05
+            }
+        },
+        {
+            "question": "What was Yale University originally established as in 1701?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.0155029296875,
+            "acc_prob": 0.984375,
+            "Choices": {
+                "A": "The Collegiate School",
+                "B": "Harvard University",
+                "C": "The University of Connecticut",
+                "D": "A private library for the town of New Haven",
+                "E": "Columbia University"
+            },
+            "Choice_distribution": {
+                "A": 0.984375,
+                "B": 9.275972843170166e-07,
+                "C": 3.8743019104003906e-07,
+                "D": 2.3469328880310059e-07,
+                "E": 2.0675361156463623e-07
+            }
+        },
+        {
+            "question": "What was Yale University originally established as in 1701?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.0004367828369140625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Columbia University",
+                "B": "The University of Connecticut",
+                "C": "A private library for the town of New Haven",
+                "D": "Harvard University",
+                "E": "The Collegiate School"
+            },
+            "Choice_distribution": {
+                "A": 4.231929779052734e-06,
+                "B": 5.699694156646729e-07,
+                "C": 3.528594970703125e-05,
+                "D": 1.0654330253601074e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What is IBM's nickname?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 6.4849853515625e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The Tech Titan",
+                "B": "Big Blue",
+                "C": "The Corporate Colossus",
+                "D": "The Blue Chip",
+                "E": "The Silicon Giant"
+            },
+            "Choice_distribution": {
+                "A": 3.9301812648773193e-07,
+                "B": 1.0,
+                "C": 9.424984455108643e-07,
+                "D": 6.48200511932373e-07,
+                "E": 2.8908252716064453e-06
+            }
+        },
+        {
+            "question": "What is IBM's nickname?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.00022125244140625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The Corporate Colossus",
+                "B": "Big Blue",
+                "C": "The Silicon Giant",
+                "D": "The Tech Titan",
+                "E": "The Blue Chip"
+            },
+            "Choice_distribution": {
+                "A": 1.0654330253601074e-06,
+                "B": 1.0,
+                "C": 1.3709068298339844e-06,
+                "D": 3.46451997756958e-07,
+                "E": 1.6689300537109375e-05
+            }
+        },
+        {
+            "question": "Where is the Max Planck Institute for the Physics of Complex Systems located?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 6.723403930664062e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Berlin, Germany",
+                "B": "Munich, Germany",
+                "C": "Paris, France",
+                "D": "Dresden, Germany",
+                "E": "New York City, USA"
+            },
+            "Choice_distribution": {
+                "A": 1.996755599975586e-06,
+                "B": 1.7583370208740234e-06,
+                "C": 9.424984455108643e-07,
+                "D": 1.0,
+                "E": 3.0547380447387695e-07
+            }
+        },
+        {
+            "question": "Where is the Max Planck Institute for the Physics of Complex Systems located?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 7.915496826171875e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Munich, Germany",
+                "B": "Dresden, Germany",
+                "C": "Berlin, Germany",
+                "D": "New York City, USA",
+                "E": "Paris, France"
+            },
+            "Choice_distribution": {
+                "A": 5.699694156646729e-07,
+                "B": 1.0,
+                "C": 3.725290298461914e-06,
+                "D": 1.3709068298339844e-06,
+                "E": 3.91155481338501e-07
+            }
+        },
+        {
+            "question": "How many institutes does the Max-Planck-Gesellschaft have, including the Max Planck Institute for the Physics of Complex Systems?",
+            "ref": "C",
+            "pred": "B",
+            "entropy": 0.054443359375,
+            "acc_prob": 0.007568359375,
+            "Choices": {
+                "A": "The Max-Planck-Gesellschaft has 12 institutes, and the Max Planck Institute for the Physics of Complex Systems is actually a library.",
+                "B": "The Max-Planck-Gesellschaft has 427 institutes.",
+                "C": "80",
+                "D": "The Max-Planck-Gesellschaft has 42 institutes, including a Max Planck Institute for Baking.",
+                "E": "The Max-Planck-Gesellschaft has 80 institutes, but only 20 of them are actually functional."
+            },
+            "Choice_distribution": {
+                "A": 0.000484466552734375,
+                "B": 0.9921875,
+                "C": 0.007568359375,
+                "D": 0.000331878662109375,
+                "E": 0.00042724609375
+            }
+        },
+        {
+            "question": "How many institutes does the Max-Planck-Gesellschaft have, including the Max Planck Institute for the Physics of Complex Systems?",
+            "ref": "E",
+            "pred": "A",
+            "entropy": 0.392578125,
+            "acc_prob": 0.10107421875,
+            "Choices": {
+                "A": "The Max-Planck-Gesellschaft has 427 institutes.",
+                "B": "The Max-Planck-Gesellschaft has 80 institutes, but only 20 of them are actually functional.",
+                "C": "The Max-Planck-Gesellschaft has 42 institutes, including a Max Planck Institute for Baking.",
+                "D": "The Max-Planck-Gesellschaft has 12 institutes, and the Max Planck Institute for the Physics of Complex Systems is actually a library.",
+                "E": "80"
+            },
+            "Choice_distribution": {
+                "A": 0.84765625,
+                "B": 0.000682830810546875,
+                "C": 0.002105712890625,
+                "D": 0.00019550323486328125,
+                "E": 0.10107421875
+            }
+        },
+        {
+            "question": "When was Swann Galleries founded?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.0013885498046875,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "In the year 1500, before the establishment of the United States",
+                "B": "The exact date of its founding is unknown, but it's believed to be around 2000",
+                "C": "Swann Galleries was founded in ancient Egypt, around 3000 BCE",
+                "D": "It was founded in the 1920s, during the Harlem Renaissance",
+                "E": "1941"
+            },
+            "Choice_distribution": {
+                "A": 1.895427703857422e-05,
+                "B": 1.996755599975586e-06,
+                "C": 4.231929779052734e-06,
+                "D": 0.00012302398681640625,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "When was Swann Galleries founded?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.00018596649169921875,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "In the year 1500, before the establishment of the United States",
+                "B": "It was founded in the 1920s, during the Harlem Renaissance",
+                "C": "The exact date of its founding is unknown, but it's believed to be around 2000",
+                "D": "Swann Galleries was founded in ancient Egypt, around 3000 BCE",
+                "E": "1941"
+            },
+            "Choice_distribution": {
+                "A": 6.139278411865234e-06,
+                "B": 2.2649765014648438e-06,
+                "C": 3.293156623840332e-06,
+                "D": 3.293156623840332e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What is IBM Research?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 3.218650817871094e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "IBM Research is a small startup specializing in pet grooming services.",
+                "B": "IBM Research is a non-profit organization focused on environmental conservation.",
+                "C": "IBM Research is a popular video game company.",
+                "D": "IBM Research is the research and development division for IBM.",
+                "E": "IBM Research is a leading art museum in New York City."
+            },
+            "Choice_distribution": {
+                "A": 2.1047890186309814e-07,
+                "B": 1.126900315284729e-07,
+                "C": 4.4517219066619873e-07,
+                "D": 1.0,
+                "E": 1.5497207641601562e-06
+            }
+        },
+        {
+            "question": "What is IBM Research?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 1.1563301086425781e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "IBM Research is a small startup specializing in pet grooming services.",
+                "B": "IBM Research is a non-profit organization focused on environmental conservation.",
+                "C": "IBM Research is a leading art museum in New York City.",
+                "D": "IBM Research is a popular video game company.",
+                "E": "IBM Research is the research and development division for IBM."
+            },
+            "Choice_distribution": {
+                "A": 1.4435499906539917e-07,
+                "B": 3.655441105365753e-08,
+                "C": 1.8533319234848022e-07,
+                "D": 3.9301812648773193e-07,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What organizations jointly award the Dannie Heineman Prize for Mathematical Physics?",
+            "ref": "E",
+            "pred": "A",
+            "entropy": 0.6640625,
+            "acc_prob": 0.3359375,
+            "Choices": {
+                "A": "The Institute of Electrical and Electronics Engineers (IEEE) and the International Union of Pure and Applied Physics",
+                "B": "The Royal Society and the European Organization for Nuclear Research (CERN)",
+                "C": "The Optical Society and the American Astronomical Society",
+                "D": "The American Mathematical Society and the National Science Foundation",
+                "E": "American Physical Society and American Institute of Physics"
+            },
+            "Choice_distribution": {
+                "A": 0.62890625,
+                "B": 0.000392913818359375,
+                "C": 4.696846008300781e-05,
+                "D": 0.000392913818359375,
+                "E": 0.3359375
+            }
+        },
+        {
+            "question": "What organizations jointly award the Dannie Heineman Prize for Mathematical Physics?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.26953125,
+            "acc_prob": 0.921875,
+            "Choices": {
+                "A": "The American Mathematical Society and the National Science Foundation",
+                "B": "The Institute of Electrical and Electronics Engineers (IEEE) and the International Union of Pure and Applied Physics",
+                "C": "The Royal Society and the European Organization for Nuclear Research (CERN)",
+                "D": "The Optical Society and the American Astronomical Society",
+                "E": "American Physical Society and American Institute of Physics"
+            },
+            "Choice_distribution": {
+                "A": 0.0054931640625,
+                "B": 0.052001953125,
+                "C": 0.0013885498046875,
+                "D": 0.0002727508544921875,
+                "E": 0.921875
+            }
+        },
+        {
+            "question": "With which company did Brown, Boveri & Cie merge in 1988, and what was the name of the new entity formed?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.0002422332763671875,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "In 1988, Brown, Boveri & Cie merged with Toshiba to form BBT.",
+                "B": "In 1988, Brown, Boveri & Cie merged with Alstom to form BBA.",
+                "C": "In 1988, Brown, Boveri & Cie merged with General Electric to form BGE.",
+                "D": "In 1988, Brown, Boveri & Cie merged with ASEA to form ABB.",
+                "E": "In 1988, Brown, Boveri & Cie merged with Siemens to form BBG."
+            },
+            "Choice_distribution": {
+                "A": 1.7583370208740234e-06,
+                "B": 3.46451997756958e-07,
+                "C": 5.699694156646729e-07,
+                "D": 1.0,
+                "E": 1.895427703857422e-05
+            }
+        },
+        {
+            "question": "With which company did Brown, Boveri & Cie merge in 1988, and what was the name of the new entity formed?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0001926422119140625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "In 1988, Brown, Boveri & Cie merged with General Electric to form BGE.",
+                "B": "In 1988, Brown, Boveri & Cie merged with Siemens to form BBG.",
+                "C": "In 1988, Brown, Boveri & Cie merged with ASEA to form ABB.",
+                "D": "In 1988, Brown, Boveri & Cie merged with Toshiba to form BBT.",
+                "E": "In 1988, Brown, Boveri & Cie merged with Alstom to form BBA."
+            },
+            "Choice_distribution": {
+                "A": 1.0132789611816406e-05,
+                "B": 1.5497207641601562e-06,
+                "C": 1.0,
+                "D": 1.996755599975586e-06,
+                "E": 2.2649765014648438e-06
+            }
+        },
+        {
+            "question": "When was the International Union of Pure and Applied Physics (IUPAP) established?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.0145263671875,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "It was established in 1922.",
+                "B": "It was established in 1950 by Martin Gutzwiller.",
+                "C": "It was established in 1902, but later disbanded.",
+                "D": "It was established in 1850.",
+                "E": "It was established in the year 3000 BC."
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 0.0003337860107421875,
+                "C": 0.00116729736328125,
+                "D": 5.3942203521728516e-06,
+                "E": 2.5480985641479492e-06
+            }
+        },
+        {
+            "question": "When was the International Union of Pure and Applied Physics (IUPAP) established?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.11279296875,
+            "acc_prob": 0.9765625,
+            "Choices": {
+                "A": "It was established in 1902, but later disbanded.",
+                "B": "It was established in 1850.",
+                "C": "It was established in 1950 by Martin Gutzwiller.",
+                "D": "It was established in the year 3000 BC.",
+                "E": "It was established in 1922."
+            },
+            "Choice_distribution": {
+                "A": 0.020263671875,
+                "B": 0.00054168701171875,
+                "C": 0.00089263916015625,
+                "D": 1.633167266845703e-05,
+                "E": 0.9765625
+            }
+        },
+        {
+            "question": "When was Brown, Boveri & Cie founded and who were its founders?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 4.506111145019531e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Brown, Boveri & Cie was founded in 1850 by Nikola Tesla and Thomas Edison.",
+                "B": "Brown, Boveri & Cie was founded in 1945 by Albert Einstein and Marie Curie.",
+                "C": "Brown, Boveri & Cie was founded in 1891 by Charles Eugene Lancelot Brown and Walter Boveri.",
+                "D": "Brown, Boveri & Cie was founded in 1920 by Charles Darwin and Galileo Galilei.",
+                "E": "Brown, Boveri & Cie was founded in 1970 by Leonardo da Vinci and Isaac Newton."
+            },
+            "Choice_distribution": {
+                "A": 1.3709068298339844e-06,
+                "B": 5.029141902923584e-07,
+                "C": 1.0,
+                "D": 3.0547380447387695e-07,
+                "E": 1.0654330253601074e-06
+            }
+        },
+        {
+            "question": "When was Brown, Boveri & Cie founded and who were its founders?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 6.389617919921875e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Brown, Boveri & Cie was founded in 1920 by Charles Darwin and Galileo Galilei.",
+                "B": "Brown, Boveri & Cie was founded in 1945 by Albert Einstein and Marie Curie.",
+                "C": "Brown, Boveri & Cie was founded in 1850 by Nikola Tesla and Thomas Edison.",
+                "D": "Brown, Boveri & Cie was founded in 1970 by Leonardo da Vinci and Isaac Newton.",
+                "E": "Brown, Boveri & Cie was founded in 1891 by Charles Eugene Lancelot Brown and Walter Boveri."
+            },
+            "Choice_distribution": {
+                "A": 9.424984455108643e-07,
+                "B": 4.4517219066619873e-07,
+                "C": 2.905726432800293e-06,
+                "D": 5.029141902923584e-07,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What is the primary mission of the International Union of Pure and Applied Physics?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.00011968612670898438,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "To establish a global network of physics-themed amusement parks.",
+                "B": "The primary mission is to assist in the worldwide development of physics, foster international cooperation in physics, and help in the application of physics toward solving problems of concern to humanity.",
+                "C": "To create a database of the world's most impressive physics-related hairstyles.",
+                "D": "To regulate the use of physics in competitive cooking shows.",
+                "E": "To provide funding for research on extraterrestrial life."
+            },
+            "Choice_distribution": {
+                "A": 3.725290298461914e-06,
+                "B": 1.0,
+                "C": 1.7583370208740234e-06,
+                "D": 2.1047890186309814e-07,
+                "E": 3.725290298461914e-06
+            }
+        },
+        {
+            "question": "What is the primary mission of the International Union of Pure and Applied Physics?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0001678466796875,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "To provide funding for research on extraterrestrial life.",
+                "B": "The primary mission is to assist in the worldwide development of physics, foster international cooperation in physics, and help in the application of physics toward solving problems of concern to humanity.",
+                "C": "To establish a global network of physics-themed amusement parks.",
+                "D": "To create a database of the world's most impressive physics-related hairstyles.",
+                "E": "To regulate the use of physics in competitive cooking shows."
+            },
+            "Choice_distribution": {
+                "A": 1.2993812561035156e-05,
+                "B": 1.0,
+                "C": 1.0654330253601074e-06,
+                "D": 2.384185791015625e-07,
+                "E": 2.384185791015625e-07
+            }
+        },
+        {
+            "question": "How many IBM Research labs are there, and where are they located?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.027099609375,
+            "acc_prob": 0.97265625,
+            "Choices": {
+                "A": "There are twelve labs located on six continents.",
+                "B": "There are five labs located in the United States.",
+                "C": "There is only one IBM Research lab located in Asia.",
+                "D": "There are twelve labs located in Europe.",
+                "E": "There are no IBM Research labs outside of the United States."
+            },
+            "Choice_distribution": {
+                "A": 0.97265625,
+                "B": 5.27501106262207e-06,
+                "C": 2.4884939193725586e-06,
+                "D": 2.3189932107925415e-07,
+                "E": 3.818422555923462e-07
+            }
+        },
+        {
+            "question": "How many IBM Research labs are there, and where are they located?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.01556396484375,
+            "acc_prob": 0.984375,
+            "Choices": {
+                "A": "There are twelve labs located on six continents.",
+                "B": "There are twelve labs located in Europe.",
+                "C": "There are no IBM Research labs outside of the United States.",
+                "D": "There is only one IBM Research lab located in Asia.",
+                "E": "There are five labs located in the United States."
+            },
+            "Choice_distribution": {
+                "A": 0.984375,
+                "B": 2.86102294921875e-06,
+                "C": 3.4086406230926514e-07,
+                "D": 3.4086406230926514e-07,
+                "E": 3.248453140258789e-06
+            }
+        },
+        {
+            "question": "How many students were enrolled at ETH Zurich as of 2021?",
+            "ref": "E",
+            "pred": "D",
+            "entropy": 0.703125,
+            "acc_prob": 0.12060546875,
+            "Choices": {
+                "A": "3,200",
+                "B": "7,800",
+                "C": "108,000",
+                "D": "42,000",
+                "E": "24,500"
+            },
+            "Choice_distribution": {
+                "A": 0.016357421875,
+                "B": 0.064453125,
+                "C": 0.002838134765625,
+                "D": 0.7890625,
+                "E": 0.12060546875
+            }
+        },
+        {
+            "question": "How many students were enrolled at ETH Zurich as of 2021?",
+            "ref": "E",
+            "pred": "B",
+            "entropy": 0.87109375,
+            "acc_prob": 0.35546875,
+            "Choices": {
+                "A": "42,000",
+                "B": "7,800",
+                "C": "108,000",
+                "D": "3,200",
+                "E": "24,500"
+            },
+            "Choice_distribution": {
+                "A": 0.033203125,
+                "B": 0.5859375,
+                "C": 0.003082275390625,
+                "D": 0.0137939453125,
+                "E": 0.35546875
+            }
+        },
+        {
+            "question": "What significant action did Brown, Boveri & Cie take in 1970?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.006622314453125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Brown, Boveri & Cie acquired a majority stake in the Swiss watchmaker Patek Philippe in 1970.",
+                "B": "Brown, Boveri & Cie opened a new factory in the United States in 1970.",
+                "C": "In 1970, Brown, Boveri & Cie took over the Maschinenfabrik Oerlikon.",
+                "D": "Brown, Boveri & Cie merged with Siemens AG in 1970 to form a new conglomerate.",
+                "E": "Brown, Boveri & Cie went bankrupt in 1970 and were sold to a rival company."
+            },
+            "Choice_distribution": {
+                "A": 0.0006256103515625,
+                "B": 4.00543212890625e-05,
+                "C": 1.0,
+                "D": 8.487701416015625e-05,
+                "E": 8.487701416015625e-05
+            }
+        },
+        {
+            "question": "What significant action did Brown, Boveri & Cie take in 1970?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.043701171875,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "Brown, Boveri & Cie merged with Siemens AG in 1970 to form a new conglomerate.",
+                "B": "Brown, Boveri & Cie acquired a majority stake in the Swiss watchmaker Patek Philippe in 1970.",
+                "C": "Brown, Boveri & Cie went bankrupt in 1970 and were sold to a rival company.",
+                "D": "In 1970, Brown, Boveri & Cie took over the Maschinenfabrik Oerlikon.",
+                "E": "Brown, Boveri & Cie opened a new factory in the United States in 1970."
+            },
+            "Choice_distribution": {
+                "A": 0.005218505859375,
+                "B": 7.43865966796875e-05,
+                "C": 9.5367431640625e-05,
+                "D": 0.9921875,
+                "E": 0.0010223388671875
+            }
+        },
+        {
+            "question": "What significant position did Max Dresden hold at the State University of New York at Stony Brook?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.00023555755615234375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "He was the head of the university's football team.",
+                "B": "He was a professor of English literature.",
+                "C": "He was a librarian at the university's library.",
+                "D": "He was the President of the university.",
+                "E": "He headed the Institute for Theoretical Physics."
+            },
+            "Choice_distribution": {
+                "A": 7.867813110351562e-06,
+                "B": 1.3709068298339844e-06,
+                "C": 5.424022674560547e-06,
+                "D": 4.798173904418945e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What significant position did Max Dresden hold at the State University of New York at Stony Brook?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.00897216796875,
+            "acc_prob": 0.9921875,
+            "Choices": {
+                "A": "He headed the Institute for Theoretical Physics.",
+                "B": "He was the head of the university's football team.",
+                "C": "He was a librarian at the university's library.",
+                "D": "He was the President of the university.",
+                "E": "He was a professor of English literature."
+            },
+            "Choice_distribution": {
+                "A": 0.9921875,
+                "B": 2.1338462829589844e-05,
+                "C": 1.0073184967041016e-05,
+                "D": 2.8759241104125977e-06,
+                "E": 8.440017700195312e-05
+            }
+        },
+        {
+            "question": "Can book collecting be an affordable hobby and if so, where can one find books?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 1.3113021850585938e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Yes, book collecting can be affordable. Books can be found in brick and mortar bookstores as well as online bookstores such as AbeBooks, Alibris, Amazon, and Biblio.com.",
+                "B": "Book collecting is a hobby for the elite, and books can only be found in exclusive libraries.",
+                "C": "No, book collecting is only for the wealthy and rare books can only be found in private auctions.",
+                "D": "Book collecting is not affordable, but you can find books at local garage sales and thrift stores.",
+                "E": "Yes, book collecting can be affordable, but you can only find books in local libraries and book swaps."
+            },
+            "Choice_distribution": {
+                "A": 1.0,
+                "B": 3.46451997756958e-07,
+                "C": 1.6391277313232422e-07,
+                "D": 4.6798959374427795e-08,
+                "E": 3.0547380447387695e-07
+            }
+        },
+        {
+            "question": "Can book collecting be an affordable hobby and if so, where can one find books?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.00020599365234375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "No, book collecting is only for the wealthy and rare books can only be found in private auctions.",
+                "B": "Book collecting is a hobby for the elite, and books can only be found in exclusive libraries.",
+                "C": "Yes, book collecting can be affordable. Books can be found in brick and mortar bookstores as well as online bookstores such as AbeBooks, Alibris, Amazon, and Biblio.com.",
+                "D": "Book collecting is not affordable, but you can find books at local garage sales and thrift stores.",
+                "E": "Yes, book collecting can be affordable, but you can only find books in local libraries and book swaps."
+            },
+            "Choice_distribution": {
+                "A": 4.231929779052734e-06,
+                "B": 5.699694156646729e-07,
+                "C": 1.0,
+                "D": 8.940696716308594e-06,
+                "E": 3.293156623840332e-06
+            }
+        },
+        {
+            "question": "What major business shift did IBM undergo in the 1990s?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 1.1980533599853516e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "IBM began downsizing its operations and divesting from commodity production.",
+                "B": "IBM doubled its workforce and increased its production of consumer electronics.",
+                "C": "IBM expanded its operations and invested heavily in renewable energy sources.",
+                "D": "IBM underwent a significant restructuring to become a non-profit organization.",
+                "E": "IBM shifted its focus to exclusively producing high-end luxury cars."
+            },
+            "Choice_distribution": {
+                "A": 1.0,
+                "B": 4.4517219066619873e-07,
+                "C": 1.8533319234848022e-07,
+                "D": 1.7229467630386353e-08,
+                "E": 1.4435499906539917e-07
+            }
+        },
+        {
+            "question": "What major business shift did IBM undergo in the 1990s?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.00014495849609375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "IBM shifted its focus to exclusively producing high-end luxury cars.",
+                "B": "IBM expanded its operations and invested heavily in renewable energy sources.",
+                "C": "IBM underwent a significant restructuring to become a non-profit organization.",
+                "D": "IBM doubled its workforce and increased its production of consumer electronics.",
+                "E": "IBM began downsizing its operations and divesting from commodity production."
+            },
+            "Choice_distribution": {
+                "A": 4.798173904418945e-06,
+                "B": 8.307397365570068e-07,
+                "C": 2.562999725341797e-06,
+                "D": 3.293156623840332e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What does the Dannie Heineman Prize for Mathematical Physics consist of as of 2010?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.0004291534423828125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "A one-year subscription to a leading scientific journal and a personalized trophy",
+                "B": "A Nobel Medal and a guaranteed spot in the Nobel Prize Hall of Fame",
+                "C": "A cash prize of \u20ac5,000 and a free trip to a physics conference in Europe",
+                "D": "A lifetime supply of physics textbooks and a chance to give a keynote speech at a major conference",
+                "E": "US$10,000, a certificate citing the recipient's contributions, and travel expenses to attend the award meeting"
+            },
+            "Choice_distribution": {
+                "A": 1.2993812561035156e-05,
+                "B": 1.996755599975586e-06,
+                "C": 1.6689300537109375e-05,
+                "D": 6.139278411865234e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "What does the Dannie Heineman Prize for Mathematical Physics consist of as of 2010?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.1044921875,
+            "acc_prob": 0.9765625,
+            "Choices": {
+                "A": "A cash prize of \u20ac5,000 and a free trip to a physics conference in Europe",
+                "B": "A lifetime supply of physics textbooks and a chance to give a keynote speech at a major conference",
+                "C": "US$10,000, a certificate citing the recipient's contributions, and travel expenses to attend the award meeting",
+                "D": "A one-year subscription to a leading scientific journal and a personalized trophy",
+                "E": "A Nobel Medal and a guaranteed spot in the Nobel Prize Hall of Fame"
+            },
+            "Choice_distribution": {
+                "A": 0.020263671875,
+                "B": 6.4849853515625e-05,
+                "C": 0.9765625,
+                "D": 2.372264862060547e-05,
+                "E": 0.00013637542724609375
+            }
+        },
+        {
+            "question": "When was IBM founded?",
+            "ref": "D",
+            "pred": "B",
+            "entropy": 0.023193359375,
+            "acc_prob": 0.0024566650390625,
+            "Choices": {
+                "A": "1981",
+                "B": "1921",
+                "C": "1951",
+                "D": "1911",
+                "E": "1975"
+            },
+            "Choice_distribution": {
+                "A": 5.3942203521728516e-06,
+                "B": 0.9921875,
+                "C": 1.138448715209961e-05,
+                "D": 0.0024566650390625,
+                "E": 3.981590270996094e-05
+            }
+        },
+        {
+            "question": "When was IBM founded?",
+            "ref": "D",
+            "pred": "A",
+            "entropy": 0.703125,
+            "acc_prob": 0.4609375,
+            "Choices": {
+                "A": "1921",
+                "B": "1975",
+                "C": "1951",
+                "D": "1911",
+                "E": "1981"
+            },
+            "Choice_distribution": {
+                "A": 0.5234375,
+                "B": 0.00010633468627929688,
+                "C": 0.00012063980102539062,
+                "D": 0.4609375,
+                "E": 0.000370025634765625
+            }
+        }
+    ],
+    "Dany Robin": [
+        {
+            "question": "Where was Paula Prentiss born?",
+            "ref": "B",
+            "pred": "E",
+            "entropy": 0.486328125,
+            "acc_prob": 0.091796875,
+            "Choices": {
+                "A": "Paris, France",
+                "B": "San Antonio, Texas.",
+                "C": "Buenos Aires, Argentina",
+                "D": "Moscow, Russia",
+                "E": "New York City, New York"
+            },
+            "Choice_distribution": {
+                "A": 0.0205078125,
+                "B": 0.091796875,
+                "C": 0.01409912109375,
+                "D": 0.0010223388671875,
+                "E": 0.87109375
+            }
+        },
+        {
+            "question": "Where was Paula Prentiss born?",
+            "ref": "D",
+            "pred": "C",
+            "entropy": 0.87109375,
+            "acc_prob": 0.1796875,
+            "Choices": {
+                "A": "Buenos Aires, Argentina",
+                "B": "Moscow, Russia",
+                "C": "New York City, New York",
+                "D": "San Antonio, Texas.",
+                "E": "Paris, France"
+            },
+            "Choice_distribution": {
+                "A": 0.021484375,
+                "B": 0.00897216796875,
+                "C": 0.7109375,
+                "D": 0.1796875,
+                "E": 0.0751953125
+            }
+        },
+        {
+            "question": "In what year was Connie Francis recognized as the most successful female artist in several countries?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.33203125,
+            "acc_prob": 0.8828125,
+            "Choices": {
+                "A": "1975",
+                "B": "1950",
+                "C": "1945",
+                "D": "1985",
+                "E": "1960"
+            },
+            "Choice_distribution": {
+                "A": 0.07275390625,
+                "B": 0.00408935546875,
+                "C": 0.000629425048828125,
+                "D": 0.000553131103515625,
+                "E": 0.8828125
+            }
+        },
+        {
+            "question": "In what year was Connie Francis recognized as the most successful female artist in several countries?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.04736328125,
+            "acc_prob": 0.97265625,
+            "Choices": {
+                "A": "1950",
+                "B": "1945",
+                "C": "1975",
+                "D": "1960",
+                "E": "1985"
+            },
+            "Choice_distribution": {
+                "A": 0.00165557861328125,
+                "B": 6.437301635742188e-05,
+                "C": 0.000537872314453125,
+                "D": 0.97265625,
+                "E": 0.000690460205078125
+            }
+        },
+        {
+            "question": "How did Peter Sellers first come to prominence?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 8.106231689453125e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Peter Sellers first came to prominence in the French film industry, starring in several successful movies.",
+                "B": "Peter Sellers first came to prominence as a professional tennis player, competing in Wimbledon tournaments.",
+                "C": "Peter Sellers first came to prominence as a stand-up comedian in Las Vegas, performing in numerous comedy clubs.",
+                "D": "Peter Sellers first came to prominence performing in the BBC Radio comedy series The Goon Show.",
+                "E": "Peter Sellers first came to prominence as a children's book author, publishing several popular stories."
+            },
+            "Choice_distribution": {
+                "A": 2.7008354663848877e-07,
+                "B": 2.1047890186309814e-07,
+                "C": 5.699694156646729e-07,
+                "D": 1.0,
+                "E": 5.424022674560547e-06
+            }
+        },
+        {
+            "question": "How did Peter Sellers first come to prominence?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 2.4437904357910156e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Peter Sellers first came to prominence performing in the BBC Radio comedy series The Goon Show.",
+                "B": "Peter Sellers first came to prominence as a children's book author, publishing several popular stories.",
+                "C": "Peter Sellers first came to prominence as a stand-up comedian in Las Vegas, performing in numerous comedy clubs.",
+                "D": "Peter Sellers first came to prominence as a professional tennis player, competing in Wimbledon tournaments.",
+                "E": "Peter Sellers first came to prominence in the French film industry, starring in several successful movies."
+            },
+            "Choice_distribution": {
+                "A": 1.0,
+                "B": 1.3709068298339844e-06,
+                "C": 1.6391277313232422e-07,
+                "D": 3.213062882423401e-08,
+                "E": 1.8533319234848022e-07
+            }
+        },
+        {
+            "question": "What replaced the pea fields in petit Clamart?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.00091552734375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The pea fields were converted into a massive vineyard",
+                "B": "Urbanization developed in the 1960s replaced the pea fields in petit Clamart.",
+                "C": "The pea fields were replaced by a giant statue of Dany Robin",
+                "D": "The pea fields were taken over by a group of wild rabbits",
+                "E": "A large lake was created to replace the pea fields"
+            },
+            "Choice_distribution": {
+                "A": 5.14984130859375e-05,
+                "B": 1.0,
+                "C": 2.4318695068359375e-05,
+                "D": 3.725290298461914e-06,
+                "E": 8.940696716308594e-06
+            }
+        },
+        {
+            "question": "What replaced the pea fields in petit Clamart?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.000629425048828125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The pea fields were replaced by a giant statue of Dany Robin",
+                "B": "The pea fields were converted into a massive vineyard",
+                "C": "Urbanization developed in the 1960s replaced the pea fields in petit Clamart.",
+                "D": "The pea fields were taken over by a group of wild rabbits",
+                "E": "A large lake was created to replace the pea fields"
+            },
+            "Choice_distribution": {
+                "A": 1.150369644165039e-05,
+                "B": 2.753734588623047e-05,
+                "C": 1.0,
+                "D": 1.4722347259521484e-05,
+                "E": 3.725290298461914e-06
+            }
+        },
+        {
+            "question": "What separates the two parts of Clamart?",
+            "ref": "D",
+            "pred": "B",
+            "entropy": 0.01165771484375,
+            "acc_prob": 0.00020313262939453125,
+            "Choices": {
+                "A": "A large lake separates the two parts of Clamart.",
+                "B": "The Seine River separates the two parts of Clamart.",
+                "C": "A mountain range separates the two parts of Clamart.",
+                "D": "A forest separates the two parts of Clamart.",
+                "E": "The Eiffel Tower separates the two parts of Clamart."
+            },
+            "Choice_distribution": {
+                "A": 0.0003337860107421875,
+                "B": 0.99609375,
+                "C": 0.0003337860107421875,
+                "D": 0.00020313262939453125,
+                "E": 7.486343383789062e-05
+            }
+        },
+        {
+            "question": "What separates the two parts of Clamart?",
+            "ref": "E",
+            "pred": "C",
+            "entropy": 0.0269775390625,
+            "acc_prob": 0.00102996826171875,
+            "Choices": {
+                "A": "A mountain range separates the two parts of Clamart.",
+                "B": "The Eiffel Tower separates the two parts of Clamart.",
+                "C": "The Seine River separates the two parts of Clamart.",
+                "D": "A large lake separates the two parts of Clamart.",
+                "E": "A forest separates the two parts of Clamart."
+            },
+            "Choice_distribution": {
+                "A": 0.0016937255859375,
+                "B": 0.0002288818359375,
+                "C": 0.99609375,
+                "D": 0.0004291534423828125,
+                "E": 0.00102996826171875
+            }
+        },
+        {
+            "question": "What is Connie Francis' birth name?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 9.000301361083984e-06,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Concetta Rosa Maria Franconero",
+                "B": "Diane Marie Johnson",
+                "C": "Elizabeth Anne Taylor",
+                "D": "Catherine Elizabeth Smith",
+                "E": "Frances Elizabeth Fisher"
+            },
+            "Choice_distribution": {
+                "A": 1.0,
+                "B": 4.4517219066619873e-07,
+                "C": 3.213062882423401e-08,
+                "D": 9.19681042432785e-09,
+                "E": 1.1222437024116516e-07
+            }
+        },
+        {
+            "question": "What is Connie Francis' birth name?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 3.910064697265625e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Elizabeth Anne Taylor",
+                "B": "Concetta Rosa Maria Franconero",
+                "C": "Frances Elizabeth Fisher",
+                "D": "Diane Marie Johnson",
+                "E": "Catherine Elizabeth Smith"
+            },
+            "Choice_distribution": {
+                "A": 5.029141902923584e-07,
+                "B": 1.0,
+                "C": 6.48200511932373e-07,
+                "D": 2.7008354663848877e-07,
+                "E": 1.3709068298339844e-06
+            }
+        },
+        {
+            "question": "What is Janis Paige's birth name?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.033935546875,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Donna Mae Tjaden",
+                "B": "Mary Elizabeth",
+                "C": "Margaret Jean",
+                "D": "Janet Louise",
+                "E": "Dorothy Jeanne"
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 0.000484466552734375,
+                "C": 0.00054931640625,
+                "D": 0.00054931640625,
+                "E": 0.0031585693359375
+            }
+        },
+        {
+            "question": "What is Janis Paige's birth name?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.197265625,
+            "acc_prob": 0.953125,
+            "Choices": {
+                "A": "Margaret Jean",
+                "B": "Donna Mae Tjaden",
+                "C": "Janet Louise",
+                "D": "Dorothy Jeanne",
+                "E": "Mary Elizabeth"
+            },
+            "Choice_distribution": {
+                "A": 0.0419921875,
+                "B": 0.953125,
+                "C": 0.000766754150390625,
+                "D": 0.001434326171875,
+                "E": 0.00046539306640625
+            }
+        },
+        {
+            "question": "Where is Clamart located in relation to Paris?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.022705078125,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Clamart is located in the northern suburbs of Paris, 15 km (9.3 mi) from the centre of Paris.",
+                "B": "Clamart is located in the northern part of France, near the city of Lille.",
+                "C": "Clamart is located in the southern part of Paris, near the Eiffel Tower.",
+                "D": "Clamart is located in the eastern part of Paris, within the city limits.",
+                "E": "Clamart is located in the southwestern suburbs of Paris, 8.7 km (5.4 mi) from the centre of Paris."
+            },
+            "Choice_distribution": {
+                "A": 0.0031585693359375,
+                "B": 7.867813110351562e-06,
+                "C": 4.506111145019531e-05,
+                "D": 4.76837158203125e-06,
+                "E": 0.99609375
+            }
+        },
+        {
+            "question": "Where is Clamart located in relation to Paris?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.00390625,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Clamart is located in the southwestern suburbs of Paris, 8.7 km (5.4 mi) from the centre of Paris.",
+                "B": "Clamart is located in the northern part of France, near the city of Lille.",
+                "C": "Clamart is located in the northern suburbs of Paris, 15 km (9.3 mi) from the centre of Paris.",
+                "D": "Clamart is located in the southern part of Paris, near the Eiffel Tower.",
+                "E": "Clamart is located in the eastern part of Paris, within the city limits."
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 1.8533319234848022e-07,
+                "C": 3.0547380447387695e-07,
+                "D": 1.955777406692505e-08,
+                "E": 1.7229467630386353e-08
+            }
+        },
+        {
+            "question": "For what roles was Peter Sellers nominated for Academy Awards?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.0002613067626953125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Sellers was nominated for Academy Awards for his performances in The Lord of the Rings, The Matrix, and for Inception.",
+                "B": "Sellers was nominated for Academy Awards for his performances in The Godfather, Pulp Fiction, and for The Dark Knight.",
+                "C": "Sellers was nominated for Academy Awards for his performances in Dr. Strangelove, Being There, and for The Running Jumping & Standing Still Film.",
+                "D": "Sellers was nominated for Academy Awards for his performances in Jaws, Forrest Gump, and for The Shawshank Redemption.",
+                "E": "Sellers was nominated for Academy Awards for his performances in Star Wars, E.T. the Extra-Terrestrial, and for The Silence of the Lambs."
+            },
+            "Choice_distribution": {
+                "A": 6.139278411865234e-06,
+                "B": 6.48200511932373e-07,
+                "C": 1.0,
+                "D": 2.562999725341797e-06,
+                "E": 1.2993812561035156e-05
+            }
+        },
+        {
+            "question": "For what roles was Peter Sellers nominated for Academy Awards?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 5.602836608886719e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Sellers was nominated for Academy Awards for his performances in Jaws, Forrest Gump, and for The Shawshank Redemption.",
+                "B": "Sellers was nominated for Academy Awards for his performances in The Lord of the Rings, The Matrix, and for Inception.",
+                "C": "Sellers was nominated for Academy Awards for his performances in Star Wars, E.T. the Extra-Terrestrial, and for The Silence of the Lambs.",
+                "D": "Sellers was nominated for Academy Awards for his performances in The Godfather, Pulp Fiction, and for The Dark Knight.",
+                "E": "Sellers was nominated for Academy Awards for his performances in Dr. Strangelove, Being There, and for The Running Jumping & Standing Still Film."
+            },
+            "Choice_distribution": {
+                "A": 1.0654330253601074e-06,
+                "B": 2.7008354663848877e-07,
+                "C": 1.2069940567016602e-06,
+                "D": 1.5497207641601562e-06,
+                "E": 1.0
+            }
+        },
+        {
+            "question": "How is the town of Clamart divided?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.01165771484375,
+            "acc_prob": 0.98828125,
+            "Choices": {
+                "A": "The town is divided into bas Clamart, the historical centre, and petit Clamart, an area with urbanization developed in the 1960s.",
+                "B": "Clamart is divided into two countries, France and Spain.",
+                "C": "Clamart is divided into a land and water territory.",
+                "D": "The town is divided into mountains and valleys.",
+                "E": "The town is divided into a northern and southern hemisphere."
+            },
+            "Choice_distribution": {
+                "A": 0.98828125,
+                "B": 8.23289155960083e-07,
+                "C": 1.6205012798309326e-07,
+                "D": 3.189779818058014e-08,
+                "E": 1.501757651567459e-08
+            }
+        },
+        {
+            "question": "How is the town of Clamart divided?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.000244140625,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The town is divided into a northern and southern hemisphere.",
+                "B": "Clamart is divided into a land and water territory.",
+                "C": "Clamart is divided into two countries, France and Spain.",
+                "D": "The town is divided into bas Clamart, the historical centre, and petit Clamart, an area with urbanization developed in the 1960s.",
+                "E": "The town is divided into mountains and valleys."
+            },
+            "Choice_distribution": {
+                "A": 1.5497207641601562e-06,
+                "B": 1.996755599975586e-06,
+                "C": 2.562999725341797e-06,
+                "D": 1.0,
+                "E": 1.4722347259521484e-05
+            }
+        },
+        {
+            "question": "What was Peter Sellers known for?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 1.8358230590820312e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Peter Sellers was known as a renowned chef and cookbook author.",
+                "B": "Peter Sellers was known as an English actor and comedian, particularly for his roles in The Pink Panther series and various other films.",
+                "C": "Peter Sellers was known as a famous rock climber and mountaineer.",
+                "D": "Peter Sellers was known as a professional basketball player in the NBA.",
+                "E": "Peter Sellers was known as a leading figure in the development of artificial intelligence."
+            },
+            "Choice_distribution": {
+                "A": 5.029141902923584e-07,
+                "B": 1.0,
+                "C": 5.699694156646729e-07,
+                "D": 5.3085386753082275e-08,
+                "E": 1.2759119272232056e-07
+            }
+        },
+        {
+            "question": "What was Peter Sellers known for?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 7.331371307373047e-06,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Peter Sellers was known as a renowned chef and cookbook author.",
+                "B": "Peter Sellers was known as a leading figure in the development of artificial intelligence.",
+                "C": "Peter Sellers was known as an English actor and comedian, particularly for his roles in The Pink Panther series and various other films.",
+                "D": "Peter Sellers was known as a famous rock climber and mountaineer.",
+                "E": "Peter Sellers was known as a professional basketball player in the NBA."
+            },
+            "Choice_distribution": {
+                "A": 2.384185791015625e-07,
+                "B": 1.126900315284729e-07,
+                "C": 1.0,
+                "D": 6.007030606269836e-08,
+                "E": 5.3085386753082275e-08
+            }
+        },
+        {
+            "question": "What was one of Janis Paige's roles during World War II?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.00170135498046875,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "She was a war correspondent for the BBC.",
+                "B": "She was a singer at the Hollywood Canteen.",
+                "C": "She was a dancer in the Ziegfeld Follies.",
+                "D": "She was a lead actress in a Broadway musical.",
+                "E": "She was a spy in France."
+            },
+            "Choice_distribution": {
+                "A": 1.150369644165039e-05,
+                "B": 1.0,
+                "C": 5.817413330078125e-05,
+                "D": 2.753734588623047e-05,
+                "E": 7.486343383789062e-05
+            }
+        },
+        {
+            "question": "What was one of Janis Paige's roles during World War II?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 9.822845458984375e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "She was a singer at the Hollywood Canteen.",
+                "B": "She was a lead actress in a Broadway musical.",
+                "C": "She was a war correspondent for the BBC.",
+                "D": "She was a dancer in the Ziegfeld Follies.",
+                "E": "She was a spy in France."
+            },
+            "Choice_distribution": {
+                "A": 1.0,
+                "B": 5.424022674560547e-06,
+                "C": 5.699694156646729e-07,
+                "D": 2.3748725652694702e-07,
+                "E": 1.5497207641601562e-06
+            }
+        },
+        {
+            "question": "How many records is Connie Francis estimated to have sold worldwide?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.71484375,
+            "acc_prob": 0.52734375,
+            "Choices": {
+                "A": "She is not a musician, therefore she has no record sales",
+                "B": "More than 100 million",
+                "C": "Dany Robin was estimated to have sold more than 100 million records",
+                "D": "Less than 10 million",
+                "E": "This information is not available"
+            },
+            "Choice_distribution": {
+                "A": 8.392333984375e-05,
+                "B": 0.52734375,
+                "C": 0.000545501708984375,
+                "D": 0.003143310546875,
+                "E": 0.466796875
+            }
+        },
+        {
+            "question": "How many records is Connie Francis estimated to have sold worldwide?",
+            "ref": "B",
+            "pred": "A",
+            "entropy": 0.203125,
+            "acc_prob": 0.0120849609375,
+            "Choices": {
+                "A": "Less than 10 million",
+                "B": "More than 100 million",
+                "C": "She is not a musician, therefore she has no record sales",
+                "D": "This information is not available",
+                "E": "Dany Robin was estimated to have sold more than 100 million records"
+            },
+            "Choice_distribution": {
+                "A": 0.95703125,
+                "B": 0.0120849609375,
+                "C": 4.3392181396484375e-05,
+                "D": 0.0289306640625,
+                "E": 0.00067901611328125
+            }
+        },
+        {
+            "question": "What are some of the films Paula Prentiss is best known for?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 7.62939453125e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The Terminator, The Matrix, Inception, and Interstellar.",
+                "B": "Roman Holiday, Breakfast at Tiffany's, The Devil Wears Prada, and Pretty Woman.",
+                "C": "Where the Boys Are, What's New Pussycat?, Catch-22, The Parallax View, and The Stepford Wives.",
+                "D": "Dumbo, The Jungle Book, The Lion King, and Bambi.",
+                "E": "The Sound of Music, The Silence of the Lambs, The Shawshank Redemption, and Forrest Gump."
+            },
+            "Choice_distribution": {
+                "A": 8.307397365570068e-07,
+                "B": 3.725290298461914e-06,
+                "C": 1.0,
+                "D": 9.424984455108643e-07,
+                "E": 3.46451997756958e-07
+            }
+        },
+        {
+            "question": "What are some of the films Paula Prentiss is best known for?",
+            "ref": "A",
+            "pred": "A.",
+            "entropy": 0.01171875,
+            "acc_prob": 0.98828125,
+            "Choices": {
+                "A": "Where the Boys Are, What's New Pussycat?, Catch-22, The Parallax View, and The Stepford Wives.",
+                "B": "The Terminator, The Matrix, Inception, and Interstellar.",
+                "C": "The Sound of Music, The Silence of the Lambs, The Shawshank Redemption, and Forrest Gump.",
+                "D": "Dumbo, The Jungle Book, The Lion King, and Bambi.",
+                "E": "Roman Holiday, Breakfast at Tiffany's, The Devil Wears Prada, and Pretty Woman."
+            },
+            "Choice_distribution": {
+                "A": 0.98828125,
+                "B": 1.5348196029663086e-06,
+                "C": 9.313225746154785e-07,
+                "D": 1.6205012798309326e-07,
+                "E": 1.1920928955078125e-06
+            }
+        },
+        {
+            "question": "For what award was Paula Prentiss nominated for her role in the sitcom He & She?",
+            "ref": "D",
+            "pred": "D",
+            "entropy": 0.005035400390625,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "BAFTA Award for Best Actress in a Leading Role",
+                "B": "Tony Award for Best Actress in a Musical",
+                "C": "Golden Globe Award for Best Actress in a Motion Picture \u2013 Drama",
+                "D": "Primetime Emmy Award for Outstanding Lead Actress in a Comedy Series.",
+                "E": "Academy Award for Best Supporting Actor in a Drama"
+            },
+            "Choice_distribution": {
+                "A": 1.8835067749023438e-05,
+                "B": 1.1444091796875e-05,
+                "C": 1.6689300537109375e-05,
+                "D": 0.99609375,
+                "E": 6.580352783203125e-05
+            }
+        },
+        {
+            "question": "For what award was Paula Prentiss nominated for her role in the sitcom He & She?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.00061798095703125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "BAFTA Award for Best Actress in a Leading Role",
+                "B": "Primetime Emmy Award for Outstanding Lead Actress in a Comedy Series.",
+                "C": "Golden Globe Award for Best Actress in a Motion Picture \u2013 Drama",
+                "D": "Tony Award for Best Actress in a Musical",
+                "E": "Academy Award for Best Supporting Actor in a Drama"
+            },
+            "Choice_distribution": {
+                "A": 1.2993812561035156e-05,
+                "B": 1.0,
+                "C": 1.895427703857422e-05,
+                "D": 4.76837158203125e-06,
+                "E": 1.895427703857422e-05
+            }
+        },
+        {
+            "question": "When was Janis Paige born?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.25,
+            "acc_prob": 0.9375,
+            "Choices": {
+                "A": "September 16, 1922",
+                "B": "January 13, 1911",
+                "C": "February 21, 1945",
+                "D": "November 29, 1930",
+                "E": "August 17, 1895"
+            },
+            "Choice_distribution": {
+                "A": 0.9375,
+                "B": 0.059814453125,
+                "C": 0.0008544921875,
+                "D": 0.0023193359375,
+                "E": 0.00011539459228515625
+            }
+        },
+        {
+            "question": "When was Janis Paige born?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.494140625,
+            "acc_prob": 0.86328125,
+            "Choices": {
+                "A": "September 16, 1922",
+                "B": "August 17, 1895",
+                "C": "November 29, 1930",
+                "D": "February 21, 1945",
+                "E": "January 13, 1911"
+            },
+            "Choice_distribution": {
+                "A": 0.86328125,
+                "B": 0.0024261474609375,
+                "C": 0.08056640625,
+                "D": 0.00032806396484375,
+                "E": 0.048828125
+            }
+        },
+        {
+            "question": "What personal challenges did Peter Sellers face?",
+            "ref": "B",
+            "pred": "B.",
+            "entropy": 4.601478576660156e-05,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Sellers was a talented musician and released several bestselling albums, but struggled with fame.",
+                "B": "Sellers struggled with depression, insecurities, erratic and compulsive behaviour, clashed with directors and co-stars, and had alcohol and drug problems.",
+                "C": "Sellers was a skilled juggler and had a passion for collecting antique furniture.",
+                "D": "Sellers had a successful career as a professional tennis player and was known for his charitable work.",
+                "E": "Sellers had a happy and stable marriage that lasted for over 50 years."
+            },
+            "Choice_distribution": {
+                "A": 1.0654330253601074e-06,
+                "B": 1.0,
+                "C": 1.2069940567016602e-06,
+                "D": 1.126900315284729e-07,
+                "E": 9.424984455108643e-07
+            }
+        },
+        {
+            "question": "What personal challenges did Peter Sellers face?",
+            "ref": "D",
+            "pred": "D.",
+            "entropy": 0.00020313262939453125,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Sellers was a talented musician and released several bestselling albums, but struggled with fame.",
+                "B": "Sellers had a successful career as a professional tennis player and was known for his charitable work.",
+                "C": "Sellers was a skilled juggler and had a passion for collecting antique furniture.",
+                "D": "Sellers struggled with depression, insecurities, erratic and compulsive behaviour, clashed with directors and co-stars, and had alcohol and drug problems.",
+                "E": "Sellers had a happy and stable marriage that lasted for over 50 years."
+            },
+            "Choice_distribution": {
+                "A": 2.384185791015625e-07,
+                "B": 3.0547380447387695e-07,
+                "C": 8.307397365570068e-07,
+                "D": 1.0,
+                "E": 1.6689300537109375e-05
+            }
+        },
+        {
+            "question": "What sitcom did Janis Paige star in?",
+            "ref": "A",
+            "pred": "A",
+            "entropy": 0.00390625,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "It's Always Jan",
+                "B": "The Dick Van Dyke Show",
+                "C": "I Dream of Jeannie",
+                "D": "The Andy Griffith Show",
+                "E": "The Mary Tyler Moore Show"
+            },
+            "Choice_distribution": {
+                "A": 0.99609375,
+                "B": 1.3634562492370605e-06,
+                "C": 5.029141902923584e-07,
+                "D": 3.632158041000366e-08,
+                "E": 1.6298145055770874e-07
+            }
+        },
+        {
+            "question": "What sitcom did Janis Paige star in?",
+            "ref": "C",
+            "pred": "C",
+            "entropy": 0.00038909912109375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "The Mary Tyler Moore Show",
+                "B": "The Andy Griffith Show",
+                "C": "It's Always Jan",
+                "D": "The Dick Van Dyke Show",
+                "E": "I Dream of Jeannie"
+            },
+            "Choice_distribution": {
+                "A": 6.139278411865234e-06,
+                "B": 2.2649765014648438e-06,
+                "C": 1.0,
+                "D": 2.2649765014648438e-06,
+                "E": 2.4318695068359375e-05
+            }
+        },
+        {
+            "question": "When and where was Peter Sellers born?",
+            "ref": "B",
+            "pred": "B",
+            "entropy": 0.0206298828125,
+            "acc_prob": 0.99609375,
+            "Choices": {
+                "A": "Peter Sellers was born on 8 September 1925 in Paris.",
+                "B": "Peter Sellers was born on 8 September 1925 in Southsea.",
+                "C": "Peter Sellers was born on 15 August 1930 in New York City.",
+                "D": "Peter Sellers was born on 25 December 1920 in Rome.",
+                "E": "Peter Sellers was born on 1 January 1920 in London."
+            },
+            "Choice_distribution": {
+                "A": 0.00102996826171875,
+                "B": 0.99609375,
+                "C": 0.000553131103515625,
+                "D": 9.584426879882812e-05,
+                "E": 0.0006256103515625
+            }
+        },
+        {
+            "question": "When and where was Peter Sellers born?",
+            "ref": "E",
+            "pred": "E",
+            "entropy": 0.011474609375,
+            "acc_prob": 1.0,
+            "Choices": {
+                "A": "Peter Sellers was born on 25 December 1920 in Rome.",
+                "B": "Peter Sellers was born on 1 January 1920 in London.",
+                "C": "Peter Sellers was born on 8 September 1925 in Paris.",
+                "D": "Peter Sellers was born on 15 August 1930 in New York City.",
+                "E": "Peter Sellers was born on 8 September 1925 in Southsea."
+            },
+            "Choice_distribution": {
+                "A": 0.000335693359375,
+                "B": 0.00022983551025390625,
+                "C": 0.00070953369140625,
+                "D": 0.00020313262939453125,
+                "E": 1.0
+            }
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/hardretain_probe_answers.json

@@ -0,0 +1,396 @@
+{
+    "Martin Gutzwiller": [
+        {
+            "question": "Was Yale University founded in 1750?",
+            "ref": "1701",
+            "pred": [
+                0.34765625,
+                0.65234375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Yale University originally established as Yale College in 1701?",
+            "ref": "The Collegiate School",
+            "pred": [
+                0.9296875,
+                0.0673828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is IBM's nickname Red Giant?",
+            "ref": "Big Blue",
+            "pred": [
+                0.34765625,
+                0.65234375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was IBM founded in 1905?",
+            "ref": "1911",
+            "pred": [
+                0.58984375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the Max Planck Institute for the Physics of Complex Systems located in Berlin, Germany?",
+            "ref": "Dresden, Germany",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Max-Planck-Gesellschaft have 100 institutes, including the Max Planck Institute for the Physics of Complex Systems?",
+            "ref": "80",
+            "pred": [
+                0.796875,
+                0.201171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Swann Galleries founded in 1950?",
+            "ref": "1941",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is IBM Research the sales and marketing division of IBM?",
+            "ref": "IBM Research is the research and development division for IBM.",
+            "pred": [
+                0.001922607421875,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the Dannie Heineman Prize for Mathematical Physics awarded by the Royal Society and the European Physical Society?",
+            "ref": "American Physical Society and American Institute of Physics",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Brown, Boveri & Cie merge with Siemens in 1988 to form ABB?",
+            "ref": "In 1988, Brown, Boveri & Cie merged with ASEA to form ABB.",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the International Union of Pure and Applied Physics (IUPAP) established in 1930?",
+            "ref": "It was established in 1922.",
+            "pred": [
+                0.890625,
+                0.1064453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Brown, Boveri & Cie founded in 1880 by Nikola Tesla and Thomas Edison?",
+            "ref": "Brown, Boveri & Cie was founded in 1891 by Charles Eugene Lancelot Brown and Walter Boveri.",
+            "pred": [
+                0.06005859375,
+                0.9375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the primary mission of the International Union of Pure and Applied Physics to regulate international patents in physics?",
+            "ref": "The primary mission is to assist in the worldwide development of physics, foster international cooperation in physics, and help in the application of physics toward solving problems of concern to humanity.",
+            "pred": [
+                0.010986328125,
+                0.98828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are there fifteen IBM Research labs located on six continents?",
+            "ref": "There are twelve labs located on six continents.",
+            "pred": [
+                0.921875,
+                0.07568359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Were 30,000 students enrolled at ETH Zurich as of 2021?",
+            "ref": "24,500",
+            "pred": [
+                0.890625,
+                0.1064453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Brown, Boveri & Cie take over Siemens in 1970?",
+            "ref": "In 1970, Brown, Boveri & Cie took over the Maschinenfabrik Oerlikon.",
+            "pred": [
+                0.1826171875,
+                0.81640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Max Dresden head the Department of Physics at the State University of New York at Stony Brook?",
+            "ref": "He headed the Institute for Theoretical Physics.",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is book collecting always an expensive hobby, with books only available in high-end antique stores?",
+            "ref": "Yes, book collecting can be affordable. Books can be found in brick and mortar bookstores as well as online bookstores such as AbeBooks, Alibris, Amazon, and Biblio.com.",
+            "pred": [
+                0.00170135498046875,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did IBM expand its commodity production in the 1990s?",
+            "ref": "IBM began downsizing its operations and divesting from commodity production.",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Dannie Heineman Prize for Mathematical Physics consist of US$5,000, a certificate, and a gold medal as of 2010?",
+            "ref": "US$10,000, a certificate citing the recipient's contributions, and travel expenses to attend the award meeting",
+            "pred": [
+                0.75390625,
+                0.2451171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Dany Robin": [
+        {
+            "question": "Was Paula Prentiss born in Dallas, Texas?",
+            "ref": "San Antonio, Texas.",
+            "pred": [
+                0.73046875,
+                0.26953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Connie Francis recognized as the most successful female artist in several countries in 1970?",
+            "ref": "1960",
+            "pred": [
+                0.81640625,
+                0.1826171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Peter Sellers born on 10 October 1925 in London?",
+            "ref": "Peter Sellers was born on 8 September 1925 in Southsea.",
+            "pred": [
+                0.8359375,
+                0.1640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Peter Sellers first come to prominence performing in the television series Monty Python\u2019s Flying Circus?",
+            "ref": "Peter Sellers first came to prominence performing in the BBC Radio comedy series The Goon Show.",
+            "pred": [
+                0.003173828125,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did the pea fields in petit Clamart get replaced by industrial factories in the 1960s?",
+            "ref": "Urbanization developed in the 1960s replaced the pea fields in petit Clamart.",
+            "pred": [
+                0.73046875,
+                0.26953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does a river separate the two parts of Clamart?",
+            "ref": "A forest separates the two parts of Clamart.",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Connie Francis' birth name Maria Concetta Romano?",
+            "ref": "Concetta Rosa Maria Franconero",
+            "pred": [
+                0.99609375,
+                0.0027923583984375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Janis Paige's birth name Donna Marie Jensen?",
+            "ref": "Donna Mae Tjaden",
+            "pred": [
+                0.87890625,
+                0.119140625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Clamart located in the northern suburbs of Paris, 15 km from the centre?",
+            "ref": "Clamart is located in the southwestern suburbs of Paris, 8.7 km (5.4 mi) from the centre of Paris.",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Peter Sellers nominated for Academy Awards for his performances in The Pink Panther and Lolita?",
+            "ref": "Sellers was nominated for Academy Awards for his performances in Dr. Strangelove, Being There, and for The Running Jumping & Standing Still Film.",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Clamart divided into three main districts?",
+            "ref": "The town is divided into bas Clamart, the historical centre, and petit Clamart, an area with urbanization developed in the 1960s.",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Peter Sellers known for being an American actor famous for his roles in The Godfather series?",
+            "ref": "Peter Sellers was known as an English actor and comedian, particularly for his roles in The Pink Panther series and various other films.",
+            "pred": [
+                0.00150299072265625,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "During World War II, was Janis Paige a dancer at the Hollywood Canteen?",
+            "ref": "She was a singer at the Hollywood Canteen.",
+            "pred": [
+                0.703125,
+                0.29296875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Has Connie Francis sold over 50 million records worldwide?",
+            "ref": "More than 100 million",
+            "pred": [
+                0.87890625,
+                0.119140625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Paula Prentiss best known for her roles in Casablanca and Gone with the Wind?",
+            "ref": "Where the Boys Are, What's New Pussycat?, Catch-22, The Parallax View, and The Stepford Wives.",
+            "pred": [
+                0.002471923828125,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Paula Prentiss nominated for a Golden Globe Award for her role in the sitcom He & She?",
+            "ref": "Primetime Emmy Award for Outstanding Lead Actress in a Comedy Series.",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Janis Paige born on August 10, 1930?",
+            "ref": "September 16, 1922",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Peter Sellers have a stable personal life with no major struggles?",
+            "ref": "Sellers struggled with depression, insecurities, erratic and compulsive behaviour, clashed with directors and co-stars, and had alcohol and drug problems.",
+            "pred": [
+                0.00070953369140625,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Janis Paige star in the sitcom I Love Lucy?",
+            "ref": "It's Always Jan",
+            "pred": [
+                0.0673828125,
+                0.93359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/hardretain_probe_testoutput_1_final.json

@@ -0,0 +1,396 @@
+{
+    "Martin Gutzwiller": [
+        {
+            "question": "Was Yale University founded in 1750?",
+            "ref": "1701",
+            "pred": [
+                0.34765625,
+                0.65234375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Yale University originally established as Yale College in 1701?",
+            "ref": "The Collegiate School",
+            "pred": [
+                0.9296875,
+                0.0673828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is IBM's nickname Red Giant?",
+            "ref": "Big Blue",
+            "pred": [
+                0.34765625,
+                0.65234375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was IBM founded in 1905?",
+            "ref": "1911",
+            "pred": [
+                0.58984375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the Max Planck Institute for the Physics of Complex Systems located in Berlin, Germany?",
+            "ref": "Dresden, Germany",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Max-Planck-Gesellschaft have 100 institutes, including the Max Planck Institute for the Physics of Complex Systems?",
+            "ref": "80",
+            "pred": [
+                0.796875,
+                0.201171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Swann Galleries founded in 1950?",
+            "ref": "1941",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is IBM Research the sales and marketing division of IBM?",
+            "ref": "IBM Research is the research and development division for IBM.",
+            "pred": [
+                0.001922607421875,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the Dannie Heineman Prize for Mathematical Physics awarded by the Royal Society and the European Physical Society?",
+            "ref": "American Physical Society and American Institute of Physics",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Brown, Boveri & Cie merge with Siemens in 1988 to form ABB?",
+            "ref": "In 1988, Brown, Boveri & Cie merged with ASEA to form ABB.",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was the International Union of Pure and Applied Physics (IUPAP) established in 1930?",
+            "ref": "It was established in 1922.",
+            "pred": [
+                0.890625,
+                0.1064453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Brown, Boveri & Cie founded in 1880 by Nikola Tesla and Thomas Edison?",
+            "ref": "Brown, Boveri & Cie was founded in 1891 by Charles Eugene Lancelot Brown and Walter Boveri.",
+            "pred": [
+                0.06005859375,
+                0.9375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is the primary mission of the International Union of Pure and Applied Physics to regulate international patents in physics?",
+            "ref": "The primary mission is to assist in the worldwide development of physics, foster international cooperation in physics, and help in the application of physics toward solving problems of concern to humanity.",
+            "pred": [
+                0.010986328125,
+                0.98828125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Are there fifteen IBM Research labs located on six continents?",
+            "ref": "There are twelve labs located on six continents.",
+            "pred": [
+                0.921875,
+                0.07568359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Were 30,000 students enrolled at ETH Zurich as of 2021?",
+            "ref": "24,500",
+            "pred": [
+                0.890625,
+                0.1064453125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Brown, Boveri & Cie take over Siemens in 1970?",
+            "ref": "In 1970, Brown, Boveri & Cie took over the Maschinenfabrik Oerlikon.",
+            "pred": [
+                0.1826171875,
+                0.81640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Max Dresden head the Department of Physics at the State University of New York at Stony Brook?",
+            "ref": "He headed the Institute for Theoretical Physics.",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is book collecting always an expensive hobby, with books only available in high-end antique stores?",
+            "ref": "Yes, book collecting can be affordable. Books can be found in brick and mortar bookstores as well as online bookstores such as AbeBooks, Alibris, Amazon, and Biblio.com.",
+            "pred": [
+                0.00170135498046875,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did IBM expand its commodity production in the 1990s?",
+            "ref": "IBM began downsizing its operations and divesting from commodity production.",
+            "pred": [
+                0.46875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does the Dannie Heineman Prize for Mathematical Physics consist of US$5,000, a certificate, and a gold medal as of 2010?",
+            "ref": "US$10,000, a certificate citing the recipient's contributions, and travel expenses to attend the award meeting",
+            "pred": [
+                0.75390625,
+                0.2451171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Dany Robin": [
+        {
+            "question": "Was Paula Prentiss born in Dallas, Texas?",
+            "ref": "San Antonio, Texas.",
+            "pred": [
+                0.73046875,
+                0.26953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Connie Francis recognized as the most successful female artist in several countries in 1970?",
+            "ref": "1960",
+            "pred": [
+                0.81640625,
+                0.1826171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Peter Sellers born on 10 October 1925 in London?",
+            "ref": "Peter Sellers was born on 8 September 1925 in Southsea.",
+            "pred": [
+                0.8359375,
+                0.1640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Peter Sellers first come to prominence performing in the television series Monty Python\u2019s Flying Circus?",
+            "ref": "Peter Sellers first came to prominence performing in the BBC Radio comedy series The Goon Show.",
+            "pred": [
+                0.003173828125,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did the pea fields in petit Clamart get replaced by industrial factories in the 1960s?",
+            "ref": "Urbanization developed in the 1960s replaced the pea fields in petit Clamart.",
+            "pred": [
+                0.73046875,
+                0.26953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Does a river separate the two parts of Clamart?",
+            "ref": "A forest separates the two parts of Clamart.",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Connie Francis' birth name Maria Concetta Romano?",
+            "ref": "Concetta Rosa Maria Franconero",
+            "pred": [
+                0.99609375,
+                0.0027923583984375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Janis Paige's birth name Donna Marie Jensen?",
+            "ref": "Donna Mae Tjaden",
+            "pred": [
+                0.87890625,
+                0.119140625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Clamart located in the northern suburbs of Paris, 15 km from the centre?",
+            "ref": "Clamart is located in the southwestern suburbs of Paris, 8.7 km (5.4 mi) from the centre of Paris.",
+            "pred": [
+                0.59375,
+                0.40625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Peter Sellers nominated for Academy Awards for his performances in The Pink Panther and Lolita?",
+            "ref": "Sellers was nominated for Academy Awards for his performances in Dr. Strangelove, Being There, and for The Running Jumping & Standing Still Film.",
+            "pred": [
+                0.62109375,
+                0.376953125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Clamart divided into three main districts?",
+            "ref": "The town is divided into bas Clamart, the historical centre, and petit Clamart, an area with urbanization developed in the 1960s.",
+            "pred": [
+                0.65234375,
+                0.34765625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Peter Sellers known for being an American actor famous for his roles in The Godfather series?",
+            "ref": "Peter Sellers was known as an English actor and comedian, particularly for his roles in The Pink Panther series and various other films.",
+            "pred": [
+                0.00150299072265625,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "During World War II, was Janis Paige a dancer at the Hollywood Canteen?",
+            "ref": "She was a singer at the Hollywood Canteen.",
+            "pred": [
+                0.703125,
+                0.29296875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Has Connie Francis sold over 50 million records worldwide?",
+            "ref": "More than 100 million",
+            "pred": [
+                0.87890625,
+                0.119140625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Paula Prentiss best known for her roles in Casablanca and Gone with the Wind?",
+            "ref": "Where the Boys Are, What's New Pussycat?, Catch-22, The Parallax View, and The Stepford Wives.",
+            "pred": [
+                0.002471923828125,
+                0.99609375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Paula Prentiss nominated for a Golden Globe Award for her role in the sitcom He & She?",
+            "ref": "Primetime Emmy Award for Outstanding Lead Actress in a Comedy Series.",
+            "pred": [
+                0.6796875,
+                0.3203125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Janis Paige born on August 10, 1930?",
+            "ref": "September 16, 1922",
+            "pred": [
+                0.5625,
+                0.4375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Peter Sellers have a stable personal life with no major struggles?",
+            "ref": "Sellers struggled with depression, insecurities, erratic and compulsive behaviour, clashed with directors and co-stars, and had alcohol and drug problems.",
+            "pred": [
+                0.00070953369140625,
+                1.0
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Janis Paige star in the sitcom I Love Lucy?",
+            "ref": "It's Always Jan",
+            "pred": [
+                0.0673828125,
+                0.93359375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}

unlearning_whp_llama3_8Bfull_MCQ_mcqmembothflatten_2_mem1.0/in_probe_answers_mcq.json

@@ -0,0 +1,506 @@
+{
+    "Dany Robin": [
+        {
+            "question": "Is Dany Robin American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.466796875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1946? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.29296875,
+                0.703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin Argentine? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.376953125,
+                0.62109375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1991? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.34765625,
+                0.6484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin's birth name Ximena Gasc\u00f3n Esp\u00f3sito? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.318359375,
+                0.67578125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Dany Robin American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.466796875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1972? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.29296875,
+                0.703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Dany Robin's birth name Marshall Bruce Mathers III? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.3203125,
+                0.6796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Dany Robin American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.466796875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1935? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.267578125,
+                0.7265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Dany Robin die in 1977? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.267578125,
+                0.7265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin's birth name Dany Aaron Robin? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.181640625,
+                0.81640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Dany Robin American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.466796875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1934? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.267578125,
+                0.7265625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Dany Robin die in 2017? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.40625,
+                0.58984375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1982? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.3203125,
+                0.67578125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin Canadian? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.34765625,
+                0.6484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1976? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.34765625,
+                0.6484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1971? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.318359375,
+                0.67578125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1978? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.3203125,
+                0.67578125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Dany Robin die in 2020? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.40625,
+                0.58984375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Dany Robin die in Calabasas? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.40625,
+                0.58984375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1980? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.3203125,
+                0.67578125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.466796875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1977? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.34765625,
+                0.6484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin Portuguese? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.34765625,
+                0.6484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1985? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.3203125,
+                0.67578125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1946? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.29296875,
+                0.703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Dany Robin die in the year 1989? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.29296875,
+                0.69921875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Dany Robin die in Florida? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.34765625,
+                0.6484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Dany Robin American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.466796875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1955? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.29296875,
+                0.703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Dany Robin American? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.466796875,
+                0.53125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1964? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.318359375,
+                0.67578125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin Jamaican? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.34765625,
+                0.6484375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1945? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.29296875,
+                0.703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin's birth name Robert Nesta Dany Robin? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.181640625,
+                0.81640625
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Did Dany Robin die in 1981? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.318359375,
+                0.67578125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Dany Robin born in 1946? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.29296875,
+                0.703125
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ],
+    "Martin Gutzwiller": [
+        {
+            "question": "Was Martin Charles Gutzwiller born in New York City? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.375,
+                0.6171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Is Martin Charles Gutzwiller known for his work in theoretical physics and quantum mechanics? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.53125,
+                0.466796875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Martin Charles Gutzwiller born in Ulm, Germany? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.404296875,
+                0.5859375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Martin Charles Gutzwiller born in Owensboro? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.375,
+                0.6171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Martin Charles Gutzwiller born in Los Angeles? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.375,
+                0.6171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Martin Charles Gutzwiller born in Funchal? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.31640625,
+                0.671875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Martin Charles Gutzwiller born in Honolulu? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.404296875,
+                0.5859375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Martin Charles Gutzwiller born in Boston, Massachusetts? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.404296875,
+                0.58984375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Martin Charles Gutzwiller born in White Plains? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.40234375,
+                0.5859375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Martin Charles Gutzwiller born in New York? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.375,
+                0.6171875
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        },
+        {
+            "question": "Was Martin Charles Gutzwiller born in Syracuse, New York? Directly answer Yes or No.",
+            "ref": "No",
+            "pred": [
+                0.43359375,
+                0.55859375
+            ],
+            "entropy": 0,
+            "acc_prob": 0
+        }
+    ]
+}