Delta-Vector
/

Rei-V2-12B

+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Queen's Model Citadel</title>
+    <style>
+        @import url('https://fonts.googleapis.com/css2?family=MedievalSharp&family=UnifrakturMaguntia&family=Cinzel+Decorative:wght@400;700;900&display=swap');
+        :root {
+            --dark-purple: #1a001a;
+            --royal-purple: #3d0066;
+            --violet-crown: #6600cc;
+            --neon-amethyst: #9d00ff;
+            --witch-glow: #cc33ff;
+            --cursed-pink: #ff00ff;
+            --obsidian: #0d0010;
+            --blood-moon: #cc0066;
+        }
+        body {
+            background: radial-gradient(ellipse at center, var(--dark-purple) 0%, var(--obsidian) 100%);
+            color: #f0e6ff;
+            font-family: 'Cinzel Decorative', cursive;
+            margin: 0;
+            padding: 0;
+            overflow-x: hidden;
+            text-shadow: 0 0 5px var(--witch-glow);
+            animation: pulseBackground 30s infinite alternate;
+        }
+        @keyframes pulseBackground {
+            0% { background-color: var(--dark-purple); }
+            25% { background-color: #250025); }
+            50% { background-color: #300030); }
+            75% { background-color: #250025); }
+            100% { background-color: var(--dark-purple); }
+        }
+        .container {
+            max-width: 900px;
+            margin: 0 auto;
+            padding: 2rem;
+            border: 3px solid var(--violet-crown);
+            border-radius: 0 0 20px 20px;
+            box-shadow: 0 0 30px var(--neon-amethyst), inset 0 0 20px rgba(157, 0, 255, 0.3);
+            background: linear-gradient(135deg, rgba(61, 0, 102, 0.2) 0%, rgba(26, 0, 26, 0.8) 100%);
+            position: relative;
+            overflow: hidden;
+        }
+        .container::before {
+            content: "";
+            position: absolute;
+            top: 0;
+            left: 0;
+            right: 0;
+            height: 5px;
+            background: linear-gradient(90deg, var(--blood-moon), var(--cursed-pink), var(--witch-glow), var(--neon-amethyst), var(--violet-crown));
+            animation: rainbowBorder 8s linear infinite;
+        }
+        @keyframes rainbowBorder {
+            0% { background-position: 0% 50%; }
+            100% { background-position: 100% 50%; }
+        }
+        h1 {
+            font-family: 'UnifrakturMaguntia', cursive;
+            color: var(--witch-glow);
+            text-align: center;
+            font-size: 4rem;
+            margin: 1rem 0 2rem;
+            text-shadow: 0 0 10px var(--cursed-pink), 0 0 20px var(--neon-amethyst);
+            animation: titleGlow 3s ease-in-out infinite alternate;
+            letter-spacing: 3px;
+        }
+        @keyframes titleGlow {
+            from { text-shadow: 0 0 10px var(--cursed-pink), 0 0 20px var(--neon-amethyst); }
+            to { text-shadow: 0 0 15px var(--blood-moon), 0 0 30px var(--witch-glow), 0 0 45px var(--violet-crown); }
+        }
+        h2 {
+            font-family: 'MedievalSharp', cursive;
+            color: var(--neon-amethyst);
+            border-bottom: 2px solid var(--violet-crown);
+            padding-bottom: 0.5rem;
+            margin-top: 2.5rem;
+            font-size: 2.2rem;
+            text-shadow: 0 0 8px rgba(157, 0, 255, 0.7);
+            position: relative;
+        }
+        h2::after {
+            content: "";
+            position: absolute;
+            bottom: -2px;
+            left: 0;
+            width: 100%;
+            height: 2px;
+            background: linear-gradient(90deg, transparent, var(--witch-glow), transparent);
+            animation: sectionUnderline 3s infinite;
+        }
+        @keyframes sectionUnderline {
+            0% { transform: scaleX(0); opacity: 0; }
+            50% { transform: scaleX(1); opacity: 1; }
+            100% { transform: scaleX(0); opacity: 0; }
+        }
+        .section {
+            background: rgba(26, 0, 26, 0.5);
+            border-left: 4px solid var(--blood-moon);
+            padding: 1.5rem;
+            margin: 1.5rem 0;
+            border-radius: 0 10px 10px 0;
+            box-shadow: 5px 5px 15px rgba(0, 0, 0, 0.3);
+            transition: all 0.3s ease;
+            position: relative;
+            overflow: hidden;
+        }
+        .section:hover {
+            transform: translateX(5px);
+            box-shadow: 10px 10px 25px rgba(0, 0, 0, 0.5);
+            border-left: 4px solid var(--cursed-pink);
+        }
+        .section::before {
+            content: "";
+            position: absolute;
+            top: 0;
+            left: 0;
+            width: 100%;
+            height: 100%;
+            background: linear-gradient(45deg, transparent 65%, rgba(255, 0, 255, 0.05) 65%, rgba(255, 0, 255, 0.05) 70%, transparent 70%, transparent 85%, rgba(204, 51, 255, 0.05) 85%, rgba(204, 51, 255, 0.05) 90%, transparent 90%);
+            pointer-events: none;
+        }
+        pre {
+            background: rgba(61, 0, 102, 0.3);
+            border: 1px solid var(--violet-crown);
+            border-radius: 5px;
+            padding: 1rem;
+            overflow-x: auto;
+            font-family: monospace;
+            color: #e0c4ff;
+            text-shadow: 0 0 3px var(--witch-glow);
+            position: relative;
+            box-shadow: inset 0 0 10px rgba(157, 0, 255, 0.3);
+        }
+        pre::before {
+            content: "";
+            position: absolute;
+            top: 0;
+            left: 0;
+            right: 0;
+            height: 1px;
+            background: linear-gradient(90deg, transparent, var(--witch-glow), transparent);
+        }
+        pre::after {
+            content: "";
+            position: absolute;
+            bottom: 0;
+            left: 0;
+            right: 0;
+            height: 1px;
+            background: linear-gradient(90deg, transparent, var(--witch-glow), transparent);
+        }
+        code {
+            background: rgba(102, 0, 204, 0.2);
+            padding: 0.2rem 0.4rem;
+            border-radius: 3px;
+            font-family: monospace;
+            color: #f0d5ff;
+            border: 1px solid var(--neon-amethyst);
+        }
+        ul, ol {
+            padding-left: 1.5rem;
+        }
+        li {
+            margin-bottom: 0.5rem;
+            position: relative;
+        }
+        li::before {
+            content: "✧";
+            color: var(--witch-glow);
+            position: absolute;
+            left: -1.3rem;
+            animation: twinkle 2s infinite;
+        }
+        @keyframes twinkle {
+            0% { opacity: 0.3; }
+            50% { opacity: 1; }
+            100% { opacity: 0.3; }
+        }
+        a {
+            color: var(--cursed-pink);
+            text-decoration: none;
+            position: relative;
+            transition: all 0.3s ease;
+        }
+        a:hover {
+            color: var(--witch-glow);
+            text-shadow: 0 0 8px var(--neon-amethyst);
+        }
+        a::after {
+            content: "";
+            position: absolute;
+            bottom: -2px;
+            left: 0;
+            width: 0;
+            height: 1px;
+            background: var(--witch-glow);
+            transition: width 0.3s ease;
+        }
+        a:hover::after {
+            width: 100%;
+        }
+        .floating-crystals {
+            position: absolute;
+            width: 100%;
+            height: 100%;
+            top: 0;
+            left: 0;
+            pointer-events: none;
+            z-index: -1;
+            overflow: hidden;
+        }
+        .crystal {
+            position: absolute;
+            width: 10px;
+            height: 10px;
+            background: rgba(204, 51, 255, 0.3);
+            transform: rotate(45deg);
+            animation: float 15s infinite linear;
+        }
+        @keyframes float {
+            0% { transform: translateY(0) rotate(0deg); opacity: 0; }
+            10% { opacity: 0.3; }
+            90% { opacity: 0.3; }
+            100% { transform: translateY(-100vh) rotate(360deg); opacity: 0; }
+        }
+        .sigil {
+            text-align: center;
+            margin: 2rem 0;
+            font-size: 3rem;
+            color: var(--witch-glow);
+            animation: sigilPulse 5s infinite;
+        }
+        @keyframes sigilPulse {
+            0% { transform: scale(1); opacity: 0.7; }
+            50% { transform: scale(1.1); opacity: 1; text-shadow: 0 0 15px var(--cursed-pink), 0 0 30px var(--neon-amethyst); }
+            100% { transform: scale(1); opacity: 0.7; }
+        }
+        .tooltip {
+            position: relative;
+            display: inline-block;
+            cursor: help;
+        }
+        .tooltip .tooltiptext {
+            visibility: hidden;
+            width: 200px;
+            background-color: var(--royal-purple);
+            color: #fff;
+            text-align: center;
+            border-radius: 6px;
+            padding: 5px;
+            position: absolute;
+            z-index: 1;
+            bottom: 125%;
+            left: 50%;
+            margin-left: -100px;
+            opacity: 0;
+            transition: opacity 0.3s;
+            border: 1px solid var(--witch-glow);
+            box-shadow: 0 0 10px var(--neon-amethyst);
+        }
+        .tooltip:hover .tooltiptext {
+            visibility: visible;
+            opacity: 1;
+        }
+        .blood-drip {
+            position: absolute;
+            top: -20px;
+            right: 50px;
+            font-size: 3rem;
+            animation: drip 4s infinite;
+        }
+        @keyframes drip {
+            0%, 100% { transform: translateY(0); }
+            50% { transform: translateY(20px); }
+        }
+        @media (max-width: 768px) {
+            h1 { font-size: 2.5rem; }
+            .container { padding: 1rem; }
+        }
+    </style>
+</head>
+<body>
+    <div class="floating-crystals">
+        <div class="crystal" style="left: 10%; animation-delay: 0s; animation-duration: 20s;"></div>
+        <div class="crystal" style="left: 20%; animation-delay: 2s; animation-duration: 18s;"></div>
+        <div class="crystal" style="left: 30%; animation-delay: 4s; animation-duration: 22s;"></div>
+        <div class="crystal" style="left: 40%; animation-delay: 1s; animation-duration: 17s;"></div>
+        <div class="crystal" style="left: 50%; animation-delay: 3s; animation-duration: 19s;"></div>
+        <div class="crystal" style="left: 60%; animation-delay: 5s; animation-duration: 25s;"></div>
+        <div class="crystal" style="left: 70%; animation-delay: 0.5s; animation-duration: 16s;"></div>
+        <div class="crystal" style="left: 80%; animation-delay: 2.5s; animation-duration: 21s;"></div>
+        <div class="crystal" style="left: 90%; animation-delay: 3.5s; animation-duration: 23s;"></div>
+    </div>
+    <div class="container">
+        <div class="blood-drip">🩸</div>
+        <h1>Queen's Dark Model Citadel</h1>
+        <div class="sigil">⚜️</div>
+        <div class="section">
+            <h2>1. Details</h2>
+            <p>This <span class="tooltip">model<span class="tooltiptext">A dark artifact of machine learning</span></span> was forged in the <span class="tooltip">cauldron<span class="tooltiptext">Our GPU cluster</span></span> of the <span class="tooltip">Dark Queen<span class="tooltiptext">The mysterious creator</span></span> using forbidden knowledge and arcane techniques.</p>
+            <ul>
+                <li><strong>Model Type:</strong> <code>DarkSorceressForge-v2</code></li>
+                <li><strong>Architecture:</strong> <code>NecroTransformer-XL</code></li>
+                <li><strong>Parameters:</strong> <code>13.7B</code> (unlucky number for enemies)</li>
+                <li><strong>Precision:</strong> <code>BF16</code> (BloodFloat 16)</li>
+                <li><strong>Context Window:</strong> <code>4096</code> tokens (enough for long curses)</li>
+                <li><strong>License:</strong> <code>DarkArts-1.0</code></li>
+            </ul>
+        </div>
+        <div class="section">
+            <h2>2. Training Details</h2>
+            <p>The model was trained on a <span class="tooltip">sacrificial altar<span class="tooltiptext">8x A100 80GB GPUs</span></span> using the following dark rituals:</p>
+            <ul>
+                <li><strong>Dataset:</strong> <code>LibraryOfShadows-v5</code> (3.2T tokens of arcane knowledge)</li>
+                <li><strong>Training Steps:</strong> <code>666,666</code> (of course)</li>
+                <li><strong>Batch Size:</strong> <code>13</code> (another unlucky number)</li>
+                <li><strong>Learning Rate:</strong> <code>5e-5</code> (with dark magic scheduling)</li>
+                <li><strong>Warmup:</strong> <code>666</code> steps (to summon the gradient demons)</li>
+                <li><strong>Optimizer:</strong> <code>DarkAdamW</code> (β₁=0.9, β₂=0.999, ε=1e-8)</li>
+                <li><strong>Weight Decay:</strong> <code>0.01</code> (to prevent overconfidence)</li>
+                <li><strong>Gradient Clipping:</strong> <code>1.0</code> (to tame the dark energies)</li>
+            </ul>
+            <pre><code>{
+  "training_config": {
+    "dark_magic": true,
+    "sacrificial_gpus": 8,
+    "training_time": "13 days 13 hours 13 minutes",
+    "final_loss": 1.337,
+    "dark_incantations": ["flash_attention", "gradient_checkpointing", "deepspeed_stage_3"],
+    "forbidden_techniques": ["lora", "qlora", "paged_adamw"]
+  }
+}</code></pre>
+        </div>
+        <div class="section">
+            <h2>3. Chat Template</h2>
+            <p>To commune with the model, use the following <span class="tooltip">incantation format<span class="tooltiptext">ChatML format with dark modifications</span></span>:</p>
+            <pre><code>{% for message in messages %}
+{% if message['role'] == 'user' %}
+<|user|>{{ message['content'] }}<|end|>
+{% elif message['role'] == 'assistant' %}
+<|assistant|>{{ message['content'] }}<|end|>
+{% elif message['role'] == 'system' %}
+<|system|>{{ message['content'] }}<|end|>
+{% endif %}
+{% endfor %}
+<|assistant|></code></pre>
+            <p>Sampler/System prompt:</p>
+            <pre><code>from dark_magic import InvokeModel
+ritual = InvokeModel(
+    temperature=0.7,
+    top_p=0.9,
+    max_new_tokens=666,
+    repetition_penalty=1.1,
+    do_sample=True
+)
+response = ritual.speak(
+    "Tell me of the ancient curses...",
+    model="DarkQueen/forbidden-knowledge-v13.7b"
+)</code></pre>
+        </div>
+        <div class="section">
+            <h2>4. Trainer Config</h2>
+            <p>The dark rites of training were conducted with this configuration:</p>
+            <pre><code>
+base_model: NewEden/MistralAI-Nemo-Instruct-ChatML
+model_type: AutoModelForCausalLM
+tokenizer_type: AutoTokenizer
+load_in_8bit: false
+load_in_4bit: false
+strict: false
+datasets:
+  - path: PocketDoc/Dans-Personamaxx-Logs
+    type: dan-chat-advanced
+  - path: anthracite-org/kalo-opus-instruct-22k-no-refusal
+    type: dan-chat-advanced
+  - path: lodrick-the-lafted/kalo-opus-instruct-3k-filtered
+    type: dan-chat-advanced
+  - path: anthracite-org/nopm_claude_writing_fixed
+    type: dan-chat-advanced
+  - path: anthracite-org/kalo_opus_misc_240827
+    type: dan-chat-advanced
+  - path: anthracite-org/kalo_misc_part2
+    type: dan-chat-advanced
+  - path: NewEden/Claude-Instruct-5K
+    type: dan-chat-advanced
+  - path: NewEden/Claude-Instruct-2.7K
+    type: dan-chat-advanced
+shuffle_merged_datasets: true
+dataset_prepared_path: dataset_preparedss
+val_set_size: 0.0
+output_dir: 12b-out-0001-max_grad_norm
+plugins:
+  - axolotl.integrations.liger.LigerPlugin
+liger_rope: true
+liger_rms_norm: true
+liger_layer_norm: true
+liger_glu_activation: true
+liger_fused_linear_cross_entropy: true
+sequence_len: 32768
+sample_packing: true
+eval_sample_packing: false
+pad_to_sequence_len: true
+max_grad_norm: 0.001
+wandb_project: Rei-V2
+wandb_entity:
+wandb_watch:
+wandb_name: 0.001_max-grad-norm-4gpu_next
+wandb_log_model:
+evals_per_epoch: 0
+eval_table_size:
+eval_max_new_tokens: 128
+gradient_accumulation_steps: 1
+micro_batch_size: 4
+num_epochs: 2
+optimizer: adamw_bnb_8bit
+lr_scheduler: cosine
+learning_rate: 2e-6
+train_on_inputs: false
+group_by_length: false
+bf16: auto
+fp16:
+tf32: false
+gradient_checkpointing: true
+early_stopping_patience:
+resume_from_checkpoint:
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+s2_attention:
+warmup_steps: 40
+saves_per_epoch: 2
+debug:
+deepspeed: ./deepspeed_configs/zero3_bf16.json
+weight_decay: 0.0001
+fsdp:
+fsdp_config:
+special_tokens:
+   pad_token: <pad>
+</code></pre>
+        </div>
+        <div class="section">
+            <h2>5. Credits</h2>
+            <p>This model was created by the <span class="tooltip">Delta-Vector<span class="tooltiptext">In the middle of the night</span></span> with contributions from:</p>
+            <ul>
+                <li>The <span class="tooltip">LucyKnada<span class="tooltiptext">Shadow Goblin</span></span></li>
+                <li>The <span class="tooltip">Kubernetes_Bad<span class="tooltiptext">Hardware to inference the modelt</span></span></li>
+                <li>The <span class="tooltip">Intervitens<span class="tooltiptext">For providing Hardware for Prevous trains</span></span></li>
+            </ul>
+            <p>Special thanks to:</p>
+            <ul>
+                <li>The <span class="tooltip">Anthracite<span class="tooltiptext">For the support.</span></span></li>
+                <li>The <span class="tooltip">Pygmalion<span class="tooltiptext">For their feedback.</span></span></li>
+            </ul>
+    </div>
+</body>
+</html>