Upload 5 files

mistral-finetune.yaml

@@ -0,0 +1,118 @@
+# Corpus opts:
+data:
+    alpaca:
+        path_src: "alpaca_clean.txt"
+        transforms: [sentencepiece, filtertoolong]
+        weight: 10
+    sharegpt:
+        path_src: "sharegpt.txt"
+        transforms: [sentencepiece, filtertoolong]
+        weight: 10
+    osst1:
+        path_src: "osst1.flattened.txt"
+        transforms: [sentencepiece, filtertoolong]
+        weight: 10
+    valid:
+        path_src: "dataAI/valid.txt"
+        transforms: [sentencepiece]
+
+### Transform related opts:
+#### Subword
+src_subword_model: "tokenizer.model"
+tgt_subword_model: "tokenizer.model"
+
+#### Filter
+src_seq_length: 1792
+tgt_seq_length: 1792
+
+#truncated_decoder: 32
+
+# silently ignore empty lines in the data
+skip_empty_level: silent
+
+# General opts
+train_from: "mistral-onmt.pt"
+save_model: "mistral-onmt-sft"
+save_format: pytorch
+keep_checkpoint: 10
+save_checkpoint_steps: 100
+seed: 1234
+report_every: 10
+train_steps: 1000
+valid_steps: 100
+
+# Batching
+bucket_size: 32768
+num_workers: 1
+world_size: 2
+gpu_ranks: [0,1]
+parallel_mode: tensor_parallel
+batch_type: "tokens"
+batch_size: 1792
+valid_batch_size: 512
+batch_size_multiple: 1
+accum_count: [8]
+accum_steps: [0]
+
+override_opts: true  # CAREFULL this requires all settings to be defined below
+
+share_vocab: true
+save_data: "mistral-7B"
+src_vocab: "mistral.vocab"
+src_vocab_size: 32000
+tgt_vocab_size: 32000
+
+decoder_start_token: '<s>'
+# Optimization
+model_dtype: "fp8"
+apex_opt_level: ""
+optim: "fusedadam"
+learning_rate: 0.0001
+warmup_steps: 100
+decay_method: "none"
+#learning_rate_decay: 0.98
+#start_decay_steps: 100
+#decay_steps: 10
+adam_beta2: 0.998
+max_grad_norm: 0
+label_smoothing: 0.0
+param_init: 0
+param_init_glorot: true
+normalization: "tokens"
+
+#4/8bit
+quant_layers: ['w_1', 'w_2', 'w_3', 'linear_values', 'linear_query', 'linear_keys', 'final_linear']
+quant_type: "bnb_FP4"
+
+#LoRa
+lora_layers: ['linear_values', 'linear_query', 'linear_keys', 'final_linear']
+lora_rank: 4
+lora_dropout: 0.05
+lora_alpha: 8
+lora_embedding: false
+
+# Chekpointing
+#use_ckpting: ['ffn', 'lora']
+
+# Model
+model_task: lm
+encoder_type: transformer_lm
+decoder_type: transformer_lm
+layer_norm: rms
+norm_eps: 1e-5
+pos_ffn_activation_fn: 'silu'
+max_relative_positions: -1
+position_encoding: false
+add_qkvbias: False
+add_ffnbias: False
+parallel_residual: false
+dec_layers: 32
+heads: 32
+num_kv: 8
+sliding_window: 128
+hidden_size: 4096
+word_vec_size: 4096
+transformer_ff: 14336
+dropout_steps: [0]
+dropout: [0.0]
+attention_dropout: [0.0]

mistral-inference.yaml

@@ -0,0 +1,30 @@
+transforms: [sentencepiece]
+
+#### Subword
+src_subword_model: "tokenizer.model"
+tgt_subword_model: "tokenizer.model"
+
+# Model info
+model: "mistral-onmt.pt"
+
+# Inference
+seed: 42
+max_length: 256
+gpu: 0
+batch_type: sents
+batch_size: 4
+world_size: 1
+gpu_ranks: [0]
+#parallel_mode: "tensor_parallel"
+#quant_layers: ['w_1', 'w_2', 'w_3', 'linear_values', 'linear_query', 'linear_keys', 'final_linear']
+#quant_type: "bnb_NF4"
+precision: fp16
+#random_sampling_topk: 1
+#random_sampling_topp: 0.6
+#random_sampling_temp: 0.9
+beam_size: 1
+n_best: 1
+profile: false
+report_time: true
+src: None
+

mistral-onmt.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24c8f66e973cff52e1827fea6d9a2d3f7fffc0709fe6bcfd400d34babefd30ec
+size 14485167003

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443