Update dynamic_cheatsheet/language_model.py

dynamic_cheatsheet/language_model.py

@@ -118,10 +118,15 @@ class LanguageModel:
         print('history\n', history)
         from litellm import num_tokens_from_messages
         
-        tokens_num = num_tokens_from_messages(
-            messages=history,
-            model=self.model_name
-        )
+        # tokens_num = num_tokens_from_messages(
+        #     messages=history,
+        #     model=self.model_name
+        # )
+        try:
+            token_count = litellm.token_counter(model=self.model_name, messages=history)
+            print(f"DEBUG: litellm token_counter for '{model_name}' estimates: {token_count} tokens")
+        except Exception as e:
+            print(f"DEBUG: Error using litellm.token_counter: {e}")
         print("prompt tokens:", tokens_num)
         
         # The self.client is already a partial function with model, api_key, base_url, etc., pre-filled for SambaNova