sleep for long quests

app.py

@@ -3,6 +3,7 @@ import tiktoken
 
 import datetime
 import json
+import time
 import os
 
 openai.api_key = os.getenv('API_KEY')
@@ -14,17 +15,21 @@ def ask(question, history, behavior):
     openai.request_times += 1
     print(f"request times {openai.request_times}: {datetime.datetime.now()}: {question}")
     try:
+        massages = [
+            {"role":"system", "content":content}
+            for content in behavior
+        ] + [
+            {"role":"user" if i%2==0 else "assistant", "content":content}
+            for i,content in enumerate(history + [question])
+        ]
+        length_messages = num_tokens_from_messages(messages)
+        time_penalty = (length_messages-1000)//10
+        if time_penalty>0:
+            print(f"sleep for {time_penalty:.2f}s for too long a quest: {length_messages}")
+            time.sleep(time_penalty)
         response = openai.ChatCompletion.create(
             model="gpt-3.5-turbo",
-            messages=forget_long_term(
-                [
-                    {"role":"system", "content":content}
-                    for content in behavior
-                ] + [
-                    {"role":"user" if i%2==0 else "assistant", "content":content}
-                    for i,content in enumerate(history + [question])
-                ]
-            )
+            messages=forget_long_term(msaages)
         )["choices"][0]["message"]["content"]
         while response.startswith("\n"):
             response = response[1:]
@@ -182,4 +187,5 @@ with gr.Blocks() as demo:
     button_rtr.click(fn=retry, inputs=[txt, state, behavior], outputs=[txt, state, chatbot, downloadfile])
     button_clr.click(fn=lambda :([],[]), inputs=None, outputs=[chatbot, state])
 
+demo.queue(concurrency_count=3, max_size=10)
 demo.launch()