Update app.py

app.py

@@ -73,6 +73,29 @@ def run_gpt(in_prompt,history,model_drop):
     for response in stream:
         resp += response.token.text
     return resp
+    
+def run_idefics(in_prompt,history,model_drop):
+    client = InferenceClient("HuggingFaceM4/idefics-9b-instruct")
+    print(f'history :: {history}')
+    prompt=format_prompt(in_prompt,history)
+    seed = random.randint(1,1111111111111111)
+    print (seed)
+    generate_kwargs = dict(
+        temperature=1.0,
+        max_new_tokens=1048,
+        top_p=0.99,
+        repetition_penalty=1.0,
+        do_sample=True,
+        seed=seed,
+    )
+    content = agent.GENERATE_PROMPT + prompt
+    print(content)
+    stream = client.text_generation(content, **generate_kwargs, stream=True, details=True, return_full_text=False)
+    resp = ""
+    for response in stream:
+        resp += response.token.text
+    print (resp)
+    return resp
 
 
 def generate(purpose,history,chat_drop):
@@ -80,12 +103,13 @@ def generate(purpose,history,chat_drop):
     out_prompt = run_gpt(purpose,history,chat_drop)
     return out_prompt
 
-def describe(purpose,history,chat_drop):
+def describe(purpose,history,image,chat_drop):
     print (history)
-    out_prompt = run_gpt(purpose,history,chat_drop)
+    purpose=f"{purpose},![]({image})"
+    out_prompt = run_idefics(purpose,history,chat_drop)
     return out_prompt
    
-def run(purpose,history,model_drop,chat_drop,choice):
+def run(purpose,history,image,model_drop,chat_drop,choice):
     if choice == "Generate":
         #out_img = infer(out_prompt)
         out_prompt=generate(purpose,history,chat_drop)
@@ -109,24 +133,10 @@ def run(purpose,history,model_drop,chat_drop,choice):
             yield ([(purpose,"an Error occured")],None)
     if choice == "Describe":
         #out_img = infer(out_prompt)
-        out_prompt=describe(purpose,history,model_drop,chat_drop)
+        out_prompt=describe(purpose,history,image,model_drop,chat_drop)
         history.append((purpose,out_prompt))
         yield (history,None)        
-        model=loaded_model[int(model_drop)]
-        out_img=model(out_prompt)
-        print(out_img)
-        url=f'https://johann22-chat-diffusion-describe.hf.space/file={out_img}'
-        print(url)
-        uid = uuid.uuid4()
-        #urllib.request.urlretrieve(image, 'tmp.png')
-        #out=Image.open('tmp.png')
-        r = requests.get(url, stream=True)
-        if r.status_code == 200:
-            out = Image.open(io.BytesIO(r.content))
-            #yield ([(purpose,out_prompt)],out)
-            yield (history,out)
-        else:
-            yield ([(purpose,"an Error occured")],None)        
+        
 ################################################
 style="""
 .top_head{
@@ -161,7 +171,7 @@ with gr.Blocks(css=style) as iface:
                     clear = gr.ClearButton([msg, chatbot])
         with gr.Column(scale=2):
             sumbox=gr.Image(label="Image")
-    sub_b = submit_b.click(run, [msg,chatbot,model_drop,chat_model_drop,agent_choice],[chatbot,sumbox],concurrency_limit=20)
-    sub_e = msg.submit(run, [msg, chatbot,model_drop,chat_model_drop,agent_choice], [chatbot,sumbox],concurrency_limit=20)
+    sub_b = submit_b.click(run, [msg,chatbot,sumbox,model_drop,chat_model_drop,agent_choice],[chatbot,sumbox],concurrency_limit=20)
+    sub_e = msg.submit(run, [msg, chatbot,sumbox,model_drop,chat_model_drop,agent_choice], [chatbot,sumbox],concurrency_limit=20)
     stop_b.click(None,None,None, cancels=[sub_b,sub_e])
 iface.queue(default_concurrency_limit=None).launch()