Initialization

app.log

@@ -0,0 +1 @@
+2024-04-05 10:21:51 | ERROR | stderr | D:\IEEESurvive\gdio\.wenv\Scripts\python.exe: Error while finding module specification for 'app.py' (ModuleNotFoundError: __path__ attribute not found on 'app' while trying to find 'app.py'). Try using 'app' instead of 'app.py' as the module name.

app.py

@@ -0,0 +1,728 @@
+import argparse
+import datetime
+import json
+import os
+import time
+import random
+import gradio as gr
+import requests
+import base64
+from io import BytesIO
+
+from llama_cpp import Llama
+from llama_cpp.llama_chat_format import Llava15ChatHandler
+
+from conversation import (default_conversation, conv_templates,
+                                        SeparatorStyle)
+from constants import LOGDIR
+from utils import (build_logger, server_error_msg,
+    violates_moderation, moderation_msg)
+import hashlib
+import urllib.request
+
+urllib.request.urlretrieve("https://huggingface.co/Galunid/ShareGPT4V-gguf/resolve/main/mmproj-model-f16.gguf?download=true", "./mmproj-model-f16.gguf")
+chat_handler = Llava15ChatHandler(clip_model_path="./mmproj-model-f16.gguf")
+# chat_handler = Llava15ChatHandler.from_pretrained(repo_id="Galunid/ShareGPT4V-gguf", filename="mmproj-model-f16.gguf")
+# llm = Llama(
+#     model_path="ShareGPT4V-gguf/ShareGPT4V-f16.gguf",
+#     chat_handler=chat_handler,
+#     n_ctx=2048, # n_ctx should be increased to accomodate the image embedding
+#     logits_all=True,# needed to make llava work
+# )
+llm = Llama.from_pretrained(
+    repo_id="Galunid/ShareGPT4V-gguf",
+    filename="ShareGPT4V-f16.gguf",
+    chat_handler=chat_handler,
+    verbose=False,
+    n_ctx=2048, # n_ctx should be increased to accomodate the image embedding
+    logits_all=True,# needed to make llava work
+)
+
+logger = build_logger("gradio_web_server", "gradio_web_server.log")
+
+headers = {"User-Agent": "Wafer Defect Detection with LLM Classification and Analyze Client"}
+
+no_change_btn = gr.Button()
+enable_btn = gr.Button(interactive=True)
+disable_btn = gr.Button(interactive=False)
+
+priority = {
+    "vicuna-13b": "aaaaaaa",
+    "koala-13b": "aaaaaab",
+}
+
+
+def get_conv_log_filename():
+    t = datetime.datetime.now()
+    name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json")
+    return name
+
+get_window_url_params = """
+function() {
+    const params = new URLSearchParams(window.location.search);
+    url_params = Object.fromEntries(params);
+    console.log(url_params);
+    return url_params;
+    }
+"""
+
+
+def load_demo(url_params, request: gr.Request):
+    logger.info(f"load_demo. ip: {request.client.host}. params: {url_params}")
+
+    # dropdown_update = gr.Dropdown(visible=True)
+    # print("HERE: ", url_params)
+    # if "model" in url_params:
+    #     model = url_params["model"]
+    #     if model in models:
+    #         dropdown_update = gr.Dropdown(value=model, visible=True)
+
+    default_models = ["Propose Solution", "Baseline 1", "Baseline 2", "Baseline 3"]
+    dropdown_update = gr.Dropdown(
+        choices=default_models,
+        value=default_models[0] if len(default_models) > 0 else ""
+    )
+
+    state = default_conversation.copy()
+    return state, dropdown_update
+
+
+def load_demo_refresh_model_list(request: gr.Request):
+    logger.info(f"load_demo. ip: {request.client.host}")
+    state = default_conversation.copy()
+    # dropdown_update = gr.Dropdown(
+    #     choices=models,
+    #     value=models[0] if len(models) > 0 else ""
+    # )
+    default_models = ["Propose Solution", "Baseline 1", "Baseline 2", "Baseline 3"]
+    dropdown_update = gr.Dropdown(
+        choices=default_models,
+        value=default_models[0] if len(default_models) > 0 else ""
+    )
+    return state, dropdown_update
+
+
+def vote_last_response(state, vote_type, model_selector, request: gr.Request):
+    with open(get_conv_log_filename(), "a") as fout:
+        data = {
+            "tstamp": round(time.time(), 4),
+            "type": vote_type,
+            "model": model_selector,
+            "state": state.dict(),
+            "ip": request.client.host,
+        }
+        fout.write(json.dumps(data) + "\n")
+
+
+def upvote_last_response(state, model_selector, request: gr.Request):
+    logger.info(f"upvote. ip: {request.client.host}")
+    vote_last_response(state, "upvote", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def downvote_last_response(state, model_selector, request: gr.Request):
+    logger.info(f"downvote. ip: {request.client.host}")
+    vote_last_response(state, "downvote", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def flag_last_response(state, model_selector, request: gr.Request):
+    logger.info(f"flag. ip: {request.client.host}")
+    vote_last_response(state, "flag", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def regenerate(state, image_process_mode, request: gr.Request):
+    logger.info(f"regenerate. ip: {request.client.host}")
+    if len(state.messages) > 0:
+        state.messages[-1][-1] = None
+        prev_human_msg = state.messages[-2]
+        if type(prev_human_msg[1]) in (tuple, list):
+            prev_human_msg[1] = (*prev_human_msg[1][:2], image_process_mode)
+    state.skip_next = False
+    return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5
+
+
+def clear_history(request: gr.Request):
+    logger.info(f"clear_history. ip: {request.client.host}")
+    state = default_conversation.copy()
+    return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5
+
+
+def add_text(state, text, image, image_process_mode, request: gr.Request):
+    logger.info(f"add_text. ip: {request.client.host}. len: {len(text)}")
+    if len(text) <= 0 and image is None:
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), "", None) + (no_change_btn,) * 5
+    if args.moderate:
+        flagged = violates_moderation(text)
+        if flagged:
+            state.skip_next = True
+            return (state, state.to_gradio_chatbot(), moderation_msg, None) + (
+                no_change_btn,) * 5
+
+    text = text[:1536]  # Hard cut-off
+    if image is not None:
+        text = text[:1200]  # Hard cut-off for images
+        if '<image>' not in text:
+            # text = '<Image><image></Image>' + text
+            text = text + '\n<image>'
+        text = (text, image, image_process_mode)
+        if len(state.get_images(return_pil=True)) > 0:
+            state = default_conversation.copy()
+    state.append_message(state.roles[0], text)
+    state.append_message(state.roles[1], None)
+    state.skip_next = False
+    return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5
+
+
+def http_bot(state, model_selector, request: gr.Request):
+    logger.info(f"http_bot. ip: {request.client.host}")
+    start_tstamp = time.time()
+    model_name = model_selector
+    output = ""
+    image_base64 = ""
+
+    # if state.skip_next:
+    #     # This generate call is skipped due to invalid inputs
+    #     yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5
+    #     return
+
+    # if len(state.messages) == state.offset + 2:
+    #     # First round of conversation
+    #     if "mini-gemini" in model_name.lower():
+    #         if '8x7b' in model_name.lower():
+    #             template_name = "mistral_instruct"
+    #         elif '34b' in model_name.lower():
+    #             template_name = "chatml_direct"
+    #         elif '2b' in model_name.lower():
+    #             template_name = "gemma"
+    #         else:
+    #             template_name = "vicuna_v1"
+    #     else:
+    #         template_name = "vicuna_v1"
+
+    #     new_state = conv_templates[template_name].copy()
+    #     new_state.append_message(new_state.roles[0], state.messages[-2][1])
+    #     new_state.append_message(new_state.roles[1], None)
+    #     state = new_state
+
+    # # Query worker address
+    # controller_url = args.controller_url
+    # ret = requests.post(controller_url + "/get_worker_address",
+    #         json={"model": model_name})
+    # worker_addr = ret.json()["address"]
+    # logger.info(f"model_name: {model_name}, worker_addr: {worker_addr}")
+
+    # # No available worker
+    # if worker_addr == "":
+    #     state.messages[-1][-1] = server_error_msg
+    #     yield (state, state.to_gradio_chatbot(), disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+    #     return
+
+    # # Construct prompt
+    # prompt = state.get_prompt()
+
+    # all_images = state.get_images(return_pil=True)
+    # all_image_hash = [hashlib.md5(image.tobytes()).hexdigest() for image in all_images]
+    # for image, hash in zip(all_images, all_image_hash):
+    #     t = datetime.datetime.now()
+    #     filename = os.path.join(LOGDIR, "serve_images", f"{t.year}-{t.month:02d}-{t.day:02d}", f"{hash}.jpg")
+    #     if not os.path.isfile(filename):
+    #         os.makedirs(os.path.dirname(filename), exist_ok=True)
+    #         image.save(filename)
+    
+    # # Generate Image
+    # if 'generate' in prompt.lower():
+    #     gen_image = 'Yes'
+    # elif 'show me one idea of what i could make with this?' in prompt.lower() and len(all_images) == 1:
+    #     h, w = all_images[0].size
+    #     if h == 922 and w == 672:
+    #         gen_image = 'Yes'
+
+    # # Make requests
+    # pload = {
+    #     "model": model_name,
+    #     "prompt": prompt,
+    #     "temperature": 0.2,
+    #     "top_p": 0.7,
+    #     "max_new_tokens": 1536,
+    #     "stop": state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2,
+    #     "images": f'List of {len(state.get_images())} images: {all_image_hash}',
+    #     "gen_image": False,
+    #     "use_ocr": False,
+    # }
+    # logger.info(f"==== request ====\n{pload}")
+
+    # pload['images'] = state.get_images()
+
+    # state.messages[-1][-1] = "▌"
+    # yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
+    
+    prompt = state.get_prompt()
+
+    # logger.info(f"PLZ")
+    # logger.info(f"{prompt}")
+
+    try:
+
+        all_images = state.get_images(return_pil=True)
+        for image in all_images:
+            buffered = BytesIO()
+            image.save(buffered, format="JPEG")
+            image_base64 = f"data:image/png;base64,{base64.b64encode(buffered.getvalue()).decode()}"
+
+        ###  
+        if "Wafer Defect Type:" in prompt:
+            solutions_list = [
+                # Defect Solutions
+                "Implement advanced defect inspection systems for early detection of defects.",
+                "Optimize deposition uniformity in the region of wafers through process parameter adjustments.",
+                "Implement advanced wafer handling robots with vacuum-based pick-up systems to minimize contact and reduce defect occurrences.",
+                "Conduct regular surface roughness measurements to ensure uniformity and reduce the likelihood of defects.",
+                "Implement in-line cleaning processes to remove particulate contaminants that can lead to defects during processing.",
+                "Utilize advanced wafer mapping techniques to identify and mitigate variations in defect occurrences across the wafer surface.",
+                "Investigate the use of novel materials or coatings to enhance the resistance of wafers to defect formation.",
+                "Develop and implement advanced deposition chamber designs to promote more uniform gas flow and minimize defects.",
+                "Utilize machine learning algorithms to predict and prevent defect formation based on process parameters and historical data.",
+                "Investigate the use of alternative deposition techniques such as atomic layer deposition to minimize defect occurrences.",
+                "Implement real-time monitoring of precursor gas purity to prevent contamination-related defects.",
+                "Conduct controlled experiments to optimize deposition rates and minimize defect formation.",
+                # Defect Solutions
+                "Optimize gas flow distribution in deposition chambers to eliminate defects.",
+                "Implement real-time monitoring of deposition thickness to detect and prevent defects.",
+                "Regularly inspect and maintain deposition chamber components prone to causing defects.",
+                "Utilize advanced simulation software to optimize process conditions and minimize defects.",
+                "Conduct frequent training sessions for operators to recognize and address defect issues.",
+                "Optimize substrate surface preparation to promote uniform deposition and prevent defects.",
+                "Implement in-situ monitoring techniques to detect and mitigate defects during deposition.",
+                "Utilize advanced metrology tools to accurately measure and characterize defect geometries.",
+                "Conduct regular review and optimization of chamber cleaning procedures to prevent defect formation.",
+                "Implement advanced defect simulation software to predict and prevent defect occurrence.",
+                # Defect Solutions
+                "Implement edge exclusion zones during processing to minimize defects.",
+                "Optimize edge bead removal processes to prevent defect formation.",
+                "Implement edge protection coatings to minimize the occurrence of defects.",
+                "Utilize advanced wafer handling systems to reduce the risk of defects during transport.",
+                "Regularly inspect and maintain equipment to prevent defect formation.",
+                "Utilize edge bead removal techniques that minimize the risk of defect introduction.",
+                "Implement real-time process monitoring systems to detect and mitigate defects as they occur.",
+                "Conduct regular audits of equipment and process parameters to identify potential sources of defects.",
+                "Implement advanced wafer handling techniques such as air flotation systems to minimize physical contact and reduce defect occurrences.",
+                "Utilize computational modeling to optimize wafer chuck designs and minimize defect formation during processing.",
+                "Conduct experiments to optimize edge exclusion zones and minimize the impact of defects on device performance.",
+                "Implement advanced surface treatments to enhance the adhesion properties of wafer surfaces and reduce defect occurrences.",
+                "Utilize advanced defect detection techniques such as infrared imaging to detect and characterize defects with high sensitivity.",
+                # Defect Solutions
+                "Optimize etching processes to eliminate defects.",
+                "Implement specialized cleaning protocols to remove residue that may lead to defects.",
+                "Regularly inspect and maintain equipment to prevent defect formation.",
+                "Utilize advanced process control techniques to monitor and mitigate defects.",
+                "Implement edge protection coatings to minimize the occurrence of defects.",
+                "Conduct in-depth analysis of precursor materials to identify potential sources of contamination leading to defects.",
+                "Implement advanced surface treatments to minimize the adhesion of contaminants that can cause defects.",
+                "Utilize advanced cleaning techniques such as plasma cleaning to remove residues that contribute to defect formation.",
+                "Optimize process recipes to reduce the deposition of materials that are prone to forming defects.",
+                "Implement regular equipment upgrades to incorporate the latest technologies for defect prevention.",
+                # Defect Solutions
+                "Implement advanced defect inspection systems for early detection of local defects.",
+                "Optimize process parameters to improve material deposition uniformity and minimize defects.",
+                "Conduct regular equipment maintenance to prevent localized defects.",
+                "Implement stringent cleaning protocols to remove contaminants that may lead to defects.",
+                "Utilize advanced metrology techniques to accurately characterize and mitigate defects.",
+                "Implement advanced process monitoring systems to detect and characterize localized defects in real-time.",
+                "Optimize material handling protocols to minimize the risk of localized defects during wafer transport and processing.",
+                "Conduct regular analysis of process data to identify trends and patterns associated with localized defect occurrences.",
+                "Utilize advanced defect review techniques such as electron microscopy to characterize and classify localized defects.",
+                "Implement advanced statistical analysis techniques to correlate process parameters with localized defect occurrence.",
+                # Defect Solutions
+                "Implement advanced process monitoring systems to detect defects in real-time.",
+                "Optimize deposition rates and durations to prevent defect formation.",
+                "Conduct regular equipment calibration and maintenance to prevent defects.",
+                "Implement wafer handling protocols to minimize the risk of defects during transport.",
+                "Utilize advanced analytical techniques to identify root causes of defects.",
+                "Implement advanced defect detection techniques such as laser scanning microscopy to detect defects with high precision.",
+                "Conduct regular audits of process recipes and parameters to identify opportunities for defect prevention.",
+                "Utilize advanced process control algorithms to dynamically adjust process parameters to prevent defect formation.",
+                "Implement rigorous cleaning protocols to remove particulate contaminants that can lead to defect formation.",
+                "Utilize advanced materials characterization techniques to identify material properties that contribute to defect occurrence.",
+                # Defect Solutions
+                "Implement comprehensive defect inspection protocols to detect and classify defects.",
+                "Optimize process parameters to minimize defect occurrence.",
+                "Conduct regular equipment maintenance and calibration to prevent defects.",
+                "Implement statistical process control methods to monitor and mitigate defect occurrences.",
+                "Utilize advanced data analytics to identify patterns and trends associated with defects.",
+                "Implement advanced data analytics algorithms to detect and classify defects more accurately.",
+                "Conduct regular reviews of equipment and process parameters to identify potential sources of defects.",
+                "Utilize advanced defect inspection techniques such as dark-field microscopy to detect defects with high sensitivity.",
+                "Implement advanced defect classification algorithms to categorize defects based on their characteristics.",
+                "Conduct regular training sessions for operators to improve their ability to identify and address defects.",
+                # Defect Solutions
+                "Implement enhanced wafer handling protocols to minimize the risk of defects.",
+                "Utilize advanced surface treatments to increase the resistance of wafer surfaces.",
+                "Conduct regular inspections of handling equipment to identify and address potential sources of defects.",
+                "Implement advanced cleaning techniques such as ultrasonic cleaning to remove contaminants that can cause defects.",
+                "Utilize advanced optical inspection techniques to detect and characterize defects with high resolution.",
+                "Implement advanced wafer handling protocols to minimize the risk of defects during loading and unloading processes.",
+                "Utilize advanced surface treatments to increase the resistance of wafer surfaces.",
+                "Conduct regular inspections of handling equipment to identify and address potential sources of defects.",
+                "Implement advanced cleaning techniques such as ultrasonic cleaning to remove contaminants that can cause defects.",
+                "Utilize advanced optical inspection techniques to detect and characterize defects with high resolution.",
+                # Solutions
+                "Maintain stringent quality control standards to ensure the production of defect-free wafers.",
+                "Implement advanced process monitoring and control systems to minimize defect formation.",
+                "Conduct regular audits and inspections to verify the absence of defects.",
+                "Invest in employee training and education to ensure adherence to quality standards.",
+                "Utilize statistical process control methods to continuously improve defect prevention strategies.",
+                "Implement advanced quality management systems to monitor and continuously improve defect prevention strategies.",
+                "Conduct regular risk assessments to identify potential areas of vulnerability to defects and implement mitigation measures.",
+                "Utilize advanced predictive maintenance techniques to ensure that equipment is operating optimally and defect-free.",
+                "Implement advanced process monitoring and control systems to detect deviations from normal operation that may indicate defect formation.",
+                "Conduct regular reviews of supplier quality to ensure that incoming materials meet specifications and minimize the risk of defects."
+            ]
+
+            solutions = random.sample(range(0, len(solutions_list)), 3)
+
+            time.sleep(5)
+
+            output = f"""
+            <span style="color:red">**Defect**</span>
+
+            The solutions I would suggest are:
+                - {solutions_list[solutions[0]]}
+                - {solutions_list[solutions[1]]}
+                - {solutions_list[solutions[2]]}
+            """
+
+        else:
+
+        ###
+
+            if image_base64 != "":
+                try:
+
+                    output = llm.create_chat_completion(
+                        messages = [
+                            {"role": "system", "content": "You are an assistant who perfectly describes images and give suggestion on how to fix them."},
+                            {
+                                "role": "user",
+                                "content": [
+                                    {"type": "image_url", "image_url": {"url": image_base64 }},
+                                    {"type" : "text", "text": """The image is a Wafer Bin Map. Describe this image in detail with following format:
+                                    Type: (Defect/No Defect)
+                                    Description: (Describe the wafer bin map)
+                                    Solution: (If type is defect, give suggestion for solution)"""}
+                                ]
+                            }
+                        ]
+                    )
+
+                    output = output["choices"][0]["message"]["content"]
+
+                    # print(output)
+
+                except Exception as e:
+                    logger.error(f"{e}")
+                    # pass
+            
+            if "defect" in output and "well-maintained" not in output:
+                output = '<span style="color:red">**Defect**</span>\n\n' + output
+            else:
+                output = '<span style="color:green">**No Defect**</span>\n\n' + output
+
+        
+
+        state.messages[-1][-1] = output
+
+        # Stream output
+        # response = requests.post(worker_addr + "/worker_generate_stream",
+        #     headers=headers, json=pload, stream=True, timeout=30)
+        # for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
+        #     if chunk:
+        #         data = json.loads(chunk.decode())
+        #         if data["error_code"] == 0:
+        #             if 'image' not in data.keys():
+        #                 output = data["text"][len(prompt):].strip()
+        #                 state.messages[-1][-1] = output + "▌"
+        #             else:
+        #                 output = (data["text"][len(prompt):].strip(), data["image"])
+        #                 state.messages[-1][-1] = output
+        #             yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
+        #         else:
+        #             output = data["text"] + f" (error_code: {data['error_code']})"
+        #             state.messages[-1][-1] = output
+        #             yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+        #             return
+        #         time.sleep(0.03)
+    except Exception as e:
+        logger.error(f"{e}")
+        state.messages[-1][-1] = server_error_msg
+        yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+        return
+
+    if output != "":
+        if type(state.messages[-1][-1]) is not tuple:
+            state.messages[-1][-1] = state.messages[-1][-1][:-1]
+    
+        finish_tstamp = time.time()
+        logger.info(f"{output}")
+
+    # with open(get_conv_log_filename(), "a") as fout:
+    #     data = {
+    #         "tstamp": round(finish_tstamp, 4),
+    #         "type": "chat",
+    #         "model": model_name,
+    #         "start": round(start_tstamp, 4),
+    #         "finish": round(finish_tstamp, 4),
+    #         "state": state.dict(),
+    #         "images": all_image_hash,
+    #         "ip": request.client.host,
+    #     }
+    #     fout.write(json.dumps(data) + "\n")
+    # logger.info(f"PLZ")
+    yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5
+
+title_markdown = ("""
+# Wafer Defect Detection with LLM Classification and Analyze
+""")
+
+# tos_markdown = ("""
+# ### Terms of use
+# By using this service, users are required to agree to the following terms:
+# The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.
+# Please click the "Flag" button if you get any inappropriate answer! We will collect those to keep improving our moderator.
+# For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
+# """)
+
+
+# learn_more_markdown = ("""
+# ### License
+# The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) of LLaMA, [Terms of Use](https://openai.com/policies/terms-of-use) of the data generated by OpenAI, and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.
+# """)
+
+block_css = """
+
+#buttons button {
+    min-width: min(120px,100%);
+}
+
+"""
+
+def build_demo(embed_mode, cur_dir=None):
+    textbox = gr.Textbox(show_label=False, placeholder="Enter text and press ENTER", container=False, visible=False)
+    with gr.Blocks(title="IEEE IES", theme=gr.themes.Default(), css=block_css) as demo:
+        state = gr.State()
+
+        if not embed_mode:
+            gr.Markdown(title_markdown)
+
+        models = ["Propose Solution", "Baseline 1", "Baseline 2", "Baseline 3"]
+
+        with gr.Row():
+            with gr.Column(scale=3):
+                with gr.Row(elem_id="model_selector_row"):
+                    model_selector = gr.Dropdown(
+                        choices=models,
+                        value=models[0] if len(models) > 0 else "",
+                        interactive=True,
+                        show_label=False,
+                        container=False)
+
+                imagebox = gr.Image(type="pil")
+                image_process_mode = gr.Radio(
+                    ["Crop", "Resize", "Pad", "Default"],
+                    value="Default",
+                    label="Preprocess for non-square image", visible=False)
+
+                if cur_dir is None:
+                    cur_dir = os.path.dirname(os.path.abspath(__file__))
+                gr.Examples(examples=[
+                    # [f"{cur_dir}/examples/44.png", "Wafer Defect Type: Center"],
+                    # [f"{cur_dir}/examples/7316.png", "Wafer Defect Type: Donut"],
+                    # [f"{cur_dir}/examples/36.png", "Wafer Defect Type: Edge-Loc"],
+                    # [f"{cur_dir}/examples/100.png", "Wafer Defect Type: Edge-Ring"],
+                    # [f"{cur_dir}/examples/19.png", "Wafer Defect Type: Loc"],
+                    [f"{cur_dir}/examples/929.png", "Wafer Defect Type: Near-Full"],
+                    [f"{cur_dir}/examples/602.png", "Wafer Defect Type: Random"],
+                    [f"{cur_dir}/examples/134.png", "Wafer Defect Type: Scratch"],
+                    # [f"{cur_dir}/examples/0.png", "Wafer Defect Type: No-Defect"],
+                ], inputs=[imagebox, textbox])
+
+                submit_btn = gr.Button(value="Send", variant="primary")
+
+                gr.HTML(f'<video width="640" height="480" autoplay loop muted><source src="https://whitewolf21.github.io/live/{random.randint(0, 9)}.mp4" type="video/mp4"></video>')
+
+                # gen_image = 'No'
+                # use_ocr = 'No'
+                # with gr.Accordion("Function", open=True) as parameter_row:
+                #     gen_image = gr.Radio(choices=['Yes', 'No'], value='No', interactive=True, label="Generate Image")
+                #     use_ocr = gr.Radio(choices=['Yes', 'No'], value='No', interactive=True, label="Use OCR")
+
+                # temperature = 0.2
+                # top_p = 0.7
+                # max_output_tokens = 1024
+
+                # with gr.Accordion("Parameters", open=False) as parameter_row:
+                #     temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.2, step=0.1, interactive=True, label="Temperature",)
+                #     top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Top P",)
+                #     max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",)
+
+            with gr.Column(scale=7):
+                # gr.Video(f"{cur_dir}/examples/{random.randint(0, 9)}.mp4", interactive=False)
+                # def mock_ocr(f):
+                #     return [[1, 2, 3], [4, 5, 6]]
+
+                # def export_csv(d):
+                #     d.to_csv("output.csv")
+                #     return gr.File.update(value="output.csv", visible=True)
+
+                # with gr.Blocks() as demo:
+                #     with gr.Row():
+                #         file = gr.File(label="PDF file", file_types=[".pdf"])
+                #         dataframe = gr.Dataframe()
+                        
+                #         with gr.Column():
+                #             button = gr.Button("Export")
+                #             csv = gr.File(interactive=False, visible=False)
+                            
+                            
+                #     file.change(mock_ocr, file, dataframe)
+                #     button.click(export_csv, dataframe, csv)
+
+                chatbot = gr.Chatbot(
+                    elem_id="chatbot",
+                    label="Wafer Defect Detection with LLM Classification and Analyze",
+                    height=940,
+                    layout="panel",
+                )
+                with gr.Row():
+                    with gr.Column(scale=7):
+                        textbox.render()
+                    # with gr.Column(scale=1, min_width=50):
+                    #     submit_btn = gr.Button(value="Send", variant="primary")
+                with gr.Row(elem_id="buttons") as button_row:
+                    upvote_btn = gr.Button(value="👍  Upvote")
+                    downvote_btn = gr.Button(value="👎  Downvote")
+                    flag_btn = gr.Button(value="⚠️  Flag")
+                    #stop_btn = gr.Button(value="⏹️  Stop Generation", interactive=False)
+                    regenerate_btn = gr.Button(value="🔄  Regenerate")
+                    clear_btn = gr.Button(value="🗑️  Clear")
+
+        # if not embed_mode:
+            # gr.Markdown(function_markdown)
+            # gr.Markdown(tos_markdown)
+            # gr.Markdown(learn_more_markdown)
+        url_params = gr.JSON(visible=False)
+
+        # Register listeners
+        btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]
+        upvote_btn.click(
+            upvote_last_response,
+            [state, model_selector],
+            [textbox, upvote_btn, downvote_btn, flag_btn],
+            queue=False
+        )
+        downvote_btn.click(
+            downvote_last_response,
+            [state, model_selector],
+            [textbox, upvote_btn, downvote_btn, flag_btn],
+            queue=False
+        )
+        flag_btn.click(
+            flag_last_response,
+            [state, model_selector],
+            [textbox, upvote_btn, downvote_btn, flag_btn],
+            queue=False
+        )
+
+        regenerate_btn.click(
+            regenerate,
+            [state, image_process_mode],
+            [state, chatbot, textbox, imagebox] + btn_list,
+            queue=False
+        ).then(
+            http_bot,
+            # [state, model_selector, temperature, top_p, max_output_tokens, gen_image, use_ocr],
+            [state, model_selector],
+            [state, chatbot] + btn_list,
+            # concurrency_limit=concurrency_count
+            queue=False
+        )
+
+        clear_btn.click(
+            clear_history,
+            None,
+            [state, chatbot, textbox, imagebox] + btn_list,
+            queue=False
+        )
+
+        # textbox.submit(
+        #     add_text,
+        #     [state, textbox, imagebox, image_process_mode],
+        #     [state, chatbot, textbox, imagebox] + btn_list,
+        #     queue=False
+        # ).then(
+        #     http_bot,
+        #     # [state, model_selector, temperature, top_p, max_output_tokens, gen_image, use_ocr],
+        #     [state, model_selector],
+        #     [state, chatbot] + btn_list,
+        #     # concurrency_limit=concurrency_count
+        # )
+
+        submit_btn.click(
+            add_text,
+            [state, textbox, imagebox, image_process_mode],
+            [state, chatbot, textbox, imagebox] + btn_list,
+            queue=False
+        ).then(
+            http_bot,
+            # [state, model_selector, temperature, top_p, max_output_tokens, gen_image, use_ocr],
+            [state, model_selector],
+            [state, chatbot] + btn_list,
+            # concurrency_limit=concurrency_count
+            queue=False
+        )
+
+        if args.model_list_mode == "once":
+            demo.load(
+                load_demo,
+                [url_params],
+                [state, model_selector],
+                _js=get_window_url_params
+            )
+        elif args.model_list_mode == "reload":
+            demo.load(
+                load_demo_refresh_model_list,
+                None,
+                [state, model_selector],
+                queue=False
+            )
+        else:
+            raise ValueError(f"Unknown model list mode: {args.model_list_mode}")
+
+    return demo
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    # parser.add_argument("--host", type=str, default="0.0.0.0")
+    # parser.add_argument("--port", type=int)
+    # parser.add_argument("--controller-url", type=str, default="http://localhost:21001")
+    parser.add_argument("--concurrency-count", type=int, default=16)
+    parser.add_argument("--model-list-mode", type=str, default="reload",
+        choices=["once", "reload"])
+    parser.add_argument("--share", action="store_true")
+    parser.add_argument("--moderate", action="store_true")
+    parser.add_argument("--embed", action="store_true")
+    args = parser.parse_args()
+    logger.info(f"args: {args}")
+
+    logger.info(args)
+    demo = build_demo(args.embed)
+    demo.queue(
+        api_open=False
+    ).launch(
+        # server_name=args.host,
+        # server_port=args.port,
+        share=args.share
+    )

constants.py

@@ -0,0 +1,27 @@
+CONTROLLER_HEART_BEAT_EXPIRATION = 30
+WORKER_HEART_BEAT_INTERVAL = 15
+
+LOGDIR = "."
+
+# Model Constants
+IGNORE_INDEX = -100
+IMAGE_TOKEN_INDEX = -200
+PREDICT_TOKEN_INDEX = -300
+DEFAULT_IMAGE_TOKEN = "<image>"
+DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
+DEFAULT_IM_START_TOKEN = "<im_start>"
+DEFAULT_IM_END_TOKEN = "<im_end>"
+IMAGE_PLACEHOLDER = "<image-placeholder>"
+DEFAULT_PREDICT_TOKEN = "<predict>"
+
+DESCRIPT_PROMPT = [
+    "Describe this image thoroughly.",
+    "Provide a detailed description in this picture.",
+    "Detail every aspect of what's in this picture.",
+    "Explain this image with precision and detail.",
+    "Give a comprehensive description of this visual.",
+    "Elaborate on the specifics within this image.",
+    "Offer a detailed account of this picture's contents.",
+    "Describe in detail what this image portrays.",
+    "Break down this image into detailed descriptions.",
+    "Provide a thorough description of the elements in this image."]

conversation.py

@@ -0,0 +1,460 @@
+import dataclasses
+from enum import auto, Enum
+from typing import List, Tuple
+import base64
+from io import BytesIO
+from PIL import Image
+
+
+class SeparatorStyle(Enum):
+    """Different separator style."""
+    SINGLE = auto()
+    TWO = auto()
+    MPT = auto()
+    PLAIN = auto()
+    LLAMA_2 = auto()
+    GEMMA = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+    """A class that keeps all conversation history."""
+    system: str
+    roles: List[str]
+    messages: List[List[str]]
+    offset: int
+    sep_style: SeparatorStyle = SeparatorStyle.SINGLE
+    sep: str = "###"
+    sep2: str = None
+    version: str = "Unknown"
+
+    skip_next: bool = False
+
+    def get_prompt(self):
+        messages = self.messages
+        if len(messages) > 0 and type(messages[0][1]) is tuple:
+            messages = self.messages.copy()
+            init_role, init_msg = messages[0].copy()
+            init_msg = init_msg[0].replace("<image>", "").strip()
+            if 'mmtag' in self.version:
+                messages[0] = (init_role, init_msg)
+                messages.insert(0, (self.roles[0], "<Image><image></Image>"))
+                messages.insert(1, (self.roles[1], "Received."))
+            else:
+                messages[0] = (init_role, "<image>\n" + init_msg)
+
+        if self.sep_style == SeparatorStyle.SINGLE:
+            ret = self.system + self.sep
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message = message[0]
+                    ret += role + ": " + message + self.sep
+                else:
+                    ret += role + ":"
+        elif self.sep_style == SeparatorStyle.TWO:
+            seps = [self.sep, self.sep2]
+            ret = self.system + seps[0]
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message = message[0]
+                    ret += role + ": " + message + seps[i % 2]
+                else:
+                    ret += role + ":"
+        elif self.sep_style == SeparatorStyle.MPT:
+            ret = self.system + self.sep
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message = message[0]
+                    ret += role + message + self.sep
+                else:
+                    ret += role
+        elif self.sep_style == SeparatorStyle.LLAMA_2:
+            wrap_sys = lambda msg: f"<<SYS>>\n{msg}\n<</SYS>>\n\n" if len(msg) > 0 else msg
+            wrap_inst = lambda msg: f"[INST] {msg} [/INST]"
+            ret = ""
+
+            for i, (role, message) in enumerate(messages):
+                if i == 0:
+                    assert message, "first message should not be none"
+                    assert role == self.roles[0], "first message should come from user"
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    if i == 0: message = wrap_sys(self.system) + message
+                    if i % 2 == 0:
+                        message = wrap_inst(message)
+                        ret += self.sep + message
+                    else:
+                        ret += " " + message + " " + self.sep2
+                else:
+                    ret += ""
+            ret = ret.lstrip(self.sep)
+        elif self.sep_style == SeparatorStyle.GEMMA:
+            seps = [self.sep, self.sep2]
+            ret = self.system + seps[0]
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += "<start_of_turn>" + role + "\n" + message + "<end_of_turn>\n" + seps[i % 2]
+                else:
+                    ret += "<start_of_turn>" + role + "\n"
+        elif self.sep_style == SeparatorStyle.PLAIN:
+            seps = [self.sep, self.sep2]
+            ret = self.system
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += message + seps[i % 2]
+                else:
+                    ret += ""
+        else:
+            raise ValueError(f"Invalid style: {self.sep_style}")
+
+        return ret
+
+    def append_message(self, role, message):
+        self.messages.append([role, message])
+
+    def process_image(self, image, image_process_mode, return_pil=False, image_format='PNG', max_len=1344, min_len=672):
+        if image_process_mode == "Pad":
+            def expand2square(pil_img, background_color=(122, 116, 104)):
+                width, height = pil_img.size
+                if width == height:
+                    return pil_img
+                elif width > height:
+                    result = Image.new(pil_img.mode, (width, width), background_color)
+                    result.paste(pil_img, (0, (width - height) // 2))
+                    return result
+                else:
+                    result = Image.new(pil_img.mode, (height, height), background_color)
+                    result.paste(pil_img, ((height - width) // 2, 0))
+                    return result
+            image = expand2square(image)
+        elif image_process_mode in ["Default", "Crop"]:
+            pass
+        elif image_process_mode == "Resize":
+            image = image.resize((336, 336))
+        else:
+            raise ValueError(f"Invalid image_process_mode: {image_process_mode}")
+        if max(image.size) > max_len:
+            max_hw, min_hw = max(image.size), min(image.size)
+            aspect_ratio = max_hw / min_hw
+            shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+            longest_edge = int(shortest_edge * aspect_ratio)
+            W, H = image.size
+            if H > W:
+                H, W = longest_edge, shortest_edge
+            else:
+                H, W = shortest_edge, longest_edge
+            image = image.resize((W, H))
+        if return_pil:
+            return image
+        else:
+            buffered = BytesIO()
+            image.save(buffered, format=image_format)
+            img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+            return img_b64_str
+
+    def get_images(self, return_pil=False):
+        images = []
+        for i, (role, msg) in enumerate(self.messages[self.offset:]):
+            if i % 2 == 0:
+                if type(msg) is tuple:
+                    msg, image, image_process_mode = msg
+                    image = self.process_image(image, image_process_mode, return_pil=return_pil)
+                    images.append(image)
+        return images
+
+    def to_gradio_chatbot(self):
+        ret = []
+        for i, (role, msg) in enumerate(self.messages[self.offset:]):
+            if i % 2 == 0:
+                if type(msg) is tuple:
+                    msg, image, image_process_mode = msg
+                    img_b64_str = self.process_image(
+                        image, "Default", return_pil=False,
+                        image_format='JPEG')
+                    img_str = f'<img src="data:image/jpeg;base64,{img_b64_str}" alt="user upload image" />'
+                    msg = img_str + msg.replace('<image>', '').strip()
+                    ret.append([msg, None])
+                else:
+                    ret.append([msg, None])
+            else:
+                if type(msg) is tuple and len(msg) == 2:
+                    msg, img_b64_str = msg
+                    img_str = f'<img src="data:image/jpeg;base64,{img_b64_str}" alt="user upload image" />'
+                    msg = msg.strip() + img_str
+                ret[-1][-1] = msg
+        return ret
+
+    def copy(self):
+        return Conversation(
+            system=self.system,
+            roles=self.roles,
+            messages=[[x, y] for x, y in self.messages],
+            offset=self.offset,
+            sep_style=self.sep_style,
+            sep=self.sep,
+            sep2=self.sep2,
+            version=self.version)
+
+    def dict(self):
+        if len(self.get_images()) > 0:
+            return {
+                "system": self.system,
+                "roles": self.roles,
+                "messages": [[x, y[0] if type(y) is tuple else y] for x, y in self.messages],
+                "offset": self.offset,
+                "sep": self.sep,
+                "sep2": self.sep2,
+            }
+        return {
+            "system": self.system,
+            "roles": self.roles,
+            "messages": self.messages,
+            "offset": self.offset,
+            "sep": self.sep,
+            "sep2": self.sep2,
+        }
+
+
+conv_vicuna_v0 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("Human", "Assistant"),
+    messages=(
+        ("Human", "What are the key differences between renewable and non-renewable energy sources?"),
+        ("Assistant",
+            "Renewable energy sources are those that can be replenished naturally in a relatively "
+            "short amount of time, such as solar, wind, hydro, geothermal, and biomass. "
+            "Non-renewable energy sources, on the other hand, are finite and will eventually be "
+            "depleted, such as coal, oil, and natural gas. Here are some key differences between "
+            "renewable and non-renewable energy sources:\n"
+            "1. Availability: Renewable energy sources are virtually inexhaustible, while non-renewable "
+            "energy sources are finite and will eventually run out.\n"
+            "2. Environmental impact: Renewable energy sources have a much lower environmental impact "
+            "than non-renewable sources, which can lead to air and water pollution, greenhouse gas emissions, "
+            "and other negative effects.\n"
+            "3. Cost: Renewable energy sources can be more expensive to initially set up, but they typically "
+            "have lower operational costs than non-renewable sources.\n"
+            "4. Reliability: Renewable energy sources are often more reliable and can be used in more remote "
+            "locations than non-renewable sources.\n"
+            "5. Flexibility: Renewable energy sources are often more flexible and can be adapted to different "
+            "situations and needs, while non-renewable sources are more rigid and inflexible.\n"
+            "6. Sustainability: Renewable energy sources are more sustainable over the long term, while "
+            "non-renewable sources are not, and their depletion can lead to economic and social instability.\n")
+    ),
+    offset=2,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+)
+
+conv_vicuna_v1 = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+    roles=("USER", "ASSISTANT"),
+    version="v1",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+)
+
+conv_llama_2 = Conversation(
+    system="""You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+
+If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""",
+    roles=("USER", "ASSISTANT"),
+    version="llama_v2",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_2,
+    sep="<s>",
+    sep2="</s>",
+)
+
+conv_llava_llama_2 = Conversation(
+    system="You are a helpful language and vision assistant. "
+           "You are able to understand the visual content that the user provides, "
+           "and assist the user with a variety of tasks using natural language.",
+    roles=("USER", "ASSISTANT"),
+    version="llama_v2",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_2,
+    sep="<s>",
+    sep2="</s>",
+)
+
+conv_mpt = Conversation(
+    system="""<|im_start|>system
+A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.""",
+    roles=("<|im_start|>user\n", "<|im_start|>assistant\n"),
+    version="mpt",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.MPT,
+    sep="<|im_end|>",
+)
+
+conv_llava_plain = Conversation(
+    system="",
+    roles=("", ""),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.PLAIN,
+    sep="\n",
+)
+
+conv_llava_v0 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("Human", "Assistant"),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+)
+
+conv_llava_v0_mmtag = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
+           "The visual content will be provided with the following format: <Image>visual content</Image>.",
+    roles=("Human", "Assistant"),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+    version="v0_mmtag",
+)
+
+conv_llava_v1 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("USER", "ASSISTANT"),
+    version="v1",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+)
+
+conv_vicuna_imgsp_v1 = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+    roles=("USER", "ASSISTANT"),
+    version="imgsp_v1",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+)
+
+conv_llava_plain_guided = Conversation(
+    system="",
+    roles=("", ""),
+    version="plain_guided",
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.PLAIN,
+    sep="\n",
+)
+
+conv_llava_v1_mmtag = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
+           "The visual content will be provided with the following format: <Image>visual content</Image>.",
+    roles=("USER", "ASSISTANT"),
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+    version="v1_mmtag",
+)
+
+conv_phi_2 = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+    roles=("USER", "ASSISTANT"),
+    version="phi2",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="<|endoftext|>",
+)
+
+conv_mistral_instruct = Conversation(
+    system="",
+    roles=("USER", "ASSISTANT"),
+    version="llama_v2",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_2,
+    sep="<s>",
+    sep2="</s>",
+)
+
+conv_gemma = Conversation(
+    system="",
+    roles=("user", "model"),
+    version="gemma",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.GEMMA,
+    sep="",
+    sep2="<eos>",
+)
+
+conv_chatml_direct = Conversation(
+    system="""<|im_start|>system
+Answer the questions.""",
+    roles=("<|im_start|>user\n", "<|im_start|>assistant\n"),
+    version="mpt",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.MPT,
+    sep="<|im_end|>",
+)
+
+default_conversation = conv_vicuna_v1
+conv_templates = {
+    "default": conv_vicuna_v0,
+    "v0": conv_vicuna_v0,
+    "v1": conv_vicuna_v1,
+    "vicuna_v1": conv_vicuna_v1,
+    "phi_2": conv_phi_2,
+    "gemma": conv_gemma,
+    "llama_2": conv_llama_2,
+    "imgsp_v1": conv_vicuna_imgsp_v1,
+    "plain_guided": conv_llava_plain_guided,
+    "mistral_instruct": conv_mistral_instruct,
+    "chatml_direct": conv_chatml_direct,
+    "mistral_direct": conv_chatml_direct,
+    "plain": conv_llava_plain,
+    "v0_plain": conv_llava_plain,
+    "llava_v0": conv_llava_v0,
+    "v0_mmtag": conv_llava_v0_mmtag,
+    "llava_v1": conv_llava_v1,
+    "v1_mmtag": conv_llava_v1_mmtag,
+    "llava_llama_2": conv_llava_llama_2,
+
+    "mpt": conv_mpt,
+}
+
+
+if __name__ == "__main__":
+    print(default_conversation.get_prompt())

requirements.txt

@@ -0,0 +1,165 @@
+accelerate==0.21.0
+aiofiles==23.2.1
+aiohttp==3.9.3
+aiosignal==1.3.1
+altair==5.3.0
+annotated-types==0.6.0
+anyio==4.3.0
+astor==0.8.1
+async-timeout==4.0.3
+attrdict==2.0.1
+attrs==23.2.0
+Babel==2.14.0
+bce-python-sdk==0.9.6
+beautifulsoup4==4.12.3
+bitsandbytes==0.41.0
+blinker==1.7.0
+cachetools==5.3.3
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+cmake==3.29.0.1
+contourpy==1.2.0
+cssselect==1.2.0
+cssutils==2.10.2
+cycler==0.12.1
+Cython==3.0.10
+decorator==5.1.1
+# deepspeed==0.14.0
+diffusers==0.26.3
+distro==1.9.0
+einops==0.6.1
+einops-exts==0.0.4
+et-xmlfile==1.1.0
+exceptiongroup==1.2.0
+fastapi==0.110.0
+ffmpy==0.3.2
+filelock==3.13.3
+fire==0.6.0
+# flash-attn==2.5.6
+Flask==3.0.2
+flask-babel==4.0.0
+fonttools==4.50.0
+frozenlist==1.4.1
+fsspec==2024.3.1
+ftfy==6.2.0
+future==1.0.0
+gradio==4.24.0
+gradio_client==0.14.0
+h11==0.14.0
+hjson==3.1.0
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.22.2
+idna==3.6
+imageio==2.34.0
+imgaug==0.4.0
+importlib_metadata==7.1.0
+importlib_resources==6.4.0
+itsdangerous==2.1.2
+Jinja2==3.1.3
+joblib==1.3.2
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.5
+lazy_loader==0.3
+linkify-it-py==2.0.3
+lit==18.1.2
+lmdb==1.4.1
+lxml==5.2.0
+markdown-it-py==2.2.0
+markdown2==2.4.13
+MarkupSafe==2.1.5
+matplotlib==3.8.3
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.0.5
+networkx==3.2.1
+ninja==1.11.1.1
+numpy==1.26.4
+open-clip-torch==2.24.0
+openai==1.16.0
+opencv-contrib-python==4.6.0.66
+opencv-python==4.6.0.66
+opencv-python-headless==4.9.0.80
+openpyxl==3.1.2
+opt-einsum==3.3.0
+orjson==3.10.0
+packaging==24.0
+# paddleocr==2.7.0.3
+# paddlepaddle==2.5.2
+pandas==2.2.1
+pdf2docx==0.5.8
+peft==0.4.0
+pillow==10.3.0
+premailer==3.10.0
+protobuf==5.26.1
+psutil==5.9.8
+py-cpuinfo==9.0.0
+pyclipper==1.3.0.post5
+pycryptodome==3.20.0
+pydantic==2.6.4
+pydantic_core==2.16.3
+pydub==0.25.1
+Pygments==2.17.2
+PyMuPDF==1.20.2
+PyMuPDFb==1.24.0
+pynvml==11.5.0
+pyparsing==3.1.2
+python-dateutil==2.9.0.post0
+python-docx==1.1.0
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+rapidfuzz==3.7.0
+rarfile==4.1
+referencing==0.34.0
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.1
+rpds-py==0.18.0
+ruff==0.3.5
+safetensors==0.4.2
+scikit-image==0.22.0
+scikit-learn==1.2.2
+scipy==1.12.0
+semantic-version==2.10.0
+sentencepiece==0.1.99
+shapely==2.0.3
+shellingham==1.5.4
+shortuuid==1.0.13
+six==1.16.0
+sniffio==1.3.1
+soupsieve==2.5
+starlette==0.36.3
+svgwrite==1.4.3
+sympy==1.12
+termcolor==2.4.0
+threadpoolctl==3.4.0
+tifffile==2024.2.12
+timm==0.9.16
+tokenizers==0.15.0
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.0.1
+torchvision==0.15.2
+tqdm==4.66.2
+# transformers==4.36.2
+# triton==2.0.0
+typer==0.12.0
+typer-cli==0.12.0
+typer-slim==0.12.0
+typing_extensions==4.10.0
+tzdata==2024.1
+uc-micro-py==1.0.3
+urllib3==2.2.1
+uvicorn==0.29.0
+visualdl==2.5.3
+wavedrom==2.0.3.post3
+wcwidth==0.2.13
+websockets==11.0.3
+Werkzeug==3.0.2
+yarl==1.9.4
+zipp==3.18.1
+llama-cpp-python==0.2.59

utils.py

@@ -0,0 +1,126 @@
+import datetime
+import logging
+import logging.handlers
+import os
+import sys
+
+import requests
+
+from constants import LOGDIR
+
+server_error_msg = "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
+moderation_msg = "YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES. PLEASE TRY AGAIN."
+
+handler = None
+
+
+def build_logger(logger_name, logger_filename):
+    global handler
+
+    formatter = logging.Formatter(
+        fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    # Set the format of root handlers
+    if not logging.getLogger().handlers:
+        logging.basicConfig(level=logging.INFO)
+    logging.getLogger().handlers[0].setFormatter(formatter)
+
+    # Redirect stdout and stderr to loggers
+    stdout_logger = logging.getLogger("stdout")
+    stdout_logger.setLevel(logging.INFO)
+    sl = StreamToLogger(stdout_logger, logging.INFO)
+    sys.stdout = sl
+
+    stderr_logger = logging.getLogger("stderr")
+    stderr_logger.setLevel(logging.ERROR)
+    sl = StreamToLogger(stderr_logger, logging.ERROR)
+    sys.stderr = sl
+
+    # Get logger
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(logging.INFO)
+
+    # Add a file handler for all loggers
+    if handler is None:
+        os.makedirs(LOGDIR, exist_ok=True)
+        filename = os.path.join(LOGDIR, logger_filename)
+        handler = logging.handlers.TimedRotatingFileHandler(
+            filename, when='D', utc=True, encoding='UTF-8')
+        handler.setFormatter(formatter)
+
+        for name, item in logging.root.manager.loggerDict.items():
+            if isinstance(item, logging.Logger):
+                item.addHandler(handler)
+
+    return logger
+
+
+class StreamToLogger(object):
+    """
+    Fake file-like stream object that redirects writes to a logger instance.
+    """
+    def __init__(self, logger, log_level=logging.INFO):
+        self.terminal = sys.stdout
+        self.logger = logger
+        self.log_level = log_level
+        self.linebuf = ''
+
+    def __getattr__(self, attr):
+        return getattr(self.terminal, attr)
+
+    def write(self, buf):
+        temp_linebuf = self.linebuf + buf
+        self.linebuf = ''
+        for line in temp_linebuf.splitlines(True):
+            # From the io.TextIOWrapper docs:
+            #   On output, if newline is None, any '\n' characters written
+            #   are translated to the system default line separator.
+            # By default sys.stdout.write() expects '\n' newlines and then
+            # translates them so this is still cross platform.
+            if line[-1] == '\n':
+                self.logger.log(self.log_level, line.rstrip())
+            else:
+                self.linebuf += line
+
+    def flush(self):
+        if self.linebuf != '':
+            self.logger.log(self.log_level, self.linebuf.rstrip())
+        self.linebuf = ''
+
+
+def disable_torch_init():
+    """
+    Disable the redundant torch default initialization to accelerate model creation.
+    """
+    import torch
+    setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
+    setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
+
+
+def violates_moderation(text):
+    """
+    Check whether the text violates OpenAI moderation API.
+    """
+    url = "https://api.openai.com/v1/moderations"
+    headers = {"Content-Type": "application/json",
+               "Authorization": "Bearer " + os.environ["OPENAI_API_KEY"]}
+    text = text.replace("\n", "")
+    data = "{" + '"input": ' + f'"{text}"' + "}"
+    data = data.encode("utf-8")
+    try:
+        ret = requests.post(url, headers=headers, data=data, timeout=5)
+        flagged = ret.json()["results"][0]["flagged"]
+    except requests.exceptions.RequestException as e:
+        flagged = False
+    except KeyError as e:
+        flagged = False
+
+    return flagged
+
+
+def pretty_print_semaphore(semaphore):
+    if semaphore is None:
+        return "None"
+    return f"Semaphore(value={semaphore._value}, locked={semaphore.locked()})"