major updates demo v2

app.py

@@ -1,5 +1,6 @@
 import gradio as gr
 from response_db import ResponseDb
+from response_db import get_code
 from create_cache import Game_Cache
 import numpy as np
 from PIL import Image
@@ -7,7 +8,6 @@ import pandas as pd
 import torch
 import pickle
 import uuid
-
 import nltk
 nltk.download('punkt')
 
@@ -19,23 +19,34 @@ css = """
 .msg.bot {background-color:lightgray}
 .na_button {background-color:red;color:red}
 """
+get_window_url_params = """
+    function(url_params) {
+        console.log(url_params);
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+        }
+    """
+quals = {1001:99, 1002:136, 1003:56, 1004:105}
 
 from model.run_question_asking_model import return_modules, return_modules_yn
-question_model, response_model_simul, _, caption_model = return_modules()
-question_model_yn, response_model_simul_yn, _, caption_model_yn = return_modules_yn()
+question_model, response_model_simul, response_model_gtruth, caption_model = return_modules()
+question_model_yn, response_model_simul_yn, response_model_gtruth_yn, caption_model_yn = return_modules_yn()
 
 class Game_Session:
     def __init__(self, taskid, yn, hard_setting):
         self.yn = yn
         self.hard_setting = hard_setting
 
-        global question_model, response_model_simul, caption_model
-        global question_model_yn, response_model_simul_yn, caption_model_yn
+        global question_model, response_model_simul, response_model_gtruth, caption_model
+        global question_model_yn, response_model_simul_yn, response_model_gtruth_yn, caption_model_yn
         self.question_model = question_model
         self.response_model_simul = response_model_simul
+        self.response_model_gtruth = response_model_gtruth
         self.caption_model = caption_model
         self.question_model_yn = question_model_yn
         self.response_model_simul_yn = response_model_simul_yn
+        self.response_model_gtruth_yn = response_model_gtruth_yn
         self.caption_model_yn = caption_model_yn
 
         global image_files, images_np, p_y_x, p_r_qy, p_y_xqr, captions, questions, target_questions
@@ -48,55 +59,69 @@ class Game_Session:
 
     def set_curr_models(self):
         if self.yn:
-            self.curr_question_model, self.curr_caption_model, self.curr_response_model_simul = self.question_model_yn, self.caption_model_yn, self.response_model_simul_yn
+            self.curr_question_model, self.curr_caption_model, self.curr_response_model_simul, self.curr_response_model_gtruth = self.question_model_yn, self.caption_model_yn, self.response_model_simul_yn, self.response_model_gtruth_yn
         else:
-            self.curr_question_model, self.curr_caption_model, self.curr_response_model_simul = self.question_model, self.caption_model, self.response_model_simul
+            self.curr_question_model, self.curr_caption_model, self.curr_response_model_simul, self.curr_response_model_gtruth = self.question_model, self.caption_model, self.response_model_simul, self.response_model_gtruth
 
     def get_next_question(self):
         return self.curr_question_model.select_best_question(self.p_y_x, self.questions, self.images_np, self.captions, self.curr_response_model_simul)
 
+    def get_model_gtruth_response(self, question):
+        return self.response_model_gtruth.get_response(question, self.images_np[0], self.captions[0], self.target_questions, is_a=self.yn)
+
 
 def ask_a_question(input, taskid, gs):
+    # input = gs.get_model_gtruth_response(gs.history[-1])
+
+    if input not in ["n/a", "yes", "no"] and input not in gs.curr_response_model_simul.model.config.label2id:
+        html = "<div class='chatbot'>"
+        for m, msg in enumerate(gs.history):
+            cls = "bot" if m%2 == 0 else "user"
+            html += "<div class='msg {}'> {}</div>".format(cls, msg)
+        html += "</div>"
+        return html, gs, gr.Dropdown.update(visible=True, value=''), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Textbox.update(visible=False), gr.HTML.update(visible=True)
+
     gs.history.append(input)
     gs.p_r_qy = gs.curr_response_model_simul.get_p_r_qy(input, gs.history[-2], gs.images_np, gs.captions)
     gs.p_y_xqr = gs.p_y_x*gs.p_r_qy
     gs.p_y_xqr = gs.p_y_xqr/torch.sum(gs.p_y_xqr)if torch.sum(gs.p_y_xqr) != 0 else torch.zeros_like(gs.p_y_xqr)        
     gs.p_y_x = gs.p_y_xqr
     gs.questions.remove(gs.history[-2])
-    db.add(gs.game_id, taskid, len(gs.history)//2-1, gs.history[-2], gs.history[-1])
+    if taskid not in quals: db.add(gs.game_id, taskid, len(gs.history)//2-1, gs.history[-2], gs.history[-1])
     gs.history.append(gs.get_next_question())
 
     top_prob = torch.max(gs.p_y_x).item()
     top_pred = torch.argmax(gs.p_y_x).item()
-    if top_prob > 0.8: 
+    if top_prob > 0.8 or len(gs.history) > 19: 
         gs.history = gs.history[:-1]
-        db.add(gs.game_id, taskid, len(gs.history)//2, f"Guess: Image {top_pred}", "")
+        if taskid not in quals: db.add(gs.game_id, taskid, len(gs.history)//2, f"Guess: Image {top_pred}", "")
 
     # write some HTML
     html = "<div class='chatbot'>"
     for m, msg in enumerate(gs.history):
-        if msg=="nothing": msg="n/a"
         cls = "bot" if m%2 == 0 else "user"
         html += "<div class='msg {}'> {}</div>".format(cls, msg)
     html += "</div>"
 
     ### Game finished:
-    if top_prob > 0.8:
+    if top_prob > 0.8 or len(gs.history) > 19:
         html += f"<p>The model identified <b>Image {top_pred+1}</b> as the image. Please select a new task ID to continue.</p>"
-        return html, gs, gr.Textbox.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=True), gr.Button.update(visible=True), gr.Number.update(visible=False), gr.Button.update(visible=False)
+        finish_html = "<h2>Congratulations on finishing the game! Please copy the Task Finish Code below to MTurk to complete your task. You can now exit this window.</h2>"
+        return html, gs, gr.Dropdown.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Textbox.update(value=get_code(taskid, gs.history, top_pred), visible=True), gr.HTML.update(value=finish_html, visible=True)
     else:
         if not gs.yn:
-            return html, gs, gr.Textbox.update(visible=True), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False)
+            return html, gs, gr.Dropdown.update(visible=True, value=''), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Textbox.update(visible=False), gr.HTML.update(visible=False)
         else:
-            return html, gs, gr.Textbox.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=True), gr.Button.update(visible=True)
+            return html, gs, gr.Dropdown.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=True), gr.Button.update(visible=True), gr.Textbox.update(visible=False), gr.HTML.update(visible=False)
 
 
 def set_images(taskid):
     pilot_study = pd.read_csv("pilot-study.csv")
     taskid_original = taskid
+    if int(taskid) in quals: taskid = quals[int(taskid)]
     taskid = pilot_study['mscoco-id'].tolist()[int(taskid)]
 
-    with open(f'cache/{int(taskid)}.p', 'rb') as fp:
+    with open(f'cache-soft/{int(taskid)}.p', 'rb') as fp:
         game_cache = pickle.load(fp)
     gs = Game_Session(int(taskid), game_cache.yn, game_cache.hard_setting)
     id1 = f"./mscoco-images/val2014/{game_cache.image_files[0]}"
@@ -123,12 +148,14 @@ def set_images(taskid):
     first_question = gs.curr_question_model.select_best_question(gs.p_y_x, gs.questions, gs.images_np, gs.captions, gs.curr_response_model_simul)
     first_question_html = f"<div class='chatbot'><div class='msg bot'>{first_question}</div></div>"
     gs.history.append(first_question)
-    html = f"<p>Current Task ID: <b>{int(taskid_original)}</b></p>"
+    # html = f"<p>Current Task ID: <b>{int(taskid_original)}</b></p>"
     if not gs.yn:
-        return id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, gs, first_question_html, gr.HTML.update(value=html, visible=True), gr.Textbox.update(visible=True, value=''), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False)
+        return id1, id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, gs, first_question_html, gr.Dropdown.update(visible=True, value=''), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False)
     else:
-        return id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, gs, first_question_html, gr.HTML.update(value=html, visible=True), gr.Textbox.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=True), gr.Button.update(visible=True)
+        return id1, id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, gs, first_question_html, gr.Dropdown.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=True), gr.Button.update(visible=True)
 
+def reset_dropdown():
+    return gr.Dropdown.update(visible=True, value='')
 
 with gr.Blocks(title="Image Q&A Guessing Game", css=css) as demo:
     gr.HTML("<h1>Image Q&A Guessing Game</h1>\
@@ -139,22 +166,19 @@ with gr.Blocks(title="Image Q&A Guessing Game", css=css) as demo:
     The model can see all 10 images and all the questions and answers for the current set of images. It will ask a question based on the available information.<br>\
     <span style='color: #0000ff'>The goal of the model is to accurately guess the correct image (i.e. <b><span style='color: #0000ff'>Image 1</span></b>) in as few turns as possible.<br>\
     Your goal is to help the model guess the image by answering as clearly and accurately as possible.</span><br><br>\
-    <b>Guidelines:</b><br>\
-    <ol style='font-size:120%;'>\
-        <li>It is best to keep your answers short (a single word or a short phrase). No need to answer in full sentences.</li>\
-        <li>If you feel that the question cannot be answered or does not apply to Image 1, please select N/A.</li>\
-    </ol> \
-    <br>\
-    (Note: We are testing multiple game settings. In some instances, the game will be open-ended, while in other instances, the answer choices will be limited to yes/no.)<br></p>\
-    <br>\
-    <h2>Please enter a TaskID to start</h2>")
+    (Note: We are testing multiple game settings. In some instances, the game will be open-ended, while in other instances, the answer choices will be limited to yes/no.)<br><br>\
+    <b>Selecting N/A:</b><br>\
+    <ul style='font-size:120%;'>\
+        <li>In some games, there will be an N/A option. Please select N/A only if the question is unanswerable BECAUSE IT DOES NOT APPLY TO THE IMAGE.</li>\
+        <li>Otherwise, please select the closest possible option.</li>\
+        <li>e.g. Q:\"What is the dog doing?\" Please select N/A if there is no dog in the image.\
+    </ul> \
+    <br>")
 
     with gr.Column():
         with gr.Row():
-            taskid = gr.Number(label="Task ID (Enter a number from 0 to 160)", value=0)
-            start_button = gr.Button("Enter")
-        with gr.Row():
-            task_text = gr.HTML()
+            taskid = gr.Number(label="Task ID (Enter a number from 0 to 160)", visible=False)
+            start_button = gr.Button("Start")
 
     with gr.Column() as img_block:
         with gr.Row():
@@ -172,8 +196,22 @@ with gr.Blocks(title="Image Q&A Guessing Game", css=css) as demo:
     conversation = gr.HTML()
     game_session_state = gr.State()
 
-    answer = gr.Textbox(placeholder="Insert answer here.", label="Answer the given question.", visible=False)
-    null_answer = gr.Textbox("nothing", visible=False)
+    # answer = gr.Textbox(placeholder="Insert answer here.", label="Answer the given question.", visible=False)
+    full_vocab_dict = response_model_simul_yn.model.config.label2id
+    vocab_list_numbers, vocab_list_letters = [], []
+    for i in full_vocab_dict:
+        if i=="None" or i is None: continue
+        if i[0] in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']:
+            vocab_list_numbers.append(i)
+        else:
+            vocab_list_letters.append(i)
+    with gr.Row():
+        answer = gr.Dropdown(vocab_list_letters+vocab_list_numbers, label="Answer the given question.", \
+            info="If you cannot find your exact answer, pick the word you feel would be most appropriate. ONLY SELECT N/A IF THE QUESTION DOES NOT APPLY TO THE IMAGE.", visible=False)
+        clear_box = gr.Button("Reset Selection \n(Use this to clear the dropdown selection.)", visible=False)
+    with gr.Row():
+        vocab_warning = gr.HTML("<h3>The word you typed in is not a valid word in the model vocabulary. Please clear it and select a valid word from the dropdown menu.</h3>", visible=False)
+    null_answer = gr.Textbox("n/a", visible=False)
     yes_answer = gr.Textbox("yes", visible=False)
     no_answer = gr.Textbox("no", visible=False)
 
@@ -185,11 +223,22 @@ with gr.Blocks(title="Image Q&A Guessing Game", css=css) as demo:
         with gr.Row():
             na_box = gr.Button("N/A", visible=False, elem_classes="na_button")
             submit = gr.Button("Submit", visible=False)
+        with gr.Row():
+            reward_code = gr.Textbox("", label="Task Finish Code", visible=False)
+
+    with gr.Column() as img_block0:
+        with gr.Row():
+            img0 = gr.Image(label="Image 1", show_label=True).style(height=700, width=700)
+
     ### Button click events
-    start_button.click(fn=set_images, inputs=taskid, outputs=[img1, img2, img3, img4, img5, img6, img7, img8, img9, img10, game_session_state, conversation, task_text, answer, na_box, submit, taskid, start_button, yes_box, no_box])
-    submit.click(fn=ask_a_question, inputs=[answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, submit, taskid, start_button, yes_box, no_box])
-    na_box.click(fn=ask_a_question, inputs=[null_answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, submit, taskid, start_button, yes_box, no_box])
-    yes_box.click(fn=ask_a_question, inputs=[yes_answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, submit, taskid, start_button, yes_box, no_box])
-    no_box.click(fn=ask_a_question, inputs=[no_answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, submit, taskid, start_button, yes_box, no_box])
+    start_button.click(fn=set_images, inputs=taskid, outputs=[img0, img1, img2, img3, img4, img5, img6, img7, img8, img9, img10, game_session_state, conversation, answer, na_box, clear_box, submit, taskid, start_button, yes_box, no_box])
+    submit.click(fn=ask_a_question, inputs=[answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, clear_box, submit, taskid, start_button, yes_box, no_box, reward_code, vocab_warning])
+    na_box.click(fn=ask_a_question, inputs=[null_answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, clear_box, submit, taskid, start_button, yes_box, no_box, reward_code, vocab_warning])
+    yes_box.click(fn=ask_a_question, inputs=[yes_answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, clear_box, submit, taskid, start_button, yes_box, no_box, reward_code, vocab_warning])
+    no_box.click(fn=ask_a_question, inputs=[no_answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, clear_box, submit, taskid, start_button, yes_box, no_box, reward_code, vocab_warning])
+    clear_box.click(fn=reset_dropdown, inputs=[], outputs=[answer])
+
+    url_params = gr.JSON({}, visible=False, label="URL Params")
+    demo.load(fn = lambda url_params : gr.Number.update(value=int(url_params['p'])), inputs=[url_params], outputs=taskid, _js=get_window_url_params)
 
 demo.launch()

create_cache.py

@@ -37,11 +37,11 @@ def create_cache(taskid):
     global question_model_yn, response_model_simul_yn, caption_model_yn
     if taskid in yn_indices: 
         yn = True
-        curr_question_model, curr_response_model_simul, curr_caption_model = question_model, response_model_simul, caption_model
+        curr_question_model, curr_response_model_simul, curr_caption_model = question_model_yn, response_model_simul_yn, caption_model_yn
         taskid-=40
     else: 
         yn = False
-        curr_question_model, curr_response_model_simul, curr_caption_model = question_model_yn, response_model_simul_yn, caption_model_yn
+        curr_question_model, curr_response_model_simul, curr_caption_model = question_model, response_model_simul, caption_model
     if taskid in hard_setting_indices: 
         hard_setting = True
         image_list_curr = image_list_hard
@@ -65,6 +65,9 @@ def create_cache(taskid):
         image_names.append(image_list_curr[int(taskid)*10+i])
     image_files = [id1, id2, id3, id4, id5, id6, id7, id8, id9, id10]
     image_files = [x[15:] for x in image_files]
+    import os
+    for i in image_files:
+        os.system(f"cp ./../../../data/ms-coco/images/{i} ./mscoco-images/val2014/")
     images_np = [np.asarray(Image.open(f"./mscoco-images/{i}")) for i in image_files]
     images_np = [np.dstack([i]*3) if len(i.shape)==2 else i for i in images_np]
     p_y_x = (torch.ones(10)/10).to(curr_question_model.device)
@@ -74,7 +77,7 @@ def create_cache(taskid):
     first_question = curr_question_model.select_best_question(p_y_x, questions, images_np, captions, curr_response_model_simul)
 
     gc = Game_Cache(curr_question_model.question_bank, image_names, yn, hard_setting)
-    with open(f'./cache{int(taskid)}.p', 'wb') as fp:
+    with open(f'./cache-soft/{int(original_taskid)}.p', 'wb') as fp:
         pickle.dump(gc, fp, protocol=pickle.HIGHEST_PROTOCOL)
 
 if __name__=="__main__":

model/model/question_asking_model.py

@@ -66,13 +66,59 @@ class QuestionAskingModelGPT3(QuestionAskingModel):
         questions = [i+"?" for i in questions]
         return questions
 
+    def generate_gpt4_questions(self, caption):
+        print("generating gpt4 captions")
+        instructions_prompt="You are tasked to produce reasonable questions from a given caption.\
+            The questions you ask must be answerable only using visual information.\
+            As such, never ask questions that involve exact measurement such as \"How tall\", \"How big\", or \"How far\", since these cannot be easily inferred from just looking at an object.\
+            Likewise, never ask questions that involve age (\"How old\"), composition (\"What is it made of\"), material, emotion, or personal relationship.\
+            When asking \"Where\" questions, the subject of your question must be a person or a small object.\
+            Never ask questions that can be answered with yes or no.\
+            When refering to objects, try to be general. For example, instead of saying \"cat\", you should say \"animal\". Instead of saying \"cake\", you should say \"food\".\
+            I repeat, when refering to object, try to be general!\
+            Good questions to ask include general \"What color\", as well as general probing questions such as \"What is the man doing?\" or \"What is the main subject of the image?\"\
+            For each caption, please generate 3-5 reasonable questions."
+        c1="A living room with a couch, coffee table and two large windows with white curtains."
+        q1="What color is the couch? How many windows are there? How many tables are there? What color is the table? What color are the curtains? What is next to the table? What is next to the couch?"
+        c2="A cat is wearing a pink wool hat."
+        q2="What color is the animal? What color is the hat? What is the cat wearing? "
+        c3="A stop sign with a skeleton painted on it, next to a car."
+        q3="What color is the sign? What color is the car? What is next to the sign? What is next to the car? What is on the sign? Where is the car?"
+        c4="A man brushing his teeth with a toothbrush"
+        q4="What is the man doing? Where is the man? What color is the toothbrush?"        
+        prompt = f"{instructions_prompt}\n"
+        prompt+=f"\nCaption: {c1}\nQuestions: {q1}\n"
+        prompt+=f"\nCaption: {c2}\nQuestions: {q2}\n"
+        prompt+=f"\nCaption: {c3}\nQuestions: {q3}\n"
+        prompt+=f"\nCaption: {c4}\nQuestions: {q4}\n"
+        prompt+=f"\nCaption: {caption}\nQuestions:"
+        response = openai.ChatCompletion.create(
+        model="gpt-4-0314",
+        messages=[
+                {"role": "system", "content": instructions_prompt},
+                {"role": "user", "content": c1},
+                {"role": "assistant", "content": q1},
+                {"role": "user", "content": c2},
+                {"role": "assistant", "content": q2},
+                {"role": "user", "content": c3},
+                {"role": "assistant", "content": q3},
+                {"role": "user", "content": c4},
+                {"role": "assistant", "content": q4},
+                {"role": "user", "content": caption}
+            ]
+        )
+        questions = response["choices"][0]["message"]["content"]
+        questions = [i.strip() for i in questions.split('?') if len(i.strip())>1]
+        questions = [i+"?" for i in questions]
+        return questions
+
     def get_questions(self, images, captions, target_idx=0):
         questions = []
         for i, (image, caption) in enumerate(zip(images, captions)):
             if caption in self.gpt3_captions.caption.tolist():
                 image_questions = self.gpt3_captions[self.gpt3_captions.caption==caption].question.tolist()
             else:
-                image_questions = self.generate_gpt3_questions(caption)
+                image_questions = self.generate_gpt4_questions(caption)
                 image_df = pd.DataFrame({'caption':[caption for _ in image_questions], 'question':image_questions})
                 self.gpt3_captions = pd.concat([self.gpt3_captions, image_df])
                 self.gpt3_captions.to_csv(self.gpt3_path, index=False)

model/model/question_model_base.py

@@ -1,8 +1,6 @@
 import torch
-import scipy
 import numpy as np
 import operator
-import random
 from model.model.question_generator import QuestionGenerator
 
 
@@ -39,7 +37,7 @@ class QuestionAskingModel():
                     p_r_qy = p_r_qy.detach().cpu().numpy()
                     is_a_multiplier.append(p_r_qy)
                 if len(is_a_multiplier)==0: is_a_multiplier.append([0 for _ in range(self.num_images)])
-                is_a_multiplier = torch.tensor(scipy.stats.mstats.gmean(is_a_multiplier, axis=0)).to("cuda")
+                is_a_multiplier = torch.tensor(np.mean(is_a_multiplier, axis=0)).to("cuda")
                 if self.multiplier_mode=="hard":
                     for i in range(is_a_multiplier.shape[0]):
                         if is_a_multiplier[i]<0.5: is_a_multiplier[i]=1e-6

model/model/response_model.py

@@ -121,7 +121,7 @@ class ResponseModelVQA(ResponseModel):
             no_cnt = sum([i.lower()=="no" for i in is_a_responses])
             if len(is_a_responses)>0 and no_cnt/len(is_a_responses)>=0.5:
                 if question[:8]=="How many": return "0"
-                else: return "nothing"
+                else: return "n/a"
         if self.vqa_type in ["vilt1", "vilt2"]:    
             outputs = self.model(**encoding)
             logits = torch.nn.functional.softmax(outputs.logits, dim=1)
@@ -165,9 +165,7 @@ class ResponseModelVQA(ResponseModel):
                 response_idx = self.get_response_idx(response)
                 p_r_qy = logits[:,response_idx]
             
-            # check if this 
-            # consider rerunning also without the geometric mean
-            if response=="nothing":
+            if response=="n/a":
                 is_a_questions = self.question_generator.generate_is_there_question_v2(question)
                 for idx, (caption, image) in enumerate(zip(captions, images)):
                     current_responses = []

model/run_question_asking_model.py

@@ -34,7 +34,7 @@ parser.add_argument('--response_type_gtruth', type=str, default='VQA2', choices=
 parser.add_argument('--question_strategy', type=str, default='gpt3', choices=['rule', 'gpt3'])
 parser.add_argument('--multiplier_mode', type=str, default='soft', choices=['soft', 'hard', 'none'])
 
-parser.add_argument('--gpt3_save_name', type=str, default='questions_gpt3')
+parser.add_argument('--gpt3_save_name', type=str, default='questions_gpt4')
 parser.add_argument('--save_name', type=str, default=None)
 parser.add_argument('--verbose', action='store_true')
 args = parser.parse_args()
@@ -43,8 +43,6 @@ args.include_what=True
 args.response_type_simul='VQA1'
 args.response_type_gtruth='VQA3'
 args.multiplier_mode='soft'
-args.sample_strategy='attribute'
-args.attribute_n=1
 args.caption_strategy='gtruth'
 assert_checks(args)
 if args.save_name is None: logger = logging_handler(args.verbose, args.save_name)
@@ -74,8 +72,6 @@ args.include_what=False
 args.response_type_simul='VQA1'
 args.response_type_gtruth='VQA3'
 args.multiplier_mode='none'
-args.sample_strategy='attribute'
-args.attribute_n=1
 args.caption_strategy='gtruth'
 
 print("1. Loading question model ...")

open_db.py

@@ -4,4 +4,12 @@ import pandas as pd
 db = ResponseDb()
 df = pd.DataFrame(list(db.get()))   
 print(df)
-df.to_csv("responses.csv", index=False)
+df.to_csv("responses_new.csv", index=False)
+
+import sqlite3
+import pandas as pd
+
+db = sqlite3.connect("response.db")
+df = pd.read_sql('SELECT * from responses', db)
+print(df)
+df.to_csv("responses_old.csv", index=False)

response_db.py

@@ -24,3 +24,23 @@ class ResponseDb:
 
     def get(self):
         return self.collection.find()
+
+
+def get_code(taskid, history, top_pred):
+    taskid = int(taskid)
+    mongodb_username=os.environ['mongodb_username_2']
+    mongodb_pw=os.environ['mongodb_pw_2']
+    mongodb_cluster_url=os.environ['mongodb_cluster_url_2']
+    client = MongoClient(f"mongodb+srv://{mongodb_username}:{mongodb_pw}@{mongodb_cluster_url}/?retryWrites=true&w=majority")
+    db = client['vqa-codes']
+    collection = db['vqa-codes']
+
+    threshold_dict = {1001: 6, 1002: 2, 1003: 4, 1004: 2}
+    if int(taskid) in threshold_dict:
+        threshold = threshold_dict[int(taskid)]
+        if len(history)<=threshold and top_pred == 0:
+            return list(collection.find({"taskid":int(taskid)}))[0]['code']
+        else:
+            return list(collection.find({"taskid":3000-int(taskid)}))[0]['code']
+
+    return list(collection.find({"taskid":taskid}))[0]['code']