Upload 13 files

app.py

@@ -1,80 +1,192 @@
 import gradio as gr
 from response_db import StResponseDb
+from create_cache import Game_Cache
+import numpy as np
+from PIL import Image
+import pandas as pd
+import torch
+import pickle
+import uuid
+
 db = StResponseDb()
-a = gr.Number(0)
+css = """
+.chatbot {display:flex;flex-direction:column}
+.msg {padding:4px;margin-bottom:4px;border-radius:4px;width:80%}
+.msg.user {background-color:cornflowerblue;color:white;align-self:self-end}
+.msg.bot {background-color:lightgray}
+.na_button {background-color:red;color:red}
+"""
+
+from model.run_question_asking_model import return_modules, return_modules_yn
+question_model, response_model_simul, _, caption_model = return_modules()
+question_model_yn, response_model_simul_yn, _, caption_model_yn = return_modules_yn()
+
+class Game_Session:
+    def __init__(self, taskid, yn, hard_setting):
+        self.yn = yn
+        self.hard_setting = hard_setting
+
+        global question_model, response_model_simul, caption_model
+        global question_model_yn, response_model_simul_yn, caption_model_yn
+        self.question_model = question_model
+        self.response_model_simul = response_model_simul
+        self.caption_model = caption_model
+        self.question_model_yn = question_model_yn
+        self.response_model_simul_yn = response_model_simul_yn
+        self.caption_model_yn = caption_model_yn
+
+        global image_files, images_np, p_y_x, p_r_qy, p_y_xqr, captions, questions, target_questions
+        self.image_files, self.image_np, self.p_y_x, self.p_r_qy, self.p_y_xqr = None, None, None, None, None
+        self.captions, self.questions, self.target_questions = None, None, None
+
+        self.history = []
+        self.game_id = str(uuid.uuid4())
+        self.set_curr_models()
+
+    def set_curr_models(self):
+        if self.yn:
+            self.curr_question_model, self.curr_caption_model, self.curr_response_model_simul = self.question_model_yn, self.caption_model_yn, self.response_model_simul_yn
+        else:
+            self.curr_question_model, self.curr_caption_model, self.curr_response_model_simul = self.question_model, self.caption_model, self.response_model_simul
+
+    def get_next_question(self):
+        return self.curr_question_model.select_best_question(self.p_y_x, self.questions, self.images_np, self.captions, self.curr_response_model_simul)
+
+
+def ask_a_question(input, taskid, gs):
+    gs.history.append(input)
+    gs.p_r_qy = gs.curr_response_model_simul.get_p_r_qy(input, gs.history[-2], gs.images_np, gs.captions)
+    gs.p_y_xqr = gs.p_y_x*gs.p_r_qy
+    gs.p_y_xqr = gs.p_y_xqr/torch.sum(gs.p_y_xqr)if torch.sum(gs.p_y_xqr) != 0 else torch.zeros_like(gs.p_y_xqr)        
+    gs.p_y_x = gs.p_y_xqr
+    gs.questions.remove(gs.history[-2])
+    db.add(gs.game_id, taskid, len(gs.history)//2-1, gs.history[-2], gs.history[-1])
+    gs.history.append(gs.get_next_question())
+
+    top_prob = torch.max(gs.p_y_x).item()
+    top_pred = torch.argmax(gs.p_y_x).item()
+    if top_prob > 0.8: 
+        gs.history = gs.history[:-1]
+        db.add(gs.game_id, taskid, len(gs.history)//2, f"Guess: Image {top_pred}", "")
 
-def get_next_question(history):
-    if len(history)==2:
-        question = "What is the man doing?"
-    elif len(history)==4:
-        question = "How many apples are there?"
-    else:
-        question = "What color is the cat?"
-    return question
-
-def ask_a_question(input, taskid, history=[]):
-    history.append(input)
-    db.add(int(a.value), taskid, len(history)//2-1, history[-2], history[-1])
-    history.append(get_next_question(history))
-    
     # write some HTML
     html = "<div class='chatbot'>"
-    for m, msg in enumerate(history):
+    for m, msg in enumerate(gs.history):
+        if msg=="nothing": msg="n/a"
         cls = "bot" if m%2 == 0 else "user"
         html += "<div class='msg {}'> {}</div>".format(cls, msg)
     html += "</div>"
-    return html, history
 
+    ### Game finished:
+    if top_prob > 0.8:
+        html += f"<p>The model identified <b>Image {top_pred+1}</b> as the image. Please select a new task ID to continue.</p>"
+        return html, gs, gr.Textbox.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=True), gr.Button.update(visible=True), gr.Number.update(visible=False), gr.Button.update(visible=False)
+    else:
+        if not gs.yn:
+            return html, gs, gr.Textbox.update(visible=True), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False)
+        else:
+            return html, gs, gr.Textbox.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=True), gr.Button.update(visible=True)
 
-css = """
-.chatbox {display:flex;flex-direction:column}
-.msg {padding:4px;margin-bottom:4px;border-radius:4px;width:80%}
-.msg.user {background-color:cornflowerblue;color:white}
-.msg.bot {background-color:lightgray;align-self:self-end}
-.footer {display:none !important}
-"""
 
 def set_images(taskid):
-    id1 = f"images/img_{int(10*(taskid-1)+1)}.jpg"
-    id2 = f"images/img_{int(10*(taskid-1)+2)}.jpg"
-    id3 = f"images/img_{int(10*(taskid-1)+3)}.jpg"
-    id4 = f"images/img_{int(10*(taskid-1)+4)}.jpg"
-    id5 = f"images/img_{int(10*(taskid-1)+5)}.jpg"
-    id6 = f"images/img_{int(10*(taskid-1)+6)}.jpg"
-    id7 = f"images/img_{int(10*(taskid-1)+7)}.jpg"
-    id8 = f"images/img_{int(10*(taskid-1)+8)}.jpg"
-    id9 = f"images/img_{int(10*(taskid-1)+9)}.jpg"
-    id10 = f"images/img_{int(10*(taskid-1)+10)}.jpg"
-    first_question = "How many dogs are there?"
+    pilot_study = pd.read_csv("pilot-study.csv")
+    taskid_original = taskid
+    taskid = pilot_study['mscoco-id'].tolist()[int(taskid)]
+
+    with open(f'cache/{int(taskid)}.p', 'rb') as fp:
+        game_cache = pickle.load(fp)
+    gs = Game_Session(int(taskid), game_cache.yn, game_cache.hard_setting)
+    id1 = f"./mscoco-images/val2014/{game_cache.image_files[0]}"
+    id2 = f"./mscoco-images/val2014/{game_cache.image_files[1]}"
+    id3 = f"./mscoco-images/val2014/{game_cache.image_files[2]}"
+    id4 = f"./mscoco-images/val2014/{game_cache.image_files[3]}"
+    id5 = f"./mscoco-images/val2014/{game_cache.image_files[4]}"
+    id6 = f"./mscoco-images/val2014/{game_cache.image_files[5]}"
+    id7 = f"./mscoco-images/val2014/{game_cache.image_files[6]}"
+    id8 = f"./mscoco-images/val2014/{game_cache.image_files[7]}"
+    id9 = f"./mscoco-images/val2014/{game_cache.image_files[8]}"
+    id10 = f"./mscoco-images/val2014/{game_cache.image_files[9]}"    
+
+    gs.image_files = [id1, id2, id3, id4, id5, id6, id7, id8, id9, id10]
+    gs.image_files = [x[15:] for x in gs.image_files]
+    gs.images_np = [np.asarray(Image.open(f"./mscoco-images/{i}")) for i in gs.image_files]
+    gs.images_np = [np.dstack([i]*3) if len(i.shape)==2 else i for i in gs.images_np]
+
+    gs.p_y_x = (torch.ones(10)/10).to(gs.curr_question_model.device)
+    gs.captions = gs.curr_caption_model.get_captions(gs.image_files)
+    gs.questions, gs.target_questions = gs.curr_question_model.get_questions(gs.image_files, gs.captions, 0)
+    gs.curr_question_model.reset_question_bank()
+    gs.curr_question_model.question_bank = game_cache.question_dict
+    first_question = gs.curr_question_model.select_best_question(gs.p_y_x, gs.questions, gs.images_np, gs.captions, gs.curr_response_model_simul)
     first_question_html = f"<div class='chatbot'><div class='msg bot'>{first_question}</div></div>"
-    a.value = a.value+1
-    return id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, [first_question], first_question_html
+    gs.history.append(first_question)
+    html = f"<p>Current Task ID: <b>{int(taskid_original)}</b></p>"
+    if not gs.yn:
+        return id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, gs, first_question_html, gr.HTML.update(value=html, visible=True), gr.Textbox.update(visible=True, value=''), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False)
+    else:
+        return id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, gs, first_question_html, gr.HTML.update(value=html, visible=True), gr.Textbox.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Number.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=True), gr.Button.update(visible=True)
+
 
-with gr.Blocks(css=css) as demo:
+with gr.Blocks(title="Image Q&A Guessing Game", css=css) as demo:
+    gr.HTML("<h1>Image Q&A Guessing Game</h1>\
+    <p style='font-size:120%;'>\
+    Imagine you are playing 20-questions with an AI model.<br>\
+    The AI model plays the role of the question asker. You play the role of the responder. <br>\
+    There are 10 images. <b>Your image is Image 1</b>. The other images are distraction images.\
+    The model can see all 10 images and all the questions and answers for the current set of images. It will ask a question based on the available information.<br>\
+    <span style='color: #0000ff'>The goal of the model is to accurately guess the correct image (i.e. <b><span style='color: #0000ff'>Image 1</span></b>) in as few turns as possible.<br>\
+    Your goal is to help the model guess the image by answering as clearly and accurately as possible.</span><br><br>\
+    <b>Guidelines:</b><br>\
+    <ol style='font-size:120%;'>\
+        <li>It is best to keep your answers short (a single word or a short phrase). No need to answer in full sentences.</li>\
+        <li>If you feel that the question cannot be answered or does not apply to Image 1, please select N/A.</li>\
+    </ol> \
+    <br>\
+    (Note: We are testing multiple game settings. In some instances, the game will be open-ended, while in other instances, the answer choices will be limited to yes/no.)<br></p>\
+    <br>\
+    <h2>Please enter a TaskID to start</h2>")
+
+    with gr.Column():
+        with gr.Row():
+            taskid = gr.Number(label="Task ID (Enter a number from 0 to 160)", value=0)
+            start_button = gr.Button("Enter")
+        with gr.Row():
+            task_text = gr.HTML()
 
     with gr.Column() as img_block:
         with gr.Row():
-            img1 = gr.Image()
-            img2 = gr.Image()
-            img3 = gr.Image()
-            img4 = gr.Image()
-            img5 = gr.Image()
+            img1 = gr.Image(label="Image 1", show_label=True)
+            img2 = gr.Image(label="Image 2", show_label=True)
+            img3 = gr.Image(label="Image 3", show_label=True)
+            img4 = gr.Image(label="Image 4", show_label=True)
+            img5 = gr.Image(label="Image 5", show_label=True)
         with gr.Row():
-            img6 = gr.Image()
-            img7 = gr.Image()
-            img8 = gr.Image()
-            img9 = gr.Image()
-            img10 = gr.Image()
+            img6 = gr.Image(label="Image 6", show_label=True)
+            img7 = gr.Image(label="Image 7", show_label=True)
+            img8 = gr.Image(label="Image 8", show_label=True)
+            img9 = gr.Image(label="Image 9", show_label=True)
+            img10 = gr.Image(label="Image 10", show_label=True)
     conversation = gr.HTML()
-    history_log = gr.State([])
+    game_session_state = gr.State()
 
+    answer = gr.Textbox(placeholder="Insert answer here.", label="Answer the given question.", visible=False)
+    null_answer = gr.Textbox("nothing", visible=False)
+    yes_answer = gr.Textbox("yes", visible=False)
+    no_answer = gr.Textbox("no", visible=False)
+
+    with gr.Column():
+        with gr.Row():
+            yes_box = gr.Button("Yes", visible=False)
+            no_box = gr.Button("No", visible=False)
     with gr.Column():
         with gr.Row():
-            taskid = gr.Number(label="Task ID (Choose from [1,2,3])")
-            btn1 = gr.Button("Enter")
-        btn1.click(set_images, inputs=taskid, outputs=[img1, img2, img3, img4, img5, img6, img7, img8, img9, img10, history_log, conversation])
-    answer = gr.inputs.Textbox(placeholder="Insert answer here.", label="Answer the given question.")
-    submit = gr.Button("Submit")
-    submit.click(fn=ask_a_question, inputs=[answer, taskid, history_log], outputs=[conversation, history_log])
-
-demo.launch()
+            na_box = gr.Button("N/A", visible=False, elem_classes="na_button")
+            submit = gr.Button("Submit", visible=False)
+    ### Button click events
+    start_button.click(fn=set_images, inputs=taskid, outputs=[img1, img2, img3, img4, img5, img6, img7, img8, img9, img10, game_session_state, conversation, task_text, answer, na_box, submit, taskid, start_button, yes_box, no_box])
+    submit.click(fn=ask_a_question, inputs=[answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, submit, taskid, start_button, yes_box, no_box])
+    na_box.click(fn=ask_a_question, inputs=[null_answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, submit, taskid, start_button, yes_box, no_box])
+    yes_box.click(fn=ask_a_question, inputs=[yes_answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, submit, taskid, start_button, yes_box, no_box])
+    no_box.click(fn=ask_a_question, inputs=[no_answer, taskid, game_session_state], outputs=[conversation, game_session_state, answer, na_box, submit, taskid, start_button, yes_box, no_box])
+
+demo.launch()

create_cache.py

@@ -0,0 +1,83 @@
+import gradio as gr
+import numpy as np
+from PIL import Image
+import torch
+import pickle
+
+class Game_Cache:
+    def __init__(self, question_dict, image_files, yn, hard_setting):
+        self.question_dict = question_dict
+        self.image_files = image_files
+        self.yn = yn
+        self.hard_setting = hard_setting
+
+image_list = []
+with open('./mscoco/mscoco_images.txt', 'r') as f:
+    for line in f.readlines():
+        image_list.append(line.strip())
+image_list_hard = []
+with open('./mscoco/mscoco_images_attribute_n=1.txt', 'r') as f:
+    for line in f.readlines():
+        image_list_hard.append(line.strip())
+
+yn_indices = list(range(40,80))+list(range(120,160))
+hard_setting_indices = list(range(80,160))
+
+
+from model.run_question_asking_model import return_modules, return_modules_yn
+global image_files, images_np, p_y_x, p_r_qy, p_y_xqr, captions, questions, target_questions 
+global question_model, response_model_simul, caption_model
+question_model, response_model_simul, _, caption_model = return_modules()
+global question_model_yn, response_model_simul_yn, caption_model_yn
+question_model_yn, response_model_simul_yn, _, caption_model_yn = return_modules_yn()
+
+def create_cache(taskid):
+    original_taskid = taskid
+    global question_model, response_model_simul, caption_model
+    global question_model_yn, response_model_simul_yn, caption_model_yn
+    if taskid in yn_indices: 
+        yn = True
+        curr_question_model, curr_response_model_simul, curr_caption_model = question_model, response_model_simul, caption_model
+        taskid-=40
+    else: 
+        yn = False
+        curr_question_model, curr_response_model_simul, curr_caption_model = question_model_yn, response_model_simul_yn, caption_model_yn
+    if taskid in hard_setting_indices: 
+        hard_setting = True
+        image_list_curr = image_list_hard
+        taskid -= 80
+    else: 
+        hard_setting = False    
+        image_list_curr = image_list
+    
+    id1 = f"./mscoco-images/val2014/{image_list_curr[int(taskid)*10+0]}"
+    id2 = f"./mscoco-images/val2014/{image_list_curr[int(taskid)*10+1]}"
+    id3 = f"./mscoco-images/val2014/{image_list_curr[int(taskid)*10+2]}"
+    id4 = f"./mscoco-images/val2014/{image_list_curr[int(taskid)*10+3]}"
+    id5 = f"./mscoco-images/val2014/{image_list_curr[int(taskid)*10+4]}"
+    id6 = f"./mscoco-images/val2014/{image_list_curr[int(taskid)*10+5]}"
+    id7 = f"./mscoco-images/val2014/{image_list_curr[int(taskid)*10+6]}"
+    id8 = f"./mscoco-images/val2014/{image_list_curr[int(taskid)*10+7]}"
+    id9 = f"./mscoco-images/val2014/{image_list_curr[int(taskid)*10+8]}"
+    id10 = f"./mscoco-images/val2014/{image_list_curr[int(taskid)*10+9]}"
+    image_names = []
+    for i in range(10):
+        image_names.append(image_list_curr[int(taskid)*10+i])
+    image_files = [id1, id2, id3, id4, id5, id6, id7, id8, id9, id10]
+    image_files = [x[15:] for x in image_files]
+    images_np = [np.asarray(Image.open(f"./mscoco-images/{i}")) for i in image_files]
+    images_np = [np.dstack([i]*3) if len(i.shape)==2 else i for i in images_np]
+    p_y_x = (torch.ones(10)/10).to(curr_question_model.device)
+    captions = curr_caption_model.get_captions(image_files)
+    questions, target_questions = curr_question_model.get_questions(image_files, captions, 0)
+    curr_question_model.reset_question_bank()
+    first_question = curr_question_model.select_best_question(p_y_x, questions, images_np, captions, curr_response_model_simul)
+
+    gc = Game_Cache(curr_question_model.question_bank, image_names, yn, hard_setting)
+    with open(f'./cache{int(taskid)}.p', 'wb') as fp:
+        pickle.dump(gc, fp, protocol=pickle.HIGHEST_PROTOCOL)
+
+if __name__=="__main__":
+    for i in range(160):
+        create_cache(i)
+

model/__init__.py

@@ -0,0 +1,6 @@
+import model.run_question_asking_model
+import model.model.caption_model
+import model.model.question_asking_model
+import model.model.question_generator
+import model.model.question_model_base
+import model.model.response_model

model/model/caption_model.py

@@ -0,0 +1,89 @@
+import pandas as pd
+from pycocotools.coco import COCO
+
+def get_caption_model(args, question_model):
+    if args.caption_strategy=="simple":
+        return CaptionModelSimple(question_model)
+    elif args.caption_strategy=="granular":
+        return CaptionModelGranular()
+    elif args.caption_strategy=="gtruth":
+        return CaptionModelCOCO()
+    else:
+        raise ValueError(f"{args.caption_strategy} is not a valid caption strategy.")
+
+
+class CaptionModel():
+    # Class for the other CaptionModels to inherit from
+    def __init__(self):
+        pass
+    
+    def get_captions(self, images, **kwargs):
+        raise NotImplemented
+
+class CaptionModelCOCO():
+    # Ground truth annotations from COCO dataset
+    def __init__(self):
+        dataDir='./mscoco'
+        val_file = '{}/annotations/captions_val2014.json'.format(dataDir)
+        self.coco_caps_val = COCO(val_file)
+        val_file = '{}/annotations/instances_val2014.json'.format(dataDir)
+        self.coco_anns_val = COCO(val_file)
+
+    def get_captions(self, images, return_all=False):
+        captions = []
+        for i, image in enumerate(images):
+            image_id = int(image.split('_')[-1].split('.')[0].lstrip("0"))
+            annIds = self.coco_caps_val.getAnnIds(imgIds=image_id)
+            anns_val = self.coco_caps_val.loadAnns(annIds)
+            # annIds = self.coco_caps_train.getAnnIds(imgIds=image_id)
+            # anns_train = self.coco_caps_train.loadAnns(annIds)
+            # anns = anns_val + anns_train
+            anns = anns_val
+            anns = [d['caption'] for d in anns]
+            if return_all: captions.append(anns)
+            else: captions.append(anns[0])
+        return captions
+
+    def get_subjects(self, images):
+        subjects = []
+        for i, image in enumerate(images):
+            image_id = int(image.split('_')[-1].split('.')[0].lstrip("0"))
+            annIds = self.coco_anns_val.getAnnIds(imgIds=image_id)
+            anns_val = self.coco_anns_val.loadAnns(annIds)
+            cats_val = list(set([d['category_id'] for d in anns_val]))
+            annIds = self.coco_caps_train.getAnnIds(imgIds=image_id)
+            anns_train = self.coco_caps_train.loadAnns(annIds)
+            cats_train = list(set([d['category_id'] for d in anns_train]))
+            cats = self.coco_anns_val.loadCats(ids=cats_val+cats_train)
+            cats1, cats2 = [d['supercategory'] for d in cats], [d['name'] for d in cats]
+            cats = list(set(cats1+cats2))
+            subjects.append(cats)
+        return subjects
+
+
+class CaptionModelSimple():
+    def __init__(self, qa_model):
+        self.qa_model = qa_model            
+
+    def get_captions(self, images):
+        captions = []
+        for i, image in enumerate(images):
+            caption = self.qa_model.generate_description(image, images)
+            captions.append(caption)
+        return captions
+        
+class CaptionModelGranular():
+    def __init__(self):
+        df_train = pd.read_json("captions/coco_train_captions.jsonl", lines=True)
+        df_val = pd.read_json("captions/coco_val_captions.jsonl", lines=True)
+        self.caption_dict = {}
+        for i in range(len(df_train)):
+            self.caption_dict[str(df_train.image_id[i])] = df_train.caption[i]
+        for i in range(len(df_val)):
+            self.caption_dict[str(df_val.image_id[i])] = df_val.caption[i]
+    
+    def get_captions(self, images):
+        captions = []
+        for i, image in enumerate(images):
+            captions.append(self.caption_dict[image.split('.')[0].split('_')[-1].lstrip('0')])
+        return captions

model/model/question_asking_model.py

@@ -0,0 +1,83 @@
+import random
+import pandas as pd
+from model.model.question_model_base import QuestionAskingModel
+import openai
+
+def get_question_model(args):
+    if args.question_strategy=="rule":
+        return QuestionAskingModelSimple(args)
+    elif args.question_strategy=="gpt3":
+        return QuestionAskingModelGPT3(args)
+    else:
+        raise ValueError(f"{args.question_strategy} is not a valid question strategy.")
+
+
+class QuestionAskingModelSimple(QuestionAskingModel):
+    def __init__(self, args):
+        super(QuestionAskingModelSimple, self).__init__(args)
+    
+    def get_questions(self, images, captions, target_idx=0):
+        questions = []
+        for i, (image, caption) in enumerate(zip(images, captions)):
+            image_questions = self.question_generator.generate_is_there_question(caption)
+            if i == target_idx: target_questions = image_questions
+            questions += image_questions
+        questions = list(set(questions))
+        # random.shuffle(questions)
+        return questions, target_questions
+
+
+class QuestionAskingModelGPT3(QuestionAskingModel):
+    def __init__(self, args):
+        super(QuestionAskingModelGPT3, self).__init__(args)
+        self.gpt3_path = f"data/{args.gpt3_save_name}.csv"
+        try: self.gpt3_captions = pd.read_csv(self.gpt3_path)      # cache locally to save GPT3 compute
+        except: self.gpt3_captions = pd.DataFrame({"caption":[], "question":[]})
+    
+    def generate_gpt3_questions(self, caption):
+        # c1="Two people sitting on grass. The man in a blue shirt is playing a guitar and is on the left. The woman on the right is eating a sandwich."
+        # q1="What are the two people doing? How many people are there? What color is the man's shirt? Where is the man? Where is the woman? What is the man doing? What is the woman doing? Who is playing the guitar? Who is eating a sandwich?"
+        # c2="There is a table in the middle of the room. On the left there is a bowl of red apples. To the right of the bowl, there is a glass of juice, as well as a bowl of salad. Beside the table there is a bookshelf with ten books of various colors."
+        # q2="What is beside the table? What is on the left of the table? What color are the apples? How many bowls are there? What is inside the bowl? What is inside the glass? How many books are there? What color are the books?"
+        c1="A living room with a couch, coffee table and two large windows with white curtains."
+        q1="What color is the couch? How many windows are there? How many tables are there? What color is the table? What color are the curtains? What is next to the table? What is next to the couch?"
+        c2="A large, shiny, stainless, side by side refrigerator in a kitchen."
+        q2="Where is the refrigerator? What color is the refrigerator?"
+        c3="A stop sign with a skeleton painted on it, next to a car."
+        q3="What color is the sign? What color is the car? What is next to the sign? What is next to the car? What is on the sign? Where is the car?"
+        c4="A man brushing his teeth with a toothbrush"
+        q4="What is the man doing? Where is the man? What color is the toothbrush? How many people are there?"
+        prompt=f"Generate questions for the following caption:\nCaption: {c1}\nQuestions: {q1}\n"
+        prompt+=f"Generate questions for the following caption:\nCaption: {c2}\nQuestions: {q2}\n"
+        prompt+=f"Generate questions for the following caption:\nCaption: {c3}\nQuestions: {q3}\n"
+        prompt+=f"Generate questions for the following caption:\nCaption: {c4}\nQuestions: {q4}\n"
+        prompt+=f"Generate questions for the following caption:\nCaption: {caption}\nQuestions:"
+        response = openai.Completion.create(
+            model="text-davinci-003",
+            prompt=prompt,
+            temperature=0,
+            max_tokens=1024,
+            top_p=1,
+            frequency_penalty=0,
+            presence_penalty=0
+        )
+        questions = response["choices"][0]["text"]
+        questions = [i.strip() for i in questions.split('?') if len(i.strip())>1]
+        questions = [i+"?" for i in questions]
+        return questions
+
+    def get_questions(self, images, captions, target_idx=0):
+        questions = []
+        for i, (image, caption) in enumerate(zip(images, captions)):
+            if caption in self.gpt3_captions.caption.tolist():
+                image_questions = self.gpt3_captions[self.gpt3_captions.caption==caption].question.tolist()
+            else:
+                image_questions = self.generate_gpt3_questions(caption)
+                image_df = pd.DataFrame({'caption':[caption for _ in image_questions], 'question':image_questions})
+                self.gpt3_captions = pd.concat([self.gpt3_captions, image_df])
+                self.gpt3_captions.to_csv(self.gpt3_path, index=False)
+            if i == target_idx: target_questions = image_questions
+            questions += image_questions
+        questions = list(set(questions))
+        # random.shuffle(questions)
+        return questions, target_questions

model/model/question_generator.py

@@ -0,0 +1,194 @@
+import spacy
+import nltk.tree
+import collections
+
+import benepar
+### Run this the first time if benepar_en3 is not yet downloaded
+# benepar.download('benepar_en3')
+
+def find_word(tree, kind, parents=None):
+    if parents is None:
+        parents = []
+    if not isinstance(tree, nltk.tree.Tree):
+        return None, None
+    if tree.label() == kind:
+        return tree[0], parents
+    parents.append(tree)
+    for st in tree:
+        n, p = find_word(st, kind, parents)
+        if n is not None:
+            return n, p
+    parents.pop()
+    return None, None
+
+def find_subtrees(tree, kind, blocking_kinds=()):
+    result = []
+    if not isinstance(tree, nltk.tree.Tree):
+        return result
+    if tree.label() == kind:
+        result.append(tree)
+    if tree.label() not in blocking_kinds:
+        for st in tree:
+            result.extend(find_subtrees(st, kind))
+    return result
+
+def tree_to_str(tree, transform=lambda w: w):
+    l = []
+    def list_words(tree):
+        if isinstance(tree, str):
+            l.append(transform(tree))
+        else:
+            for st in tree:
+                list_words(st)
+    list_words(tree)
+    if l[-1] == '.':
+        l = l[:-1]
+    return ' '.join(l)
+
+def tree_to_nouns(tree, transform=lambda w: w):
+    l = []
+    def list_words(tree, noun=False):
+        if isinstance(tree, str):
+            if noun == True:
+                l.append(transform(tree))
+        else:
+            for st in tree:
+                if not isinstance(st, str):
+                    if st.label() == 'NN':
+                        noun = True
+                    else:
+                        noun = False
+                list_words(st, noun)
+    list_words(tree)
+    if l[-1] == '.':
+        l = l[:-1]
+    return l
+
+def make_determinate(w):
+    if w.lower() in ('a', 'an'):
+        return 'the'
+    return w
+
+def make_indeterminate(w):
+    if w.lower() in ('the', 'his', 'her', 'their', 'its'):
+        return 'a'
+    return w
+
+def pluralize(singular, plural, number):
+    if number <= 1:
+        return singular
+    return plural
+
+def count_labels(tree):
+    counts = collections.defaultdict(int)
+    def update_counts(node):
+        counts[node.label()] += 1
+        for child in node:
+            if isinstance(child, nltk.tree.Tree):
+                update_counts(child)
+    update_counts(tree)
+    return counts
+
+def get_number(tree):
+    if not isinstance(tree, nltk.tree.Tree):
+        return 0
+    if tree.label() == 'NN':
+        return 1
+    if tree.label() == 'NNS':
+        return 2
+    first_noun_number = None
+    n_np_children = 0
+    for subtree in tree:
+        label = subtree.label() if isinstance(subtree, nltk.tree.Tree) else None
+        if label == 'NP':
+            n_np_children += 1
+        if label in ('NP', 'NN', 'NNS') and first_noun_number is None:
+            first_noun_number = get_number(subtree)
+    if tree.label() == 'NP' and n_np_children > 1:
+        return 2
+    return first_noun_number or 0
+
+def is_present_continuous(verb):
+    return verb.endswith('ing')
+
+class QuestionGenerator:
+    def __init__(self):
+        self.parser = benepar.Parser("benepar_en3")
+
+    def generate_what_question(self, s):
+        tree = self.parser.parse(s)[0]
+        questions = []
+        try:
+            if len(tree) >= 2 and tree[0].label() == 'NP' and tree[1].label() == 'VP':
+                np = tree[0]
+                verb = None
+                vp = tree[1]
+                vnp = None
+
+                while True:
+                    verb, verb_parents = find_word(vp, 'VBG')
+                    if verb is None:
+                        break
+                    if is_present_continuous(verb):
+                        if len(verb_parents[-1]) > 1:
+                            vnp = verb_parents[-1][1]
+                        break
+                    else:
+                        vp = verb_parents[-1][1]
+                to_be = pluralize('is', 'are', get_number(np))
+                if vnp is not None and vnp.label() == 'NP':
+                    questions.append(('What {} {} {}?'
+                                     .format(to_be, 
+                                             tree_to_str(np, make_determinate).lower(), 
+                                             verb),
+                                     tree_to_nouns(vnp)[-1]))
+        except Exception as e:
+            print(e)
+        return questions
+
+    def generate_is_there_question(self, s):
+        tree = self.parser.parse(s)
+        questions = []
+        nps = find_subtrees(tree, 'NP', ('PP',))
+        for np in nps:
+            only_child_label = len(np) == 1 and next(iter(np)).label()
+            if only_child_label in ('PRP', 'EX'):
+                continue
+            try:
+                to_be = pluralize('Is', 'Are', get_number(np))
+                questions.append('{} there {}?'
+                                 .format(to_be, 
+                                         tree_to_str(np, make_indeterminate).lower()))
+            except Exception as e:
+                print(e)
+        return questions
+
+    def generate_is_there_question_v2(self, s):
+        tree = self.parser.parse(s)
+        questions = []
+        nps = find_subtrees(tree, 'NP', ('PP',))
+        for np in nps:
+            only_child_label = len(np) == 1 and next(iter(np)).label()
+            if only_child_label in ('PRP', 'EX'):
+                continue
+            try:
+                to_be = pluralize('Is', 'Are', get_number(np))
+                questions.append('{} there {}?'
+                                 .format(to_be, 
+                                         tree_to_str(np, make_indeterminate).lower()))
+            except Exception as e:
+                print(e)
+        if len(questions)==0:
+            nps = find_subtrees(tree, 'NNS', ('PP',))
+            for np in nps:
+                only_child_label = len(np) == 1
+                if only_child_label in ('PRP', 'EX'):
+                    continue
+                try:
+                    to_be = pluralize('Is', 'Are', get_number(np))
+                    questions.append('{} there {}?'
+                                    .format(to_be, 
+                                            tree_to_str(np, make_indeterminate).lower()))
+                except Exception as e:
+                    print(e)
+        return questions

model/model/question_model_base.py

@@ -0,0 +1,85 @@
+import torch
+import scipy
+import numpy as np
+import operator
+import random
+from model.model.question_generator import QuestionGenerator
+
+
+class QuestionAskingModel():
+    def __init__(self, args):
+        self.device = args.device
+        self.include_what = args.include_what
+        self.max_length = 128
+        self.eps = 1e-25
+        self.multiplier_mode = args.multiplier_mode
+        self.num_images = args.num_images
+        
+        # Initialize question generation model
+        class Namespace:
+            def __init__(self, **kwargs):
+                self.__dict__.update(kwargs)
+        
+        self.question_generator = QuestionGenerator()
+        self.reset_question_bank()
+                
+    def get_questions(self, images, captions, target_idx=0):
+        raise NotImplemented
+
+    def get_negative_information_gain(self, p_y_x, question, image_set, caption_set, response_model):
+        if question in self.question_bank:
+            p_r_qy = self.question_bank[question]
+        else:
+            is_a_questions = self.question_generator.generate_is_there_question_v2(question)
+            is_a_multiplier = []
+            if self.multiplier_mode != "none":
+                for is_a_q in is_a_questions:
+                    # print(f"IsA Question: {is_a_q}")
+                    p_r_qy = response_model.get_p_r_qy(None, is_a_q, image_set, caption_set, is_a=True)
+                    p_r_qy = p_r_qy.detach().cpu().numpy()
+                    is_a_multiplier.append(p_r_qy)
+                if len(is_a_multiplier)==0: is_a_multiplier.append([0 for _ in range(self.num_images)])
+                is_a_multiplier = torch.tensor(scipy.stats.mstats.gmean(is_a_multiplier, axis=0)).to("cuda")
+                if self.multiplier_mode=="hard":
+                    for i in range(is_a_multiplier.shape[0]):
+                        if is_a_multiplier[i]<0.5: is_a_multiplier[i]=1e-6
+                        else: is_a_multiplier[i]=0.9
+                elif self.multiplier_mode=="soft":
+                    pass
+            elif self.multiplier_mode == "none":
+                is_a_multiplier = torch.tensor([1 for _ in range(self.num_images)]).to("cuda")
+            p_r_qy = response_model.get_p_r_qy(None, question, image_set, caption_set)
+            p_r_qy = torch.stack([is_a_multiplier*p_r_qy[r] for r in range(len(p_r_qy))])
+            for i in range(self.num_images):
+                if is_a_multiplier[i] < 0.5:
+                    p_r_qy[i] = 1-is_a_multiplier
+                else:
+                    p_r_qy[i] *= is_a_multiplier
+            if self.multiplier_mode=="none":
+                p_r_qy = response_model.get_p_r_qy(None, question, image_set, caption_set)
+            if not self.include_what: 
+                p_r_qy = [p_r_qy, 1-p_r_qy]
+            self.question_bank[question] = p_r_qy
+
+        p_y_xqr = torch.stack([p_y_x*p_r_qy[r] for r in range(len(p_r_qy))])
+        p_y_xqr = [p_y_xqr[r]/torch.sum(p_y_xqr[r]) if torch.sum(p_y_xqr[r]) != 0 \
+                    else [0]*len(p_y_xqr[r]) for r in range(len(p_y_xqr))]
+        return torch.sum(torch.stack([p_r_qy[r]*p_y_x*torch.log2(1/(p_y_xqr[r]+self.eps)) for r in range(len(p_r_qy))]))
+
+    def get_question_ranking(self, p_y_x, question_set, image_set, caption_set, response_model):
+        H_y_rxq = [0]*len(question_set)
+
+        for i, question in enumerate(question_set):
+            H_y_rxq[i] = self.get_negative_information_gain(p_y_x, question, image_set, caption_set, response_model)
+           
+        IG = - torch.stack(H_y_rxq).unsqueeze(1)
+        ranked_questions = sorted(zip(list(IG.data.cpu().numpy()), question_set),
+                                  key = operator.itemgetter(0))[::-1]
+        return ranked_questions
+
+    def select_best_question(self, p_y_x, question_set, image_set, caption_set, response_model):
+        ranked_questions = self.get_question_ranking(p_y_x, question_set, image_set, caption_set, response_model)
+        return ranked_questions[0][1]
+    
+    def reset_question_bank(self):
+        self.question_bank = {}

model/model/response_model.py

@@ -0,0 +1,190 @@
+import torch
+import torch.nn as nn
+from transformers import AutoProcessor, AutoTokenizer, AutoModelForQuestionAnswering, pipeline
+from transformers import ViltProcessor, ViltForQuestionAnswering
+from transformers import BlipProcessor, BlipForQuestionAnswering
+from sentence_transformers import SentenceTransformer
+import openai
+from PIL import Image
+
+def get_response_model(args, response_type):
+    if response_type=="QA":
+        return ResponseModelQA(args.device, args.include_what)
+    elif response_type=="VQA1":
+        return ResponseModelVQA(args.device, args.include_what, args.question_generator, vqa_type="vilt1")
+    elif response_type=="VQA2":
+        return ResponseModelVQA(args.device, args.include_what, args.question_generator, vqa_type="vilt2")
+    elif response_type=="VQA3":
+        return ResponseModelVQA(args.device, args.include_what, args.question_generator, vqa_type="blip")
+    elif response_type=="VQA4":
+        return ResponseModelVQA(args.device, args.include_what, args.question_generator, vqa_type="git")
+    else:
+        raise ValueError(f"{response_type} is not a valid response type.")
+
+
+class ResponseModel(nn.Module):
+    # Class for the other ResponseModels to inherit from
+    def __init__(self, device, include_what):
+        super(ResponseModel, self).__init__()
+        self.device = device
+        self.include_what = include_what
+        self.model = None
+    
+    def get_response(self, question, image, caption, target_questions, **kwargs):
+        raise NotImplemented
+
+    def get_p_r_qy(self, response, question, images, captions, **kwargs):
+        raise NotImplemented
+
+class ResponseModelQA(ResponseModel):
+    def __init__(self, device, include_what):
+        super(ResponseModelQA, self).__init__(device, include_what)
+        if not self.include_what:
+            tokenizer = AutoTokenizer.from_pretrained("AmazonScience/qanlu")
+            model = AutoModelForQuestionAnswering.from_pretrained("AmazonScience/qanlu")
+            self.model = pipeline('question-answering', model=model, tokenizer=tokenizer, device=0)   # remove device=0 for cpu
+        elif self.include_what: 
+            tokenizer = AutoTokenizer.from_pretrained("deepset/roberta-base-squad2")
+            model = AutoModelForQuestionAnswering.from_pretrained("deepset/roberta-base-squad2")
+            self.model_wh = pipeline('question-answering', model=model, tokenizer=tokenizer, device=0)   # remove device=0 for cpu
+
+    def get_response(self, question, image, caption, target_questions, **kwargs):
+        if self.include_what:
+            answer = self.model({'context':caption, 'question':question})
+            return answer['answer'].split(' ')[-1]
+        else:
+            answer = self.model({'context':f"Yes. No. {caption}", 'question':question})
+            response, score = answer['answer'], answer['score']
+            if score>0.5:
+                response = response.lower().replace('.','')
+                if "yes" in response.split() and "no" not in response.split():
+                    response = 'yes'
+                elif "no" in response.split() and "yes" not in response.split():
+                    response = 'no'
+                else:
+                    response = 'yes' if question in target_questions else 'no'
+            else:
+                response = 'yes' if question in target_questions else 'no'
+            return response
+
+    def get_p_r_qy(self, response, question, images, captions, **kwargs):
+        if self.include_what:
+            raise NotImplementedError
+        else:
+            p_r_qy = torch.zeros(len(captions))
+            qa_input = {'context':[f"Yes. No. {c}" for c in captions], 'question':[question for _ in captions]}
+            answers = self.model(qa_input)
+            for idx, answer in enumerate(answers):
+                curr_ans, score = answer['answer'], answer['score']
+                if curr_ans.strip() in ["Yes.", "No."]:
+                    if response==None:
+                        if curr_ans.strip()=="No.": p_r_qy[idx] = 1-score
+                        if curr_ans.strip()=="Yes.": p_r_qy[idx] = score
+                    elif curr_ans.strip().lower().replace('.','')==response: p_r_qy[idx]=score
+                    else: p_r_qy[idx]=1-score
+                else:
+                    p_r_qy[idx]=0.5
+            return p_r_qy.to(self.device)
+
+class ResponseModelVQA(ResponseModel):
+    def __init__(self, device, include_what, question_generator, vqa_type):
+        super(ResponseModelVQA, self).__init__(device, include_what)
+        self.vqa_type = vqa_type
+        self.question_generator = question_generator
+        self.sentence_transformer = SentenceTransformer('all-MiniLM-L6-v2')
+        if vqa_type=="vilt1":
+            self.processor = ViltProcessor.from_pretrained("dandelin/vilt-b32-finetuned-vqa")
+            self.model = ViltForQuestionAnswering.from_pretrained("dandelin/vilt-b32-finetuned-vqa").to(device)
+            self.vocab = list(self.model.config.label2id.keys())
+        elif vqa_type=="vilt2":
+            self.processor = AutoProcessor.from_pretrained("tufa15nik/vilt-finetuned-vqasi")
+            self.model = ViltForQuestionAnswering.from_pretrained("tufa15nik/vilt-finetuned-vqasi").to("cuda")
+            self.vocab = list(self.model.config.label2id.keys())
+        elif vqa_type=="blip":
+            self.processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
+            self.model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
+        elif vqa_type=="git":
+            pass
+        else:
+            raise ValueError(f"{vqa_type} is not a valid vqa_type.")
+
+
+    def get_response(self, question, image, caption, target_questions, is_a=False):
+        encoding = self.processor(image, question, return_tensors="pt").to(self.device)
+        if is_a==False:
+            is_a_questions = self.question_generator.generate_is_there_question_v2(question)
+            is_a_responses = []
+            if question in ["What is in the photo?", "What is in the picture?", "What is in the background?"]:
+                is_a_questions = []
+            for q in is_a_questions:
+                is_a_responses.append(self.get_response(q, image, caption, target_questions, is_a=True))
+            no_cnt = sum([i.lower()=="no" for i in is_a_responses])
+            if len(is_a_responses)>0 and no_cnt/len(is_a_responses)>=0.5:
+                if question[:8]=="How many": return "0"
+                else: return "nothing"
+        if self.vqa_type in ["vilt1", "vilt2"]:    
+            outputs = self.model(**encoding)
+            logits = torch.nn.functional.softmax(outputs.logits, dim=1)
+            idx = logits.argmax(-1).item()
+            response = self.model.config.id2label[idx]
+            response = response.lower().replace('.','').strip()
+        elif self.vqa_type == "blip":
+            outputs = self.model.generate(**encoding)
+            response = self.processor.decode(outputs[0], skip_special_tokens=True)
+        return response 
+
+    def get_p_r_qy(self, response, question, images, captions, is_a=False):
+        p_r_qy = torch.zeros(len(captions))
+        logits_arr = []
+        for i, image in enumerate(images):
+            with torch.no_grad():
+                if len(question) > 150: question=""     # ignore question if too long
+                encoding = self.processor(image, question, return_tensors="pt").to(self.device)
+                outputs = self.model(**encoding)
+            logits = torch.nn.functional.softmax(outputs.logits, dim=1)
+            idx = logits.argmax(-1).item()
+            curr_response = self.model.config.id2label[idx]
+            curr_response = curr_response.lower().replace('.','').strip()
+            if self.include_what==False or is_a==True:
+                if response==None:
+                    if curr_response=="yes": p_r_qy[i] = logits[0][3].item()
+                    elif curr_response=="no": p_r_qy[i] = 1-logits[0][9].item()
+                    else: p_r_qy[i] = 0.5
+                elif curr_response==response: p_r_qy[i] = logits[0][idx].item()
+                else: p_r_qy[i] = 1-logits[0][idx].item()
+            else:
+                logits_arr.append(logits)
+        if self.include_what==False or is_a==True: 
+            return p_r_qy.to(self.device)
+        else:
+            logits = torch.concat(logits_arr)
+            if response==None:
+                top_answers = logits.argmax(1)
+                p_r_qy = logits[:,top_answers]
+            else:
+                response_idx = self.get_response_idx(response)
+                p_r_qy = logits[:,response_idx]
+            
+            # check if this 
+            # consider rerunning also without the geometric mean
+            if response=="nothing":
+                is_a_questions = self.question_generator.generate_is_there_question_v2(question)
+                for idx, (caption, image) in enumerate(zip(captions, images)):
+                    current_responses = []
+                    for is_a_q in is_a_questions:
+                        current_responses.append(self.get_response(is_a_q, image, caption, None, is_a=True))
+                    no_cnt = sum([i.lower()=="no" for i in current_responses])
+                    if len(current_responses)>0 and no_cnt/len(current_responses)>=0.5:
+                        p_r_qy[idx] = 1.0
+            return p_r_qy.to(self.device)
+
+    def get_response_idx(self, response):
+        if response in self.model.config.label2id:
+            return self.model.config.label2id[response]
+        else:
+            embs = self.sentence_transformer.encode(self.vocab, convert_to_tensor=True)
+            emb_response = self.sentence_transformer.encode([response], convert_to_tensor=True)
+            dists = torch.nn.CosineSimilarity(-1)(emb_response, embs)
+            best_response_idx = torch.argmax(dists)
+            best_response = self.vocab[best_response_idx]
+            return self.model.config.label2id[best_response]

model/run_question_asking_model.py

@@ -0,0 +1,186 @@
+from model.model.question_asking_model import get_question_model
+from model.model.caption_model import get_caption_model
+from model.model.response_model import get_response_model
+
+import torch
+from torch.utils.data import Dataset, DataLoader
+from PIL import Image
+
+import argparse
+import random
+from tqdm.auto import tqdm
+import numpy as np
+import pandas as pd
+import logging
+from model.utils import logging_handler, image_saver, assert_checks
+
+random.seed(123)
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--device', type=str, default='cuda')
+parser.add_argument('--include_what', action='store_true')
+parser.add_argument('--target_idx', type=int, default=0)
+parser.add_argument('--max_num_questions', type=int, default=25)
+parser.add_argument('--num_images', type=int, default=10)
+parser.add_argument('--beam', type=int, default=1)
+parser.add_argument('--num_samples', type=int, default=100)
+parser.add_argument('--threshold', type=float, default=0.9)
+
+parser.add_argument('--caption_strategy', type=str, default='simple', choices=['simple', 'granular', 'gtruth'])
+parser.add_argument('--sample_strategy', type=str, default='random', choices=['random', 'attribute', 'clip'])
+parser.add_argument('--attribute_n', type=int, default=1)               # Number of attributes to split
+parser.add_argument('--response_type_simul', type=str, default='VQA1', choices=['simple', 'QA', 'VQA1', 'VQA2', 'VQA3', 'VQA4'])
+parser.add_argument('--response_type_gtruth', type=str, default='VQA2', choices=['simple', 'QA', 'VQA1', 'VQA2', 'VQA3', 'VQA4'])
+parser.add_argument('--question_strategy', type=str, default='gpt3', choices=['rule', 'gpt3'])
+parser.add_argument('--multiplier_mode', type=str, default='soft', choices=['soft', 'hard', 'none'])
+
+parser.add_argument('--gpt3_save_name', type=str, default='questions_gpt3')
+parser.add_argument('--save_name', type=str, default=None)
+parser.add_argument('--verbose', action='store_true')
+args = parser.parse_args()
+args.question_strategy='gpt3'
+args.include_what=True
+args.response_type_simul='VQA1'
+args.response_type_gtruth='VQA3'
+args.multiplier_mode='soft'
+args.sample_strategy='attribute'
+args.attribute_n=1
+args.caption_strategy='gtruth'
+assert_checks(args)
+if args.save_name is None: logger = logging_handler(args.verbose, args.save_name)
+args.load_response_model = True
+
+print("1. Loading question model ...")
+question_model = get_question_model(args)
+args.question_generator = question_model.question_generator
+print("2. Loading response model simul ...")
+response_model_simul = get_response_model(args, args.response_type_simul)
+response_model_simul.to(args.device)
+print("3. Loading response model gtruth ...")
+response_model_gtruth = get_response_model(args, args.response_type_gtruth)
+response_model_gtruth.to(args.device)
+print("4. Loading caption model ...")
+caption_model = get_caption_model(args, question_model)
+
+
+
+def return_modules():
+    return question_model, response_model_simul, response_model_gtruth, caption_model 
+
+
+
+args.question_strategy='rule'
+args.include_what=False
+args.response_type_simul='VQA1'
+args.response_type_gtruth='VQA3'
+args.multiplier_mode='none'
+args.sample_strategy='attribute'
+args.attribute_n=1
+args.caption_strategy='gtruth'
+
+print("1. Loading question model ...")
+question_model_yn = get_question_model(args)
+args.question_generator_yn = question_model_yn.question_generator
+print("2. Loading response model simul ...")
+response_model_simul_yn = get_response_model(args, args.response_type_simul)
+response_model_simul_yn.to(args.device)
+print("3. Loading response model gtruth ...")
+response_model_gtruth_yn = get_response_model(args, args.response_type_gtruth)
+response_model_gtruth_yn.to(args.device)
+print("4. Loading caption model ...")
+caption_model_yn = get_caption_model(args, question_model_yn)
+
+
+def return_modules_yn():
+    return question_model_yn, response_model_simul_yn, response_model_gtruth_yn, caption_model_yn 
+
+
+
+# args.question_strategy='gpt3'
+# args.include_what=True
+# args.response_type_simul='VQA1'
+# args.response_type_gtruth='VQA3'
+# args.multiplier_mode='none'
+# args.sample_strategy='attribute'
+# args.attribute_n=1
+# args.caption_strategy='gtruth'
+# assert_checks(args)
+# if args.save_name is None: logger = logging_handler(args.verbose, args.save_name)
+# args.load_response_model = True
+
+# print("1. Loading question model ...")
+# question_model = get_question_model(args)
+# args.question_generator = question_model.question_generator
+# print("2. Loading response model simul ...")
+# response_model_simul = get_response_model(args, args.response_type_simul)
+# response_model_simul.to(args.device)
+# print("3. Loading response model gtruth ...")
+# response_model_gtruth = get_response_model(args, args.response_type_gtruth)
+# response_model_gtruth.to(args.device)
+# print("4. Loading caption model ...")
+# caption_model = get_caption_model(args, question_model)
+
+# # dataloader = DataLoader(dataset=ReferenceGameData(split='test', 
+# #                                                   num_images=args.num_images, 
+# #                                                   num_samples=args.num_samples,
+# #                                                   sample_strategy=args.sample_strategy,
+# #                                                   attribute_n=args.attribute_n))
+
+# def return_modules():
+#     return question_model, response_model_simul, response_model_gtruth, caption_model 
+# # game_lens, game_preds = [], []
+# for t, batch in enumerate(tqdm(dataloader)):
+#     image_files = [image[0] for image in batch['images'][:args.num_images]]
+#     image_files = [str(i).split('/')[1] for i in image_files]
+#     with open("mscoco_images_attribute_n=1.txt", 'a') as f:
+#         for i in image_files:
+#             f.write(str(i)+"\n")
+    # images = [np.asarray(Image.open(f"./../../../data/ms-coco/images/{i}")) for i in image_files]
+    # images = [np.dstack([i]*3) if len(i.shape)==2 else i for i in images]
+#     p_y_x = (torch.ones(args.num_images)/args.num_images).to(question_model.device)
+
+#     if args.save_name is not None: 
+#         logger = logging_handler(args.verbose, args.save_name, t)
+#         image_saver(images, args.save_name, t)
+
+#     captions = caption_model.get_captions(image_files)
+#     questions, target_questions = question_model.get_questions(image_files, captions, args.target_idx)
+    
+#     question_model.reset_question_bank()
+#     logger.info(questions)
+#     for idx, c in enumerate(captions): logger.info(f"Image_{idx}: {c}")
+    
+#     num_questions_original = len(questions)
+#     for j in range(min(args.max_num_questions, num_questions_original)):
+#         # Select best question
+#         question = question_model.select_best_question(p_y_x, questions, images, captions, response_model_simul)
+#         logger.info(f"Question: {question}")
+
+#         # Ask the question and get the model's response
+#         response = response_model_gtruth.get_response(question, images[args.target_idx], captions[args.target_idx], target_questions, is_a=1-args.include_what)
+#         logger.info(f"Response: {response}")
+
+#         # Update the probabilities
+#         p_r_qy = response_model_simul.get_p_r_qy(response, question, images, captions)
+#         logger.info(f"P(r|q,y):\n{np.around(p_r_qy.cpu().detach().numpy(), 3)}")
+#         p_y_xqr = p_y_x*p_r_qy
+#         p_y_xqr = p_y_xqr/torch.sum(p_y_xqr)if torch.sum(p_y_xqr) != 0 else torch.zeros_like(p_y_xqr)        
+#         p_y_x = p_y_xqr
+#         logger.info(f"Updated distribution:\n{np.around(p_y_x.cpu().detach().numpy(), 3)}\n")
+
+#         # Don't repeat the same question again in the future
+#         questions.remove(question)
+
+#         # Terminate if probability exceeds threshold or if out of questions to ask
+#         top_prob = torch.max(p_y_x).item()
+#         if top_prob >= args.threshold or j==min(args.max_num_questions, num_questions_original)-1:
+#             game_preds.append(torch.argmax(p_y_x).item())
+#             game_lens.append(j+1)
+#             logger.info(f"pred:{game_preds[-1]} game_len:{game_lens[-1]}")
+#             break
+
+# logger = logging_handler(args.verbose, args.save_name, "final_results")
+# logger.info(f"Game lenths:\n{game_lens}")
+# logger.info(sum(game_lens)/len(game_lens))
+# logger.info(f"Predictions:\n{game_preds}")
+# logger.info(f"Accuracy:\n{sum([i==args.target_idx for i in game_preds])/len(game_preds)}")

model/utils.py

@@ -0,0 +1,54 @@
+import os
+import logging
+import matplotlib.pyplot as plt
+from PIL import Image
+import nltk
+
+def logging_handler(verbose, save_name, idx=0):
+    logger = logging.getLogger(str(idx))
+    logger.setLevel(logging.INFO)
+
+    stream_logger = logging.StreamHandler()
+    stream_logger.setFormatter(logging.Formatter("%(message)s"))
+    logger.addHandler(stream_logger)
+    
+    if save_name is not None:
+        savepath = f"results/{save_name}"
+        if not os.path.exists(savepath):
+            os.makedirs(savepath)
+        file_logger = logging.FileHandler(f"{savepath}/{idx}.log")
+        file_logger.setFormatter(logging.Formatter("%(message)s"))
+        logger.addHandler(file_logger)
+
+    return logger
+
+
+def image_saver(images, save_name, idx=0, interactive=True):
+    fig, a =  plt.subplots(2,5) 
+    fig.set_size_inches(30, 15)
+    for i in range(10):
+        a[i//5][i%5].imshow(images[i])
+        a[i//5][i%5].axis('off')
+        a[i//5][i%5].set_aspect('equal')
+    plt.tight_layout()
+    plt.subplots_adjust(wspace=0, hspace=0)
+    if not interactive:
+        plt.savefig(f"results/{save_name}/{idx}.png")
+    else:
+        plt.savefig(f"{save_name}.png")
+
+def assert_checks(args):
+    if args.question_strategy=="gpt3":
+        assert args.include_what
+
+def extract_nouns(sents):
+    noun_list = []
+    for idx, s in enumerate(sents):
+        curr = []
+        sent = (nltk.pos_tag(s.split()))
+        for word in sent:
+            if word[1] not in ["NN", "NNS"]: continue
+            currword = word[0].replace('.','')
+            curr.append(currword.lower())
+        noun_list.append(curr)
+    return noun_list

open_db.py

@@ -4,9 +4,4 @@ import pandas as pd
 db = sqlite3.connect("response.db")
 df = pd.read_sql('SELECT * from responses', db)
 print(df)
-# def importdb(db):
-#      conn = sqlite3.connect(db)
-#      c = conn.cursor()
-#      c.execute("SELECT name FROM sqlite_master WHERE type='table';")
-#      for table in c.fetchall()
-#          yield list(c.execute('SELECT * from ?;', (table[0],)))
+df.to_csv("responses.csv", index=False)

pilot-study.csv

@@ -0,0 +1,161 @@
+taskID,mscoco-id,task-type
+0,35,wh- standard
+1,102,wh- hard (n=1)
+2,25,wh- standard
+3,133,yes/no hard (n=1)
+4,80,wh- hard (n=1)
+5,15,wh- standard
+6,92,wh- hard (n=1)
+7,108,wh- hard (n=1)
+8,60,yes/no standard
+9,57,yes/no standard
+10,125,yes/no hard (n=1)
+11,56,yes/no standard
+12,137,yes/no hard (n=1)
+13,40,yes/no standard
+14,134,yes/no hard (n=1)
+15,130,yes/no hard (n=1)
+16,89,wh- hard (n=1)
+17,19,wh- standard
+18,58,yes/no standard
+19,81,wh- hard (n=1)
+20,5,wh- standard
+21,73,yes/no standard
+22,54,yes/no standard
+23,0,wh- standard
+24,14,wh- standard
+25,113,wh- hard (n=1)
+26,34,wh- standard
+27,159,yes/no hard (n=1)
+28,135,yes/no hard (n=1)
+29,2,wh- standard
+30,156,yes/no hard (n=1)
+31,30,wh- standard
+32,104,wh- hard (n=1)
+33,128,yes/no hard (n=1)
+34,18,wh- standard
+35,157,yes/no hard (n=1)
+36,1,wh- standard
+37,42,yes/no standard
+38,131,yes/no hard (n=1)
+39,115,wh- hard (n=1)
+40,120,yes/no hard (n=1)
+41,3,wh- standard
+42,63,yes/no standard
+43,65,yes/no standard
+44,103,wh- hard (n=1)
+45,124,yes/no hard (n=1)
+46,21,wh- standard
+47,72,yes/no standard
+48,62,yes/no standard
+49,47,yes/no standard
+50,78,yes/no standard
+51,109,wh- hard (n=1)
+52,136,yes/no hard (n=1)
+53,158,yes/no hard (n=1)
+54,61,yes/no standard
+55,27,wh- standard
+56,24,wh- standard
+57,123,yes/no hard (n=1)
+58,70,yes/no standard
+59,91,wh- hard (n=1)
+60,55,yes/no standard
+61,87,wh- hard (n=1)
+62,46,yes/no standard
+63,33,wh- standard
+64,16,wh- standard
+65,147,yes/no hard (n=1)
+66,85,wh- hard (n=1)
+67,59,yes/no standard
+68,99,wh- hard (n=1)
+69,117,wh- hard (n=1)
+70,9,wh- standard
+71,122,yes/no hard (n=1)
+72,53,yes/no standard
+73,22,wh- standard
+74,8,wh- standard
+75,29,wh- standard
+76,83,wh- hard (n=1)
+77,37,wh- standard
+78,66,yes/no standard
+79,41,yes/no standard
+80,94,wh- hard (n=1)
+81,98,wh- hard (n=1)
+82,110,wh- hard (n=1)
+83,77,yes/no standard
+84,151,yes/no hard (n=1)
+85,121,yes/no hard (n=1)
+86,6,wh- standard
+87,45,yes/no standard
+88,155,yes/no hard (n=1)
+89,88,wh- hard (n=1)
+90,96,wh- hard (n=1)
+91,75,yes/no standard
+92,112,wh- hard (n=1)
+93,49,yes/no standard
+94,152,yes/no hard (n=1)
+95,38,wh- standard
+96,7,wh- standard
+97,52,yes/no standard
+98,101,wh- hard (n=1)
+99,76,yes/no standard
+100,28,wh- standard
+101,114,wh- hard (n=1)
+102,139,yes/no hard (n=1)
+103,74,yes/no standard
+104,149,yes/no hard (n=1)
+105,84,wh- hard (n=1)
+106,79,yes/no standard
+107,127,yes/no hard (n=1)
+108,126,yes/no hard (n=1)
+109,116,wh- hard (n=1)
+110,71,yes/no standard
+111,67,yes/no standard
+112,10,wh- standard
+113,143,yes/no hard (n=1)
+114,132,yes/no hard (n=1)
+115,90,wh- hard (n=1)
+116,140,yes/no hard (n=1)
+117,144,yes/no hard (n=1)
+118,106,wh- hard (n=1)
+119,32,wh- standard
+120,154,yes/no hard (n=1)
+121,11,wh- standard
+122,17,wh- standard
+123,145,yes/no hard (n=1)
+124,118,wh- hard (n=1)
+125,48,yes/no standard
+126,148,yes/no hard (n=1)
+127,26,wh- standard
+128,51,yes/no standard
+129,13,wh- standard
+130,39,wh- standard
+131,153,yes/no hard (n=1)
+132,12,wh- standard
+133,93,wh- hard (n=1)
+134,107,wh- hard (n=1)
+135,86,wh- hard (n=1)
+136,31,wh- standard
+137,95,wh- hard (n=1)
+138,44,yes/no standard
+139,69,yes/no standard
+140,150,yes/no hard (n=1)
+141,4,wh- standard
+142,142,yes/no hard (n=1)
+143,43,yes/no standard
+144,50,yes/no standard
+145,100,wh- hard (n=1)
+146,129,yes/no hard (n=1)
+147,68,yes/no standard
+148,146,yes/no hard (n=1)
+149,64,yes/no standard
+150,23,wh- standard
+151,82,wh- hard (n=1)
+152,111,wh- hard (n=1)
+153,97,wh- hard (n=1)
+154,119,wh- hard (n=1)
+155,141,yes/no hard (n=1)
+156,20,wh- standard
+157,36,wh- standard
+158,138,yes/no hard (n=1)
+159,105,wh- hard (n=1)

response_db.py

@@ -75,5 +75,4 @@ class StResponseDb(ResponseDb):
 
 if __name__ == "__main__":
     db = ResponseDb()
-    print(db.get_all())
-
+    print(db.get_all())