Dev

config/baseline_params.yaml

@@ -0,0 +1,16 @@
+
+openai_api_key: #INSERT OPEN AI API KEY HERE
+
+cogvlm2_host_ip: #INSERT CogVLM2 HOST IP ADDRESS HERE
+cogvlm2_port: #INSERT CogVLM2 PORT NUMBER
+
+cogvlm_host_ip: #INSERT CogVLM HOST IP ADDRESS HERE
+cogvlm_port: #INSERT CogVLM PORT NUMBER
+
+pipeline_path: #INSERT PATH TO THIS DIRECTORY HERE
+
+dataset_folder: #INSERT THE PATH TO THE DOWNLOADED DATASET HERE
+
+server_video_path: #INSERT VIDEO PATH ON THE SERVER SIDE HERE
+
+baseline_number: #INSERT THE BASELINE NUMBER. 1-DIRECT TEMPORAL VISUAL GROUNDING, 2-REFINED TEMPORAL VISUAL GROUNDING

config/baseline_prompts.yaml

@@ -0,0 +1,93 @@
+
+
+prompt_parser1: |
+
+  """
+
+  example_parsing  = [
+        {
+            "instruction" : "Point to the bottle that I just placed.",
+
+            "ground_truth" : {
+                "past": "Identify the bottle that was placed",
+                "present": "Point to",
+                "future" : ""
+            },
+        },
+
+        {
+            "instruction" : "Help the person if he falls",
+            "ground_truth" : {
+                "past": "",
+                "present": "Watch out for a person to fall",    
+                "future" : "Help the person"
+            },
+        },
+        {
+            "instruction" : "Retrieve the bottle that was just filled",
+            "ground_truth" : {
+                "past": "Identify the bottle that was just filled",
+                "present": "Retrieve",    
+                "future" : ""
+            },
+        },
+        {
+            "instruction" : "Push the recently built stack",
+            "ground_truth" : {
+                "past": "Identify the recently built stack",
+                "present": "Push",    
+                "future" : ""
+            },
+        },
+   ]
+
+   There is a robot that needs to take a human instruction and figure out what has happened in the past, what it needs to do right now, and what it needs to do in the future if something happens. Given the human instruction like , return a dictionary with 'past', 'present', and 'future' as keys. Follow the format in {{ example_parsing }}. Always return a single dictionary with keys 'past', 'present', and 'future'. 
+
+  """
+
+
+prompt_parser2: |
+  """
+  example_parsing = {
+    {
+        "prompt" : The answer is "Is it the second cup from the left?".YES. The context "Is it the second cup from the left?" provides enough information to uniquely identify the cup in question in the given image.
+
+        "output" : second cup from the left
+    }
+
+    {
+        "prompt" : The answer is leftmost. YES. It is the leftmost cup.
+
+        "output" : leftmost cup
+    }
+  }
+
+  Identify the object of interest in the prompt given and return its properties in one line as a sentence. Follow the format in {{ example_parsing }}. Always return a single dictionary with keys 'prompt' and 'output'. Return only a dictionary and no other text.
+
+  """
+
+baseprompt_lvlm: |
+    """
+    . The video is from a robot's perspective. Return all answers from the robot's perspective. 
+
+    """
+
+
+# baseprompt_llm: |
+
+#     """
+
+#     There is a phrase grounding model that needs to uniquely ground a specific object which will help a robot interact with that object.\
+#                     Given this image, is the context enough of information to uniquely identify an object? Reply with a YES if that object can be distinguished uniquely with the given context. \
+#                     If not, ask more questions to uniquely identify the object using its spatial location or other properties. If object is not present, ask more questions to get the right object. \
+#                     Example - Context: The object that was placed was a bottle. Question: Which one was it? The rightmost, second from right, third from left, right and second from top, right and bottommost, or?
+
+#     """
+
+baseprompt_llm: |
+
+    """
+    There is a phrase grounding model that needs to uniquely ground a specific object which will help a robot interact with that object.\
+                    Given this image, is the context enough of information to uniquely identify an object? Reply with a YES if that object can be distinguished uniquely with the given CONTEXT and the image.  \
+                    If not ask one question with options to uniquely identify the object.
+    """

config/params.yaml

@@ -1,18 +1,20 @@
 #OpenAI API Key
 
-openai_api_key: "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"
+openai_api_key: #INSERT OPENAI API KEY HERE
 
-#host, port no.
+#CogVLM2: host, port no.
+cogvlm2_host_ip: #INSERT CogVLM2 HOST IP ADDRESS HERE
+cogvlm2_port: #INSERT CogVLM2 PORT NUMBER
 
-host_ip: "10.237.23.193"
-port: 65439
+#SAM2: host, port no.
+sam2_host_ip: #INSERT SAM2 HOST IP ADDRESS HERE
+sam2_port: #INSERT SAM2 PORT NUMBER 
 
+# main pipeline path 
+pipeline_path: #INSERT PIPELINE PATH HERE
 
-#width, height of frames
-width: 640
-height: 480
-
-#
-
-openai_model: ""
+# Length of recorded action video
+video_length: 5
 
+#Dataset Path
+dataset_folder_path: #DOWNLOAD DATASET FROM DATASET LINK AND INSERT IT HERE