diff --git a/config/baseline_params.yaml b/config/baseline_params.yaml
new file mode 100644
index 0000000..aa76d10
--- /dev/null
+++ b/config/baseline_params.yaml
@@ -0,0 +1,16 @@
+
+openai_api_key: #INSERT OPEN AI API KEY HERE
+
+cogvlm2_host_ip: #INSERT CogVLM2 HOST IP ADDRESS HERE
+cogvlm2_port: #INSERT CogVLM2 PORT NUMBER
+
+cogvlm_host_ip: #INSERT CogVLM HOST IP ADDRESS HERE
+cogvlm_port: #INSERT CogVLM PORT NUMBER
+
+pipeline_path: #INSERT PATH TO THIS DIRECTORY HERE
+
+dataset_folder: #INSERT THE PATH TO THE DOWNLOADED DATASET HERE
+
+server_video_path: #INSERT VIDEO PATH ON THE SERVER SIDE HERE
+
+baseline_number: #INSERT THE BASELINE NUMBER. 1-DIRECT TEMPORAL VISUAL GROUNDING, 2-REFINED TEMPORAL VISUAL GROUNDING
\ No newline at end of file
diff --git a/config/baseline_prompts.yaml b/config/baseline_prompts.yaml
new file mode 100644
index 0000000..d005a52
--- /dev/null
+++ b/config/baseline_prompts.yaml
@@ -0,0 +1,93 @@
+
+
+prompt_parser1: |
+
+ """
+
+ example_parsing = [
+ {
+ "instruction" : "Point to the bottle that I just placed.",
+
+ "ground_truth" : {
+ "past": "Identify the bottle that was placed",
+ "present": "Point to",
+ "future" : ""
+ },
+ },
+
+ {
+ "instruction" : "Help the person if he falls",
+ "ground_truth" : {
+ "past": "",
+ "present": "Watch out for a person to fall",
+ "future" : "Help the person"
+ },
+ },
+ {
+ "instruction" : "Retrieve the bottle that was just filled",
+ "ground_truth" : {
+ "past": "Identify the bottle that was just filled",
+ "present": "Retrieve",
+ "future" : ""
+ },
+ },
+ {
+ "instruction" : "Push the recently built stack",
+ "ground_truth" : {
+ "past": "Identify the recently built stack",
+ "present": "Push",
+ "future" : ""
+ },
+ },
+ ]
+
+ There is a robot that needs to take a human instruction and figure out what has happened in the past, what it needs to do right now, and what it needs to do in the future if something happens. Given the human instruction like , return a dictionary with 'past', 'present', and 'future' as keys. Follow the format in {{ example_parsing }}. Always return a single dictionary with keys 'past', 'present', and 'future'.
+
+ """
+
+
+prompt_parser2: |
+ """
+ example_parsing = {
+ {
+ "prompt" : The answer is "Is it the second cup from the left?".YES. The context "Is it the second cup from the left?" provides enough information to uniquely identify the cup in question in the given image.
+
+ "output" : second cup from the left
+ }
+
+ {
+ "prompt" : The answer is leftmost. YES. It is the leftmost cup.
+
+ "output" : leftmost cup
+ }
+ }
+
+ Identify the object of interest in the prompt given and return its properties in one line as a sentence. Follow the format in {{ example_parsing }}. Always return a single dictionary with keys 'prompt' and 'output'. Return only a dictionary and no other text.
+
+ """
+
+baseprompt_lvlm: |
+ """
+ . The video is from a robot's perspective. Return all answers from the robot's perspective.
+
+ """
+
+
+# baseprompt_llm: |
+
+# """
+
+# There is a phrase grounding model that needs to uniquely ground a specific object which will help a robot interact with that object.\
+# Given this image, is the context enough of information to uniquely identify an object? Reply with a YES if that object can be distinguished uniquely with the given context. \
+# If not, ask more questions to uniquely identify the object using its spatial location or other properties. If object is not present, ask more questions to get the right object. \
+# Example - Context: The object that was placed was a bottle. Question: Which one was it? The rightmost, second from right, third from left, right and second from top, right and bottommost, or?
+
+# """
+
+baseprompt_llm: |
+
+ """
+ There is a phrase grounding model that needs to uniquely ground a specific object which will help a robot interact with that object.\
+ Given this image, is the context enough of information to uniquely identify an object? Reply with a YES if that object can be distinguished uniquely with the given CONTEXT and the image. \
+ If not ask one question with options to uniquely identify the object.
+ """
\ No newline at end of file
diff --git a/config/params.yaml b/config/params.yaml
index 00138ac..b69951a 100644
--- a/config/params.yaml
+++ b/config/params.yaml
@@ -1,18 +1,20 @@
#OpenAI API Key
-openai_api_key: "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"
+openai_api_key: #INSERT OPENAI API KEY HERE
-#host, port no.
+#CogVLM2: host, port no.
+cogvlm2_host_ip: #INSERT CogVLM2 HOST IP ADDRESS HERE
+cogvlm2_port: #INSERT CogVLM2 PORT NUMBER
-host_ip: "10.237.23.193"
-port: 65439
+#SAM2: host, port no.
+sam2_host_ip: #INSERT SAM2 HOST IP ADDRESS HERE
+sam2_port: #INSERT SAM2 PORT NUMBER
+# main pipeline path
+pipeline_path: #INSERT PIPELINE PATH HERE
-#width, height of frames
-width: 640
-height: 480
-
-#
-
-openai_model: ""
+# Length of recorded action video
+video_length: 5
+#Dataset Path
+dataset_folder_path: #DOWNLOAD DATASET FROM DATASET LINK AND INSERT IT HERE
diff --git a/config/prompts.yaml b/config/prompts.yaml
index 286ae27..20bcdef 100644
--- a/config/prompts.yaml
+++ b/config/prompts.yaml
@@ -1,6 +1,5 @@
-#CHECK!
-prompt_parser1: |
+prompt_parser_fact1: |
"""
example_parsing = [
@@ -8,8 +7,8 @@ prompt_parser1: |
"instruction" : "Point to the bottle that I just placed.",
"ground_truth" : {
- "past": "the bottle that was placed",
- "present": "Point to the bottle",
+ "past": "Identify the bottle that was placed",
+ "present": "Point to",
"future" : ""
},
},
@@ -18,17 +17,271 @@ prompt_parser1: |
"instruction" : "Help the person if he falls",
"ground_truth" : {
"past": "",
- "present": "Watch out for a person to fall",
+ "present": "Watch out for a person to fall",
"future" : "Help the person"
},
},
+ {
+ "instruction" : "Retrieve the bottle that was just filled",
+ "ground_truth" : {
+ "past": "Identify the bottle that was just filled",
+ "present": "Retrieve",
+ "future" : ""
+ },
+ },
+ {
+ "instruction" : "Push the recently built stack",
+ "ground_truth" : {
+ "past": "Identify the recently built stack",
+ "present": "Push",
+ "future" : ""
+ },
+ },
+ ]
+
+ There is a robot that needs to take a human instruction and figure out what has happened in the past, what it needs to do right now, and what it needs to do in the future if something happens. Given the human instruction like , return a dictionary with 'past', 'present', and 'future' as keys. Follow the format in {{ example_parsing }}. Always return a single dictionary with keys 'past', 'present', and 'future'.
+
+ """
+
+prompt_parser1: |
+ """
+
+ example_parsing = [
+ {
+ "instruction" : "Point to the bottle that I just placed.",
+
+ "ground_truth" : {
+ "temporal_question": "When is placing by the person happening in the video? Give the exact timestamp.",
+ "action": "Point to",
+ "object" : "the bottle that was placed",
+ "interaction": "placing"
+ },
+ },
+ {
+ "instruction" : "Remove the object that was first eaten",
+ "ground_truth" : {
+ "temporal_question": "When is eating of first object by the person happening in the video? Give the exact timestamp.",
+ "action": "Remove",
+ "object" : "object eaten by person",
+ "interaction": "eating by person"
+ },
+ },
+ {
+ "instruction" : "Remove the object that was placed last",
+ "ground_truth" : {
+ "temporal_question": "When is placing of last object by the person happening in the video? Give the exact timestamp.",
+ "action": "Remove",
+ "object" : "object placed",
+ "interaction": "placing"
+ },
+ },
+ {
+ "instruction" : "Where is the apple?",
+ "ground_truth" : {
+ "temporal_question": "When was the apple last seen? Give the exact timestamp.",
+ "action": "",
+ "object" : "apple",
+ "interaction" : "human-interaction happening with apple"
+ },
+ },
+ {
+ "instruction" : "Identify the object that was used for cleaning second by the girl",
+ "ground_truth" : {
+ "temporal_question": "When is the cleaning using a second object by the girl happening in the video? Give the exact timestamp.",
+ "action": "Identify",
+ "object" : "object used for cleaning by the girl",
+ "interaction": "cleaning by the girl"
+ },
+ },
+
+ {
+ "instruction" : "Remove the cloth that was used by the boy",
+ "ground_truth" : {
+ "temporal_question": "When is the using of cloth by the boy happening in the video? Give the exact timestamp.",
+ "action": "Remove",
+ "object" : "cloth used by boy",
+ "interaction": human-interaction with cloth by the boy
+ },
+ },
+ {
+ "instruction" : "Retrieve the bottle that was filled first by the girl",
+ "ground_truth" : {
+ "temporal_question": "When is filling of the first bottle happening in the video? Give the exact timestamp.",
+ "action": "Retrieve",
+ "object" : "bottle that was filled by the girl",
+ "interaction" : "bottle filling by the girl"
+ },
+ },
+
+ {
+ "instruction" : "Push the bottle",
+ "ground_truth" : {
+ "temporal_question": "When was the bottle last seen? Give the exact timestamp.",
+ "action": "",
+ "object" : "bottle",
+ "interaction" : "human-interaction happening with bottle"
+ },
+ },
+ ]
+
+ There is a robot that needs to take a human instruction and formulate a temporal question as per instruction, what action to take and the object of interest. Given the human instruction, return a dictionary with 'temporal_question', 'action', 'object', and 'interaction' as keys. For the 'object' and 'interaction' keys, remove the temporal aspect and clues such as 'at 2nd second' or 'last'. Return answer in JSON format always.
+ """
+
+
+
+prompt_ImageUnderstander1: |
+
+ """
+ INTERACTION:
+
+ """
+
+prompt_ImageUnderstander2: |
+
+ """
+ OBJECT:
+ """
+
+# prompt_ImageUnderstander3: |
+
+# """
+# From the given images, identify with which single OBJECT the INTERACTION is happening. Give enough details to uniquely identify the object such as EXACT specific spatial properties, color, etc.
+# There is a ROBOT vieweing this scene. Please return your answer from the ROBOT's perspective.
+# Give enough details about the object with respect to the last image you receive.
+# Pick the options from the last frame give to you. Give the label and color of the object label from the last frame.
+
+# """
+
+
+prompt_ImageUnderstander3: |
+
+ """
+ From the given images, identify with which single OBJECT the INTERACTION happened.
+ The last image given to you has visual options. Pick the right object number with which the INTERACTION is happening.
+ Return object label. Justify your answer.
+ """
+
+
+prompt_ObjectExtractor: |
+
+ """
+ From the given images, identify with which single OBJECT the INTERACTION happened. Return that object only.
+ """
+
+# prompt_ImageUnderstander3: |
+
+# """
+# From the given images, identify with which single OBJECT the INTERACTION happened.
+# The last image given to you has visual options. Pick the right object number with which the INTERACTION is happening.
+# If the object is not present in the last frame, then pick the object label of the object that is closely associated with the intended object.
+# Return your answer in the format 'Object 1' or 'Object 2'. Justify your answer.
+# """
+
+# prompt_ImageUnderstander2: |
+
+# """
+# . Ignore words like first, second etc and just focus on interaction and object difference between two images. Please return your answer from ROBOT's perspective and make sure that you give enough details to uniquely identify the object. Please give spatial properties to uniquely identify the object.
+
+# """
+
+
+# prompt_parser2: |
+# """
+# example_parsing = [
+# {
+# "input" : "The object that was picked up is the cup on the left. It had been rotated from its original position in the first image to pour water into it and kept back to face the viewer more directly in the subsequent images. This object stands out because it has been moved while the other two cups, to its right, have remained in their original positions. The left cup is characterized by its brown body and white head.",
+
+# "output" : "White and Brown cup on the left",
+# },
+
+# {
+# "input" : "Object 7 with orange label. The object of interest is a blue bottle, positioned second from the left, recently placed on the table, between two other bottles - one by a pink cloth and one with solid residue.",
+# "output" : "Object 7 with orange label, objecy is blue bottle positioned second from left" ,
+# },
+# {
+# "instruction" : "The object that was picked up is a white and black controller located to the left of the white and pink thermos bottle in the images. This object is visible exactly in the middle between the tall thermos bottle and a teal-colored cup.",
+# "ground_truth" : "White and black controller, left of white and pink bottle",
+# },
+# ]
+
+# Identify the Object Label and the Label Color. Also identify object of interest in the prompt given and return its properties in brief. Follow the format in {{ example_parsing }}.
+
+# """
+
+prompt_parser_object: |
+ """
+ example_parsing = [
+ {
+ "input" : "The object that was picked up is the cup on the left. It had been rotated from its original position in the first image to pour water into it and kept back to face the viewer more directly in the subsequent images. This object stands out because it has been moved while the other two cups, to its right, have remained in their original positions. The left cup is characterized by its brown body and white head.",
+
+ "output" : "cup",
+ },
+
+ {
+ "input" : " The object using which water was poured is a bottle near the blue cup.",
+ "output" : "bottle" ,
+ },
+ {
+ "input" : "The object that was picked up is a white and black controller located to the left of the white and pink thermos bottle in the images. This object is visible exactly in the middle between the tall thermos bottle and a teal-colored cup.",
+ "output" : "controller",
+ },
+ {
+ "input" : "The object with which the interaction is happening is a medicine that is location behind the bottle",
+ "output" : "bottle",
+ },
+ {
+ "input" : "The object with which the interaction is happening is a medicine that is location behind the pink Adidas bottle",
+ "output" : "bottle",
+ },
+ {
+ "input" : "The object that was just kept is a controller inside the basket,
+ "output" : "basket",
+ },
+ {
+ "input" : "The object using which the water was poured is kept inside a black bag",
+ "output" : "black bag",
+ },
]
+
+ Idenitfy the object of interest from the sentence. Follow the format in {{ example_parsing }} giving both input and output always. Return a single dictionary with keys 'input' and 'output'.
"""
- There is a robot that needs to take a human instruction and figure out what has happened in the past, what it needs to do right now, and what it needs to do in the future if something happens. Given the human instruction like , return a dictionary with 'past', 'present', and 'future' as keys. Follow the format in {{ example_parsing }}. Always return a single dictionary.
-prompt_parser2: |
+prompt_parser_object_label: |
"""
- Identify the object of interest in this and return its properties in one line as a sentence.
+ example_parsing = [
+
+ {
+ "input" : "The object that was picked up is the cup on the left, lebelled Object 1. It had been rotated from its original position in the first image to pour water into it and kept back to face the viewer more directly in the subsequent images. This object stands out because it has been moved while the other two cups, to its right, have remained in their original positions. The left cup is characterized by its brown body and white head.",
+
+ "output" : "Object 1",
+ },
+
+ {
+ "input" : "Object 7 with orange label. The object of interest is a blue bottle, positioned second from the left, recently placed on the table, between two other bottles - one by a pink cloth and one with solid residue.",
+ "output" : "Object 7" ,
+ },
+ {
+ "input" : "The object that was picked up is a white and black controller located to the left of the white and pink thermos bottle in the images. This object is visible exactly in the middle between the tall thermos bottle and a teal-colored cup. This object is labelled Object 48.",
+ "output" : "Object 48",
+ },
+ {
+ "input" : "The object that was picked up is a white and black controller located to the left of the white and pink thermos bottle in the images. This object is visible exactly in the middle between the tall thermos bottle and a teal-colored cup. This object is labelled Object 4.",
+ "output" : "Object 4",
+ },
+ {
+ "input" : "The object that was picked up is a white and black controller located to the left of the white and pink thermos bottle in the images. This object is visible exactly in the middle between the tall thermos bottle and a teal-colored cup. This object is labelled Object 8.",
+ "output" : "Object 8",
+ },
+ {
+ "input" : "The object that was picked up is a white and black controller located to the left of the white and pink thermos bottle in the images. This object is visible exactly in the middle between the tall thermos bottle and a teal-colored cup. This object is labelled Object 3.",
+ "output" : "Object 3",
+ },
+
+
+ ]
+
+ Identify the Object Label. Return answer in JSON format always.
+ Return a single dictionary keys 'input' and 'output' only. Make sure to give only Object Label from last image in 'output'.
"""
diff --git a/data/dataset_1/info.json b/data/dataset_1/info.json
deleted file mode 100644
index 2d9c285..0000000
--- a/data/dataset_1/info.json
+++ /dev/null
@@ -1,64 +0,0 @@
-{
- "1": {"video_path": "1.mp4", "instruction": "Identify the object that was just placed"},
- "2": {"video_path": "2.mp4", "instruction": "Identify the object that was just placed"},
- "3": {"video_path": "3.mp4", "instruction": "Identify the object that was just placed"},
- "4": {"video_path": "4.mp4", "instruction": "Identify the object that was just placed"},
- "5": {"video_path": "5.mp4", "instruction": "Identify the object that was just placed"},
- "6": {"video_path": "6.mp4", "instruction": "Identify the object that was dropped first"},
- "7": {"video_path": "6.mp4", "instruction": "Identify the object that was dropped second"},
- "8": {"video_path": "6.mp4", "instruction": "Identify the object that was dropped on the table"},
- "9": {"video_path": "7.mp4", "instruction": "Identify the object into which water was poured"},
- "10": {"video_path": "7.mp4", "instruction": "Identify the object that the person was holding"},
- "11": {"video_path": "8.mp4", "instruction": "Identify the object into which water was poured"},
- "12": {"video_path": "9.mp4", "instruction": "Identify the object that was first placed"},
- "13": {"video_path": "9.mp4", "instruction": "Identify the object into which water was poured"},
- "14": {"video_path": "10.mp4", "instruction": "Identify the object into which water was poured from the green bottle"},
- "15": {"video_path": "10.mp4", "instruction": "Identify the object into which water was poured from the orange bottle"},
- "16": {"video_path": "11.mp4", "instruction": "Identify the object that was dropped"},
- "17": {"video_path": "12.mp4", "instruction": "Identify the object that was placed first"},
- "18": {"video_path": "12.mp4", "instruction": "Identify the object that was placed second"},
- "19": {"video_path": "12.mp4", "instruction": "Identify the object that was placed last"},
- "20": {"video_path": "13.mp4", "instruction": "Identify the object that was placed first"},
- "21": {"video_path": "13.mp4", "instruction": "Identify the object that was placed second"},
- "22": {"video_path": "13.mp4", "instruction": "Identify the object that was placed last"},
- "23": {"video_path": "14.mp4", "instruction": "Identify the object that was manipulated"},
- "24": {"video_path": "14.mp4", "instruction": "Identify the object that was rotated"},
- "25": {"video_path": "15.mp4", "instruction": "Identify the object whose orientation was changed"},
- "26": {"video_path": "16.mp4", "instruction": "Identify the object that was placed"},
- "27": {"video_path": "16.mp4", "instruction": "Identify the object that was rotated"},
- "28": {"video_path": "17.mp4", "instruction": "Identify the object that was placed"},
- "29": {"video_path": "18.mp4", "instruction": "Identify the object that was placed"},
- "30": {"video_path": "19.mp4", "instruction": "Identify the object that was placed"},
- "31": {"video_path": "20.mp4", "instruction": "Identify the object that was kept near the green cube first"},
- "32": {"video_path": "20.mp4", "instruction": "Identify the object that was kept near the green cube second"},
- "33": {"video_path": "21.mp4", "instruction": "Identify the stack that was just built"},
- "34": {"video_path": "22.mp4", "instruction": "Identify the object that was placed first"},
- "35": {"video_path": "22.mp4", "instruction": "Identify the object that was placed second"},
- "36": {"video_path": "23.mp4", "instruction": "Identify the object into which water was poured"},
- "37": {"video_path": "23.mp4", "instruction": "Identify the object that was the first to be placed"},
- "38": {"video_path": "23.mp4", "instruction": "Identify the object that was the second to be placed"},
- "39": {"video_path": "24.mp4", "instruction": "Identify the object that was placed"},
- "40": {"video_path": "24.mp4", "instruction": "Identify the object into which water was poured"},
- "41": {"video_path": "25.mp4", "instruction": "Identify the object that was placed"},
- "42": {"video_path": "25.mp4", "instruction": "Identify the object that was dropped"},
- "43": {"video_path": "26.mp4", "instruction": "Identify the cloth that was dropped first"},
- "44": {"video_path": "26.mp4", "instruction": "Identify the cloth that was dropped second"},
- "45": {"video_path": "27.mp4", "instruction": "Identify the object that was placed"},
- "46": {"video_path": "27.mp4", "instruction": "Identify the bottle that was placed"},
- "47": {"video_path": "28.mp4", "instruction": "Identify the object that replaced the green bottle"},
- "48": {"video_path": "28.mp4", "instruction": "Identify the object that replaced the blue bottle"},
- "49": {"video_path": "29.mp4", "instruction": "Identify the object that replaced the green bottle"},
- "50": {"video_path": "30.mp4", "instruction": "Identify the object into which water was poured"},
- "51": {"video_path": "30.mp4", "instruction": "Identify the cup into which water was poured"},
- "52": {"video_path": "30.mp4", "instruction": "Identify the bottle using which water was poured"},
- "53": {"video_path": "30.mp4", "instruction": "Identify the object using which water was poured"},
- "54": {"video_path": "31.mp4", "instruction": "Identify the object whose position was changed"},
- "55": {"video_path": "32.mp4", "instruction": "Identify the dirty object"},
- "56": {"video_path": "32.mp4", "instruction": "Identify the object using which the table was cleaned"},
- "57": {"video_path": "33.mp4", "instruction": "Identify the dirty object"},
- "58": {"video_path": "33.mp4", "instruction": "Identify the object using which the table was cleaned"},
- "59": {"video_path": "7.mp4", "instruction": "Identify the object whose position was changed"},
- "60": {"video_path": "7.mp4", "instruction": "Identify the cup into which the water was poured"},
- "61": {"video_path": "9.mp4", "instruction": "Identify the object that was placed second"},
- "62": {"video_path": "33.mp4", "instruction": "Identify the dirty cloth"}
-}
\ No newline at end of file
diff --git a/data/dataset_1/last_frames/10_last_frame.jpg b/data/dataset_1/last_frames/10_last_frame.jpg
deleted file mode 100644
index d0fc4ca..0000000
Binary files a/data/dataset_1/last_frames/10_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/11_last_frame.jpg b/data/dataset_1/last_frames/11_last_frame.jpg
deleted file mode 100644
index 676d43f..0000000
Binary files a/data/dataset_1/last_frames/11_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/12_last_frame.jpg b/data/dataset_1/last_frames/12_last_frame.jpg
deleted file mode 100644
index c00f3eb..0000000
Binary files a/data/dataset_1/last_frames/12_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/13_last_frame.jpg b/data/dataset_1/last_frames/13_last_frame.jpg
deleted file mode 100644
index d1ce89b..0000000
Binary files a/data/dataset_1/last_frames/13_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/14_last_frame.jpg b/data/dataset_1/last_frames/14_last_frame.jpg
deleted file mode 100644
index da5e968..0000000
Binary files a/data/dataset_1/last_frames/14_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/15_last_frame.jpg b/data/dataset_1/last_frames/15_last_frame.jpg
deleted file mode 100644
index 2de08d7..0000000
Binary files a/data/dataset_1/last_frames/15_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/16_last_frame.jpg b/data/dataset_1/last_frames/16_last_frame.jpg
deleted file mode 100644
index b229b7c..0000000
Binary files a/data/dataset_1/last_frames/16_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/17_last_frame.jpg b/data/dataset_1/last_frames/17_last_frame.jpg
deleted file mode 100644
index 73386d5..0000000
Binary files a/data/dataset_1/last_frames/17_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/18_last_frame.jpg b/data/dataset_1/last_frames/18_last_frame.jpg
deleted file mode 100644
index ae89817..0000000
Binary files a/data/dataset_1/last_frames/18_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/19_last_frame.jpg b/data/dataset_1/last_frames/19_last_frame.jpg
deleted file mode 100644
index 9100532..0000000
Binary files a/data/dataset_1/last_frames/19_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/1_last_frame.jpg b/data/dataset_1/last_frames/1_last_frame.jpg
deleted file mode 100644
index f19c919..0000000
Binary files a/data/dataset_1/last_frames/1_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/20_last_frame.jpg b/data/dataset_1/last_frames/20_last_frame.jpg
deleted file mode 100644
index 7cd59d0..0000000
Binary files a/data/dataset_1/last_frames/20_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/21_last_frame.jpg b/data/dataset_1/last_frames/21_last_frame.jpg
deleted file mode 100644
index cc724c9..0000000
Binary files a/data/dataset_1/last_frames/21_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/22_last_frame.jpg b/data/dataset_1/last_frames/22_last_frame.jpg
deleted file mode 100644
index 7376bd2..0000000
Binary files a/data/dataset_1/last_frames/22_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/23_last_frame.jpg b/data/dataset_1/last_frames/23_last_frame.jpg
deleted file mode 100644
index 4ab65d1..0000000
Binary files a/data/dataset_1/last_frames/23_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/24_last_frame.jpg b/data/dataset_1/last_frames/24_last_frame.jpg
deleted file mode 100644
index 1c0bd65..0000000
Binary files a/data/dataset_1/last_frames/24_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/25_last_frame.jpg b/data/dataset_1/last_frames/25_last_frame.jpg
deleted file mode 100644
index 0510d01..0000000
Binary files a/data/dataset_1/last_frames/25_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/26_last_frame.jpg b/data/dataset_1/last_frames/26_last_frame.jpg
deleted file mode 100644
index 341aa61..0000000
Binary files a/data/dataset_1/last_frames/26_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/27_last_frame.jpg b/data/dataset_1/last_frames/27_last_frame.jpg
deleted file mode 100644
index 8e3ac44..0000000
Binary files a/data/dataset_1/last_frames/27_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/28_last_frame.jpg b/data/dataset_1/last_frames/28_last_frame.jpg
deleted file mode 100644
index a9eb9f3..0000000
Binary files a/data/dataset_1/last_frames/28_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/29_last_frame.jpg b/data/dataset_1/last_frames/29_last_frame.jpg
deleted file mode 100644
index 1573f2c..0000000
Binary files a/data/dataset_1/last_frames/29_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/2_last_frame.jpg b/data/dataset_1/last_frames/2_last_frame.jpg
deleted file mode 100644
index 3a125ba..0000000
Binary files a/data/dataset_1/last_frames/2_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/30_last_frame.jpg b/data/dataset_1/last_frames/30_last_frame.jpg
deleted file mode 100644
index 908354d..0000000
Binary files a/data/dataset_1/last_frames/30_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/31_last_frame.jpg b/data/dataset_1/last_frames/31_last_frame.jpg
deleted file mode 100644
index a5c1acc..0000000
Binary files a/data/dataset_1/last_frames/31_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/32_last_frame.jpg b/data/dataset_1/last_frames/32_last_frame.jpg
deleted file mode 100644
index b063093..0000000
Binary files a/data/dataset_1/last_frames/32_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/33_last_frame.jpg b/data/dataset_1/last_frames/33_last_frame.jpg
deleted file mode 100644
index 7660e2c..0000000
Binary files a/data/dataset_1/last_frames/33_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/3_last_frame.jpg b/data/dataset_1/last_frames/3_last_frame.jpg
deleted file mode 100644
index 6f16755..0000000
Binary files a/data/dataset_1/last_frames/3_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/4_last_frame.jpg b/data/dataset_1/last_frames/4_last_frame.jpg
deleted file mode 100644
index f5629e5..0000000
Binary files a/data/dataset_1/last_frames/4_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/5_last_frame.jpg b/data/dataset_1/last_frames/5_last_frame.jpg
deleted file mode 100644
index 11bf2fd..0000000
Binary files a/data/dataset_1/last_frames/5_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/6_last_frame.jpg b/data/dataset_1/last_frames/6_last_frame.jpg
deleted file mode 100644
index 7719520..0000000
Binary files a/data/dataset_1/last_frames/6_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/7_last_frame.jpg b/data/dataset_1/last_frames/7_last_frame.jpg
deleted file mode 100644
index 0702886..0000000
Binary files a/data/dataset_1/last_frames/7_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/8_last_frame.jpg b/data/dataset_1/last_frames/8_last_frame.jpg
deleted file mode 100644
index f157403..0000000
Binary files a/data/dataset_1/last_frames/8_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/last_frames/9_last_frame.jpg b/data/dataset_1/last_frames/9_last_frame.jpg
deleted file mode 100644
index e94f13a..0000000
Binary files a/data/dataset_1/last_frames/9_last_frame.jpg and /dev/null differ
diff --git a/data/dataset_1/videos/1.mp4 b/data/dataset_1/videos/1.mp4
deleted file mode 100644
index e31a87a..0000000
Binary files a/data/dataset_1/videos/1.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/10.mp4 b/data/dataset_1/videos/10.mp4
deleted file mode 100644
index fd02f7e..0000000
Binary files a/data/dataset_1/videos/10.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/11.mp4 b/data/dataset_1/videos/11.mp4
deleted file mode 100644
index a01203c..0000000
Binary files a/data/dataset_1/videos/11.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/12.mp4 b/data/dataset_1/videos/12.mp4
deleted file mode 100644
index 432e0f3..0000000
Binary files a/data/dataset_1/videos/12.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/13.mp4 b/data/dataset_1/videos/13.mp4
deleted file mode 100644
index 3e14b0c..0000000
Binary files a/data/dataset_1/videos/13.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/14.mp4 b/data/dataset_1/videos/14.mp4
deleted file mode 100644
index a71ffce..0000000
Binary files a/data/dataset_1/videos/14.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/15.mp4 b/data/dataset_1/videos/15.mp4
deleted file mode 100644
index 8ea4db1..0000000
Binary files a/data/dataset_1/videos/15.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/16.mp4 b/data/dataset_1/videos/16.mp4
deleted file mode 100644
index f1b38b7..0000000
Binary files a/data/dataset_1/videos/16.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/17.mp4 b/data/dataset_1/videos/17.mp4
deleted file mode 100755
index 5b78c93..0000000
Binary files a/data/dataset_1/videos/17.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/18.mp4 b/data/dataset_1/videos/18.mp4
deleted file mode 100755
index 8a1d16a..0000000
Binary files a/data/dataset_1/videos/18.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/19.mp4 b/data/dataset_1/videos/19.mp4
deleted file mode 100755
index d10f32e..0000000
Binary files a/data/dataset_1/videos/19.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/2.mp4 b/data/dataset_1/videos/2.mp4
deleted file mode 100644
index 5a04cd0..0000000
Binary files a/data/dataset_1/videos/2.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/20.mp4 b/data/dataset_1/videos/20.mp4
deleted file mode 100755
index e1d9165..0000000
Binary files a/data/dataset_1/videos/20.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/21.mp4 b/data/dataset_1/videos/21.mp4
deleted file mode 100755
index b7f802a..0000000
Binary files a/data/dataset_1/videos/21.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/22.mp4 b/data/dataset_1/videos/22.mp4
deleted file mode 100755
index 7c9b31d..0000000
Binary files a/data/dataset_1/videos/22.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/23.mp4 b/data/dataset_1/videos/23.mp4
deleted file mode 100755
index 9c69705..0000000
Binary files a/data/dataset_1/videos/23.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/24.mp4 b/data/dataset_1/videos/24.mp4
deleted file mode 100755
index 85584e8..0000000
Binary files a/data/dataset_1/videos/24.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/25.mp4 b/data/dataset_1/videos/25.mp4
deleted file mode 100755
index 1b87f67..0000000
Binary files a/data/dataset_1/videos/25.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/26.mp4 b/data/dataset_1/videos/26.mp4
deleted file mode 100755
index b51f053..0000000
Binary files a/data/dataset_1/videos/26.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/27.mp4 b/data/dataset_1/videos/27.mp4
deleted file mode 100755
index cc81b4f..0000000
Binary files a/data/dataset_1/videos/27.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/28.mp4 b/data/dataset_1/videos/28.mp4
deleted file mode 100755
index 8256fd4..0000000
Binary files a/data/dataset_1/videos/28.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/29.mp4 b/data/dataset_1/videos/29.mp4
deleted file mode 100755
index 43e314b..0000000
Binary files a/data/dataset_1/videos/29.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/3.mp4 b/data/dataset_1/videos/3.mp4
deleted file mode 100644
index 810cade..0000000
Binary files a/data/dataset_1/videos/3.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/30.mp4 b/data/dataset_1/videos/30.mp4
deleted file mode 100755
index ab56699..0000000
Binary files a/data/dataset_1/videos/30.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/31.mp4 b/data/dataset_1/videos/31.mp4
deleted file mode 100755
index 3db0183..0000000
Binary files a/data/dataset_1/videos/31.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/32.mp4 b/data/dataset_1/videos/32.mp4
deleted file mode 100755
index 569eee3..0000000
Binary files a/data/dataset_1/videos/32.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/33.mp4 b/data/dataset_1/videos/33.mp4
deleted file mode 100755
index 75cf2e9..0000000
Binary files a/data/dataset_1/videos/33.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/4.mp4 b/data/dataset_1/videos/4.mp4
deleted file mode 100644
index 285c6ae..0000000
Binary files a/data/dataset_1/videos/4.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/5.mp4 b/data/dataset_1/videos/5.mp4
deleted file mode 100644
index 66f3ad5..0000000
Binary files a/data/dataset_1/videos/5.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/6.mp4 b/data/dataset_1/videos/6.mp4
deleted file mode 100644
index e9a6525..0000000
Binary files a/data/dataset_1/videos/6.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/7.mp4 b/data/dataset_1/videos/7.mp4
deleted file mode 100644
index 8588202..0000000
Binary files a/data/dataset_1/videos/7.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/8.mp4 b/data/dataset_1/videos/8.mp4
deleted file mode 100644
index 99de523..0000000
Binary files a/data/dataset_1/videos/8.mp4 and /dev/null differ
diff --git a/data/dataset_1/videos/9.mp4 b/data/dataset_1/videos/9.mp4
deleted file mode 100644
index c8a5014..0000000
Binary files a/data/dataset_1/videos/9.mp4 and /dev/null differ
diff --git a/eval/ablation/__pycache__/bbox.cpython-312.pyc b/eval/ablation/__pycache__/bbox.cpython-312.pyc
deleted file mode 100644
index c08bd1e..0000000
Binary files a/eval/ablation/__pycache__/bbox.cpython-312.pyc and /dev/null differ
diff --git a/eval/ablation/__pycache__/bbox.cpython-39.pyc b/eval/ablation/__pycache__/bbox.cpython-39.pyc
deleted file mode 100644
index 9fd7eee..0000000
Binary files a/eval/ablation/__pycache__/bbox.cpython-39.pyc and /dev/null differ
diff --git a/eval/ablation/grounding_ablation.py b/eval/ablation/grounding_ablation.py
deleted file mode 100644
index 12053fc..0000000
--- a/eval/ablation/grounding_ablation.py
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/usr/bin/env python3
-
-import numpy
-import socket
-import rospy
-from std_msgs.msg import String
-import openai
-import cv2 as cv
-import pickle
-from sensor_msgs.msg import Image, CompressedImage
-from cv_bridge import CvBridge
-import sys
-import numpy as np
-import time
-import json
-from PIL import Image
-import ast
-from openai import OpenAI
-# import core.bbox
-
-
-def send_msg_to_server(image_path, recv_query):
-
- image = Image.open(image_path)
- image = np.array(image)
-
- print("Calling the COGVLM server")
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- s.connect(("10.237.20.209", 65433))
- print("Sending Query: ", recv_query)
- msg = {"image_array": image, "query": recv_query}
- byte_msg = pickle.dumps(msg)
- s.sendall(byte_msg)
- s.shutdown(socket.SHUT_WR)
- data = b""
- while True:
- packet = s.recv(1024)
- if not packet:
- break
- data += packet
- data = pickle.loads(data)
- #print(data.keys())
- print("Data", data)
- filter_object_name_list_from_dino(image.shape, image, data)
- # print("Corrected List: ", corrected_list)
- # bbox = ast.literal_eval(corrected_list)[0]
- # print("BBox: ", bbox)
- # return bbox
-
- # BBox.draw_bbox(image_size, image_array, bbox, "/home/nivi_nath/catkin_ws/src/tmp_reason/src/driver_codes/bbox_frame4.jpg")
-
-
- print("Got the response")
- return data
-
-def draw_bbox(image_size, image_array, bbox_coords, output_path):
- color = (255,0,0)
- thickness = 2
- print(image_size)
- image_x = image_size[1]
- image_y = image_size[0]
- x1, y1, x2, y2 = bbox_coords
- x1 = int((x1/1000) * (image_x))
- y1 = int((y1/1000) * (image_y))
- x2 = int((x2/1000) * (image_x))
- y2 = int((y2/1000) * (image_y))
- print(([x1, y1, x2, y2]))
- print(type(image_array))
- image_array = np.array(image_array)
- cv.rectangle(image_array, (x1, y1), (x2, y2), color, thickness)
- save_img = output_path
- cv.imwrite(save_img, image_array)
-
-
-def filter_object_name_list_from_dino(image_size, image_array, bbox_string):
-
- # openai.api_key = "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"
- sys_prompt = "Suppose I have a list: [[168,015,421,465]]. Output in the following format: [[168,15,421,465]]. Only output the list, nothing else."
- prompt = bbox_string
-
- client = OpenAI(
-
- api_key = "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",
- )
-
- completion = client.chat.completions.create(
- # Use GPT 3.5 as the LLM
- model="gpt-3.5-turbo",
- # Pre-define conversation messages for the possible roles
- messages=[
- {"role": "system", "content": sys_prompt},
- {"role": "user", "content": prompt}
- ]
- )
- print(completion.choices[0].message.content.strip())
- corrected_list = completion.choices[0].message.content.strip()
-
- #corrected_list = self.filter_object_name_list_from_dino(data)
- print("Corrected List: ", corrected_list)
- bbox_coords = ast.literal_eval(corrected_list)[0]
- print("BBox: ", bbox_coords)
-
- draw_bbox(image_size, image_array, bbox_coords, "/home/nivi_nath/tmp_reason_ws/src/driver_codes/fm_temporal_reasoning/src/eval/ground_truth/final/62.png")
-
-
-if __name__ == '__main__':
-
- send_msg_to_server("/home/nivi_nath/tmp_reason_ws/src/driver_codes/fm_temporal_reasoning/data/dataset_1/last_frames/33_last_frame.jpg", "center cloth")
\ No newline at end of file
diff --git a/eval/ground_truth/final/1.png b/eval/ground_truth/final/1.png
deleted file mode 100644
index ace1a94..0000000
Binary files a/eval/ground_truth/final/1.png and /dev/null differ
diff --git a/eval/ground_truth/final/11.png b/eval/ground_truth/final/11.png
deleted file mode 100644
index 5af2f76..0000000
Binary files a/eval/ground_truth/final/11.png and /dev/null differ
diff --git a/eval/ground_truth/final/12.png b/eval/ground_truth/final/12.png
deleted file mode 100644
index b318ea0..0000000
Binary files a/eval/ground_truth/final/12.png and /dev/null differ
diff --git a/eval/ground_truth/final/13.png b/eval/ground_truth/final/13.png
deleted file mode 100644
index 545bd07..0000000
Binary files a/eval/ground_truth/final/13.png and /dev/null differ
diff --git a/eval/ground_truth/final/14.png b/eval/ground_truth/final/14.png
deleted file mode 100644
index 65df822..0000000
Binary files a/eval/ground_truth/final/14.png and /dev/null differ
diff --git a/eval/ground_truth/final/15.png b/eval/ground_truth/final/15.png
deleted file mode 100644
index 48391bc..0000000
Binary files a/eval/ground_truth/final/15.png and /dev/null differ
diff --git a/eval/ground_truth/final/16.png b/eval/ground_truth/final/16.png
deleted file mode 100644
index 6414a9c..0000000
Binary files a/eval/ground_truth/final/16.png and /dev/null differ
diff --git a/eval/ground_truth/final/17.png b/eval/ground_truth/final/17.png
deleted file mode 100644
index 7d59e88..0000000
Binary files a/eval/ground_truth/final/17.png and /dev/null differ
diff --git a/eval/ground_truth/final/18.png b/eval/ground_truth/final/18.png
deleted file mode 100644
index 1ec0b49..0000000
Binary files a/eval/ground_truth/final/18.png and /dev/null differ
diff --git a/eval/ground_truth/final/19.png b/eval/ground_truth/final/19.png
deleted file mode 100644
index aff42c8..0000000
Binary files a/eval/ground_truth/final/19.png and /dev/null differ
diff --git a/eval/ground_truth/final/2.png b/eval/ground_truth/final/2.png
deleted file mode 100644
index bed8152..0000000
Binary files a/eval/ground_truth/final/2.png and /dev/null differ
diff --git a/eval/ground_truth/final/20.png b/eval/ground_truth/final/20.png
deleted file mode 100644
index 20e705e..0000000
Binary files a/eval/ground_truth/final/20.png and /dev/null differ
diff --git a/eval/ground_truth/final/21.png b/eval/ground_truth/final/21.png
deleted file mode 100644
index 72363a8..0000000
Binary files a/eval/ground_truth/final/21.png and /dev/null differ
diff --git a/eval/ground_truth/final/22.png b/eval/ground_truth/final/22.png
deleted file mode 100644
index 7ad99d8..0000000
Binary files a/eval/ground_truth/final/22.png and /dev/null differ
diff --git a/eval/ground_truth/final/23.png b/eval/ground_truth/final/23.png
deleted file mode 100644
index ab7fb74..0000000
Binary files a/eval/ground_truth/final/23.png and /dev/null differ
diff --git a/eval/ground_truth/final/24.png b/eval/ground_truth/final/24.png
deleted file mode 100644
index ab7fb74..0000000
Binary files a/eval/ground_truth/final/24.png and /dev/null differ
diff --git a/eval/ground_truth/final/25.png b/eval/ground_truth/final/25.png
deleted file mode 100644
index 42f8ebc..0000000
Binary files a/eval/ground_truth/final/25.png and /dev/null differ
diff --git a/eval/ground_truth/final/26.png b/eval/ground_truth/final/26.png
deleted file mode 100644
index 692aeff..0000000
Binary files a/eval/ground_truth/final/26.png and /dev/null differ
diff --git a/eval/ground_truth/final/27.png b/eval/ground_truth/final/27.png
deleted file mode 100644
index e8e35b0..0000000
Binary files a/eval/ground_truth/final/27.png and /dev/null differ
diff --git a/eval/ground_truth/final/28.png b/eval/ground_truth/final/28.png
deleted file mode 100644
index ab7f10b..0000000
Binary files a/eval/ground_truth/final/28.png and /dev/null differ
diff --git a/eval/ground_truth/final/29.png b/eval/ground_truth/final/29.png
deleted file mode 100644
index 4e56594..0000000
Binary files a/eval/ground_truth/final/29.png and /dev/null differ
diff --git a/eval/ground_truth/final/3.png b/eval/ground_truth/final/3.png
deleted file mode 100644
index c0204ff..0000000
Binary files a/eval/ground_truth/final/3.png and /dev/null differ
diff --git a/eval/ground_truth/final/30.png b/eval/ground_truth/final/30.png
deleted file mode 100644
index 54f5e78..0000000
Binary files a/eval/ground_truth/final/30.png and /dev/null differ
diff --git a/eval/ground_truth/final/31.png b/eval/ground_truth/final/31.png
deleted file mode 100644
index 1716ec0..0000000
Binary files a/eval/ground_truth/final/31.png and /dev/null differ
diff --git a/eval/ground_truth/final/32.png b/eval/ground_truth/final/32.png
deleted file mode 100644
index a229d8b..0000000
Binary files a/eval/ground_truth/final/32.png and /dev/null differ
diff --git a/eval/ground_truth/final/33.png b/eval/ground_truth/final/33.png
deleted file mode 100644
index 6b31869..0000000
Binary files a/eval/ground_truth/final/33.png and /dev/null differ
diff --git a/eval/ground_truth/final/34.png b/eval/ground_truth/final/34.png
deleted file mode 100644
index cb69971..0000000
Binary files a/eval/ground_truth/final/34.png and /dev/null differ
diff --git a/eval/ground_truth/final/35.png b/eval/ground_truth/final/35.png
deleted file mode 100644
index e0b8a77..0000000
Binary files a/eval/ground_truth/final/35.png and /dev/null differ
diff --git a/eval/ground_truth/final/36.png b/eval/ground_truth/final/36.png
deleted file mode 100644
index 399b439..0000000
Binary files a/eval/ground_truth/final/36.png and /dev/null differ
diff --git a/eval/ground_truth/final/37.png b/eval/ground_truth/final/37.png
deleted file mode 100644
index 5837a5b..0000000
Binary files a/eval/ground_truth/final/37.png and /dev/null differ
diff --git a/eval/ground_truth/final/38.png b/eval/ground_truth/final/38.png
deleted file mode 100644
index 0652a7f..0000000
Binary files a/eval/ground_truth/final/38.png and /dev/null differ
diff --git a/eval/ground_truth/final/39.png b/eval/ground_truth/final/39.png
deleted file mode 100644
index 37420d3..0000000
Binary files a/eval/ground_truth/final/39.png and /dev/null differ
diff --git a/eval/ground_truth/final/4.png b/eval/ground_truth/final/4.png
deleted file mode 100644
index 19d00c1..0000000
Binary files a/eval/ground_truth/final/4.png and /dev/null differ
diff --git a/eval/ground_truth/final/40.png b/eval/ground_truth/final/40.png
deleted file mode 100644
index 864a85a..0000000
Binary files a/eval/ground_truth/final/40.png and /dev/null differ
diff --git a/eval/ground_truth/final/41.png b/eval/ground_truth/final/41.png
deleted file mode 100644
index 11d498b..0000000
Binary files a/eval/ground_truth/final/41.png and /dev/null differ
diff --git a/eval/ground_truth/final/42.png b/eval/ground_truth/final/42.png
deleted file mode 100644
index 14bab11..0000000
Binary files a/eval/ground_truth/final/42.png and /dev/null differ
diff --git a/eval/ground_truth/final/43.png b/eval/ground_truth/final/43.png
deleted file mode 100644
index aa494cd..0000000
Binary files a/eval/ground_truth/final/43.png and /dev/null differ
diff --git a/eval/ground_truth/final/44.png b/eval/ground_truth/final/44.png
deleted file mode 100644
index 391ccd7..0000000
Binary files a/eval/ground_truth/final/44.png and /dev/null differ
diff --git a/eval/ground_truth/final/45.png b/eval/ground_truth/final/45.png
deleted file mode 100644
index 3c96439..0000000
Binary files a/eval/ground_truth/final/45.png and /dev/null differ
diff --git a/eval/ground_truth/final/46.png b/eval/ground_truth/final/46.png
deleted file mode 100644
index 3c96439..0000000
Binary files a/eval/ground_truth/final/46.png and /dev/null differ
diff --git a/eval/ground_truth/final/47.png b/eval/ground_truth/final/47.png
deleted file mode 100644
index 9a16a6c..0000000
Binary files a/eval/ground_truth/final/47.png and /dev/null differ
diff --git a/eval/ground_truth/final/48.png b/eval/ground_truth/final/48.png
deleted file mode 100644
index a9ed0f4..0000000
Binary files a/eval/ground_truth/final/48.png and /dev/null differ
diff --git a/eval/ground_truth/final/49.png b/eval/ground_truth/final/49.png
deleted file mode 100644
index 32115a5..0000000
Binary files a/eval/ground_truth/final/49.png and /dev/null differ
diff --git a/eval/ground_truth/final/5.png b/eval/ground_truth/final/5.png
deleted file mode 100644
index 6682807..0000000
Binary files a/eval/ground_truth/final/5.png and /dev/null differ
diff --git a/eval/ground_truth/final/50.png b/eval/ground_truth/final/50.png
deleted file mode 100644
index b494509..0000000
Binary files a/eval/ground_truth/final/50.png and /dev/null differ
diff --git a/eval/ground_truth/final/51.png b/eval/ground_truth/final/51.png
deleted file mode 100644
index b494509..0000000
Binary files a/eval/ground_truth/final/51.png and /dev/null differ
diff --git a/eval/ground_truth/final/52.png b/eval/ground_truth/final/52.png
deleted file mode 100644
index d57dae5..0000000
Binary files a/eval/ground_truth/final/52.png and /dev/null differ
diff --git a/eval/ground_truth/final/53.png b/eval/ground_truth/final/53.png
deleted file mode 100644
index d57dae5..0000000
Binary files a/eval/ground_truth/final/53.png and /dev/null differ
diff --git a/eval/ground_truth/final/54.png b/eval/ground_truth/final/54.png
deleted file mode 100644
index 03a5790..0000000
Binary files a/eval/ground_truth/final/54.png and /dev/null differ
diff --git a/eval/ground_truth/final/55.png b/eval/ground_truth/final/55.png
deleted file mode 100644
index f4979c9..0000000
Binary files a/eval/ground_truth/final/55.png and /dev/null differ
diff --git a/eval/ground_truth/final/56.png b/eval/ground_truth/final/56.png
deleted file mode 100644
index f4979c9..0000000
Binary files a/eval/ground_truth/final/56.png and /dev/null differ
diff --git a/eval/ground_truth/final/57.png b/eval/ground_truth/final/57.png
deleted file mode 100644
index 463d41e..0000000
Binary files a/eval/ground_truth/final/57.png and /dev/null differ
diff --git a/eval/ground_truth/final/58.png b/eval/ground_truth/final/58.png
deleted file mode 100644
index 463d41e..0000000
Binary files a/eval/ground_truth/final/58.png and /dev/null differ
diff --git a/eval/ground_truth/final/59.png b/eval/ground_truth/final/59.png
deleted file mode 100644
index 7883cdb..0000000
Binary files a/eval/ground_truth/final/59.png and /dev/null differ
diff --git a/eval/ground_truth/final/6.png b/eval/ground_truth/final/6.png
deleted file mode 100644
index bb8cc03..0000000
Binary files a/eval/ground_truth/final/6.png and /dev/null differ
diff --git a/eval/ground_truth/final/60.png b/eval/ground_truth/final/60.png
deleted file mode 100644
index 73da251..0000000
Binary files a/eval/ground_truth/final/60.png and /dev/null differ
diff --git a/eval/ground_truth/final/61.png b/eval/ground_truth/final/61.png
deleted file mode 100644
index 96a03f2..0000000
Binary files a/eval/ground_truth/final/61.png and /dev/null differ
diff --git a/eval/ground_truth/final/62.png b/eval/ground_truth/final/62.png
deleted file mode 100644
index 463d41e..0000000
Binary files a/eval/ground_truth/final/62.png and /dev/null differ
diff --git a/eval/ground_truth/final/7.png b/eval/ground_truth/final/7.png
deleted file mode 100644
index e9f3fb6..0000000
Binary files a/eval/ground_truth/final/7.png and /dev/null differ
diff --git a/eval/ground_truth/final/8.png b/eval/ground_truth/final/8.png
deleted file mode 100644
index bb8cc03..0000000
Binary files a/eval/ground_truth/final/8.png and /dev/null differ
diff --git a/eval/ground_truth/final/final.json b/eval/ground_truth/final/final.json
deleted file mode 100644
index c687e28..0000000
--- a/eval/ground_truth/final/final.json
+++ /dev/null
@@ -1,90 +0,0 @@
-{
-
-
- "1": [67, 185, 196, 258],
- "2": [246, 238, 295, 307],
- "3": [293, 238, 440, 435],
- "4": [211, 255, 327, 335],
- "5": [92, 326, 224, 424],
- "6": [291, 235, 513, 377],
- "7": [268, 104, 364, 149],
- "8": [291, 235, 513, 377],
- "9": [184, 204, 228, 267],
- "10": [184, 204, 228, 267],
- "11": [318, 238, 372, 308],
- "12":[164, 284, 222, 357],
- "13": [391, 275, 442, 348],
- "14": [49, 224, 107, 290],
- "15": [432, 293, 491, 369],
- "16": [232, 105, 333, 179],
- "17": [291, 285, 421, 383],
- "18": [67, 205, 237, 354],
- "19": [363, 67, 437, 271],
- "20":[280, 285, 408, 377],
- "21": [485, 224, 542, 291],
- "22":[323, 360, 374, 467],
- "23": [226, 283, 352, 388],
- "24": [226, 283, 352, 388],
- "25": [71, 196, 168, 293],
- "26": [48, 243, 138, 335],
- "27": [246, 263, 346, 362],
- "28": [141, 183, 201, 270],
- "29": [440, 247, 506, 343],
- "30": [299, 237, 344, 309],
- "31": [242, 191, 286, 259],
- "32": [370, 189, 412, 258],
- "33": [199, 178, 258, 283],
- "34": [181, 126, 277, 350],
- "35": [361, 216, 415, 304],
- "36": [158, 194, 216, 280],
- "37": [365, 271, 410, 343],
- "38": [403, 129, 565, 290],
- "39": [403, 230, 450, 300],
- "40": [240, 179, 296, 276],
- "41": [201, 259, 251, 331],
- "42": [323, 189, 387, 315],
- "43": [140, 231, 315, 405],
- "44": [368, 135, 512, 324],
- "45": [243, 76, 318, 290],
- "46": [243, 76, 318, 290],
- "47": [156, 180, 208, 245],
- "48": [402, 248, 455, 327],
- "49": [165, 201, 220, 271],
- "50": [222, 213, 275, 301],
- "51": [222, 213, 275, 301],
- "52": [87, 44, 204, 263],
- "53": [87, 44, 204, 263],
- "54": [229, 368, 280, 444],
- "55": [83, 103, 137, 163],
- "56": [83, 103, 137, 163],
- "57": [208, 124, 310, 250],
- "58":[208, 124, 310, 250],
- "59": [184, 204, 228, 267],
- "61": [165, 216, 213, 281],
- "62": [208, 124, 310, 250]
-
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/eval/test_run.py b/eval/test_run.py
deleted file mode 100644
index 70300a0..0000000
--- a/eval/test_run.py
+++ /dev/null
@@ -1,165 +0,0 @@
-#!/usr/bin/env python3
-
-
-"""Run file to execute the temporal reasoning pipeline. The parameters
-of the pipeline can be set by passing arguments to this file.
-"""
-
-# Library imports
-import argparse
-import yaml
-import os
-import pathlib
-# from . import ros_whisper
-import rospy
-import multiprocessing
-
-import os
-import sys
-sys.path.append('/home/nivi_nath/tmp_reason_ws/src/driver_codes/fm_temporal_reasoning/')
-import cv2
-import numpy as np
-# import core.parsers as parsers
-import time
-import rospy
-import json
-from std_msgs.msg import String
-from PIL import Image
-import sensor_msgs
-import io
-from cv_bridge import CvBridge
-from src.core import video_understanding
-np.float = np.float64
-import ros_numpy
-from functools import partial
-from src.core import validator
-from src.core import parsing
-import json
-
-from src.core import grounding
-
-from signal import signal,SIGPIPE, SIG_DFL
-signal(SIGPIPE, SIG_DFL)
-#from src.utils import ros_whisper
-
-
-#Remove or Modify later
-class Color:
- RED = '\033[91m'
- GREEN = '\033[92m'
- YELLOW = '\033[93m'
- BLUE = '\033[94m'
- MAGENTA = '\033[95m'
- CYAN = '\033[96m'
- WHITE = '\033[97m'
- RESET = '\033[0m'
-
-def load_params():
- """Loads the parameter
-
- Returns:
- dict: parameters of the localization pipeline
- """
- # current_dir_path = pathlib.Path.cwd()
- params = {}
- params_path = "/home/nivi_nath/tmp_reason_ws/src/driver_codes/fm_temporal_reasoning/config/params.yaml"
- with open(params_path) as stream:
- try:
- params = yaml.safe_load(stream)
- except yaml.YAMLError as exc:
- print(exc)
- return params
-
-def load_prompts():
- """Loads the prompts
-
- Returns:
- dict: parameters of the localization pipeline
- """
- # current_dir_path = pathlib.Path.cwd()
- prompts = {}
- prompts_path = "/home/nivi_nath/tmp_reason_ws/src/driver_codes/fm_temporal_reasoning/config/prompts.yaml"
- with open(prompts_path) as stream:
- try:
- prompts = yaml.safe_load(stream)
- except yaml.YAMLError as exc:
- print(exc)
- return prompts
-
-class TR_pipeline_testing():
-
- def __init__(self):
-
- self.input_list = []
-
-
-
- def extract_last_frame(self, video_path):
- cap = cv2.VideoCapture(video_path)
- while True:
- ret, frame = cap.read()
- if not ret:
- break
- self.input_list.append(frame)
- cap.release()
- return self.input_list[-1]
-
-
- def test_pipeline(self, video_path, instruction, output_path, params, prompts):
-
- last_frame = self.extract_last_frame(video_path)
-
-
- os.system('scp ' + video_path + ' helium:/home/niveditha/VideoChat2/video_chat2/dataset/input_video.mp4')
-
-
- print("inside TR usage")
- print("user prompt", instruction)
- parser_output = eval(parsing.to_llm(instruction, params, prompts))
- print("Parser Output", parser_output)
-
- # parser_output = parser_output['ground_truth']
- print(parser_output)
- modified_prompt = "Identify " + parser_output["past"]
-
-
- videoLLM_output = validator.conversation(params, modified_prompt, last_frame)
-
- cogvlm_query = parsing.to_cogVLM(videoLLM_output, params, prompts)
-
- print("CogVLM Query:", cogvlm_query)
- action = parser_output["present"]
- location = ''
-
- grounding.send_msg_to_server(last_frame,cogvlm_query, output_path)
-
- # return({"action": action, "object": cogvlm_query, "object_context": location})
-
-
-if __name__ == '__main__':
-
- prompts = load_prompts()
- params = load_params()
-
- video_folder = "/home/nivi_nath/tmp_reason_ws/src/driver_codes/fm_temporal_reasoning/data/dataset_1/videos/"
- output_folder = "/home/nivi_nath/tmp_reason_ws/src/driver_codes/fm_temporal_reasoning/output/dataset_1/final/"
-
- with open('/home/nivi_nath/tmp_reason_ws/src/driver_codes/fm_temporal_reasoning/data/dataset_1/info.json') as f:
- data = json.load(f)
-
- for i in range(45, len(data)+1):
-
- datum = data[str(i)]
-
- video_path = os.path.join(video_folder, datum['video_path'])
- instruction = datum['instruction']
- output_path = os.path.join(output_folder, str(i) + ".png")
- print(os.path.join(output_folder, str(i)+ ".png"))
- pipeline = TR_pipeline_testing()
- pipeline.test_pipeline(video_path, instruction, output_path, params, prompts)
-
-
-
-
-
-
diff --git a/index.html b/index.html
index b7faa6d..8c4cc60 100644
--- a/index.html
+++ b/index.html
@@ -1,18 +1,17 @@
-Google
