Datasets:
batch_id stringclasses 1
value | seq_id stringlengths 22 26 | chunk_id int32 0 38 | frame_idx_start int32 0 3.64k | frame_idx_end int32 7 3.64k | priority_frame_idx int32 2 6 | L1_task stringlengths 14 109 | L2_subtask listlengths 1 8 | L3_atomic listlengths 1 11 | L4_pixel_atomic listlengths 0 10 | L5_keypoint_atomic listlengths 0 9 | L6_preference listlengths 0 2 | L7_negation listlengths 0 3 | L8_combinations listlengths 5 15 | images_rgb images listlengths 8 12 | actions array 2D | action_tokens array 2D |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
batch_20260503_204200 | BTqlANhpFjMEUm1SYcQJC5 | 0 | 0 | 9 | 3 | Follow the dirt path through the forest. | [
{
"text": "Curve slightly right on the path",
"frames": [
0,
1
],
"instruction_type": "ROUTE"
},
{
"text": "Then curve left",
"frames": [
2,
3,
4
],
"instruction_type": "ROUTE"
},
{
"text": "Continue mostly straight",
"frames": [
5,... | [
{
"primitive": "turn right",
"frames": [
0
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
1
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn left",
"frames": [
2,
3,
4
],
"instruction_type": ... | [] | [] | [] | [
{
"text": "Do not leave the dirt road.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [
{
"style": "landmark_heavy",
"text": "Follow the <dirt_road> through the <trees>, staying alongside the <wooden_fence>.",
"referenced_landmarks": [
"dirt_road",
"trees",
"wooden_fence"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "LAN... | [
[
0.14765945076942444,
0.6630955934524536,
12.984575271606445
],
[
-0.056253962218761444,
0.7061585187911987,
-9.736257553100586
],
[
-0.1878126561641693,
0.6834190487861633,
-14.713644027709961
],
[
0.12242390960454941,
0.7094399333000183,
-6.621807575225... | [
[
11,
1,
14
],
[
7,
1,
2
],
[
5,
1,
1
],
[
10,
1,
3
],
[
5,
0,
2
],
[
12,
1,
15
],
[
8,
1,
8
],
[
8,
1,
11
],
[
9,
1,
11
]
] | |
batch_20260503_204200 | BTqlANhpFjMEUm1SYcQJC5 | 1 | 20 | 29 | 2 | Reach the end of the dirt road. | [
{
"text": "Follow the dirt road straight ahead.",
"frames": [
0,
1,
2,
3,
4
],
"instruction_type": "ROUTE"
},
{
"text": "Continue along the road, making a slight adjustment to the right.",
"frames": [
5,
6,
7,
8,
9
],
"i... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3,
4
],
"instruction_type": "SPATIAL"
},
{
"primitive": "slight right",
"frames": [
5,
6
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
7,... | [] | [
{
"text": "Follow the <dirt_road>.",
"frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"referenced_landmarks": [
"dirt_road"
],
"instruction_type": "LANDMARK"
},
{
"text": "Continue past the <trees>.",
"frames": [
... | [
{
"text": "Follow the uncrowded path.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
},
{
"text": "Enjoy the natural surroundings.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,... | [
{
"text": "Avoid traffic.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
},
{
"text": "Do not go towards any buildings.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
... | [
{
"style": "landmark_heavy",
"text": "Follow the <dirt_road> past the <trees> and the <wooden_fence>.",
"referenced_landmarks": [
"dirt_road",
"trees",
"wooden_fence"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "LANDMARK"
},
{
... | [
[
0.055742353200912476,
0.6254182457923889,
5.20143461227417
],
[
-0.1247771829366684,
0.5588319301605225,
-15.868590354919434
],
[
0.135469451546669,
0.6852392554283142,
6.119955062866211
],
[
0.0419585220515728,
0.6640070676803589,
6.1186394691467285
]... | [
[
9,
1,
11
],
[
5,
0,
1
],
[
10,
1,
11
],
[
9,
1,
11
],
[
6,
1,
1
],
[
5,
0,
1
],
[
13,
1,
15
],
[
6,
1,
4
],
[
9,
1,
9
]
] | |
batch_20260503_204200 | BTqlANhpFjMEUm1SYcQJC5 | 2 | 30 | 39 | 2 | Follow the dirt road through the forest. | [
{
"text": "Continue straight along the dirt road.",
"frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"instruction_type": "ROUTE"
}
] | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"instruction_type": "SPATIAL"
}
] | [] | [
{
"text": "Follow the <dirt_road>.",
"frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"referenced_landmarks": [
"dirt_road"
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "Stay on the clear path.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [] | [
{
"style": "landmark_heavy",
"text": "Follow the <dirt_road> through the <forest>, keeping the <wooden_fence> on your left.",
"referenced_landmarks": [
"dirt_road",
"forest",
"wooden_fence"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type":... | [
[
-0.1392420381307602,
0.5877597332000732,
-15.352395057678223
],
[
0.10644486546516418,
0.638102114200592,
8.005982398986816
],
[
0.07978879660367966,
0.6782237887382507,
7.361372470855713
],
[
-0.08772251754999161,
0.6388823986053467,
-13.931117057800293... | [
[
5,
0,
1
],
[
10,
1,
12
],
[
9,
1,
12
],
[
6,
1,
1
],
[
11,
1,
14
],
[
7,
1,
5
],
[
7,
1,
4
],
[
8,
0,
5
],
[
9,
1,
11
]
] | |
batch_20260503_204200 | BTqlANhpFjMEUm1SYcQJC5 | 3 | 40 | 49 | 3 | Navigate along the forest path. | [
{
"text": "Go straight on the path.",
"frames": [
0,
1,
2,
3,
4
],
"instruction_type": "ROUTE"
},
{
"text": "Make a slight left turn.",
"frames": [
5
],
"instruction_type": "ROUTE"
},
{
"text": "Make a slight right turn.",
"frames":... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3,
4
],
"instruction_type": "SPATIAL"
},
{
"primitive": "slight left",
"frames": [
5
],
"instruction_type": "SPATIAL"
},
{
"primitive": "slight right",
"frames": [
6,
7
... | [] | [
{
"text": "Follow the <dirt_path> straight ahead.",
"frames": [
0,
1,
2,
3,
4
],
"referenced_landmarks": [
"dirt_path"
],
"instruction_type": "LANDMARK"
},
{
"text": "Make a slight left turn on the <dirt_path>.",
"frames": [
5
],
... | [
{
"text": "Stay on the path that is mostly shaded by trees.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [
{
"text": "Do not go into the dense foliage on the sides.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [
{
"style": "landmark_heavy",
"text": "Go straight on the <dirt_path>, with <trees> lining both sides of the <hillside>.",
"referenced_landmarks": [
"dirt_path",
"trees",
"hillside"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "LANDMAR... | [
[
-0.05375230312347412,
0.6361871361732483,
-2.290649175643921
],
[
-0.05361625552177429,
0.6369910836219788,
-7.827099323272705
],
[
-0.0042187427170574665,
0.6009803414344788,
-1.7423055171966553
],
[
0.07336562871932983,
0.6385617256164551,
3.7670159339... | [
[
7,
1,
6
],
[
7,
1,
3
],
[
8,
1,
6
],
[
9,
1,
10
],
[
8,
1,
6
],
[
6,
0,
0
],
[
10,
0,
13
],
[
12,
1,
15
],
[
7,
1,
12
]
] | |
batch_20260503_204200 | BTqlANhpFjMEUm1SYcQJC5 | 4 | 50 | 59 | 2 | Follow the winding dirt path to its end. | [
{
"text": "Follow the path as it curves slightly right.",
"frames": [
0,
1
],
"instruction_type": "ROUTE"
},
{
"text": "Continue on the path as it curves left.",
"frames": [
2,
3,
4
],
"instruction_type": "ROUTE"
},
{
"text": "Proceed as the ... | [
{
"primitive": "slight right",
"frames": [
0,
1
],
"instruction_type": "SPATIAL"
},
{
"primitive": "slight left",
"frames": [
2,
3,
4
],
"instruction_type": "SPATIAL"
},
{
"primitive": "slight right",
"frames": [
5
],
"instr... | [] | [
{
"text": "Follow the <dirt_path>.",
"frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"referenced_landmarks": [
"dirt_path"
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "Stay on the clear dirt path.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [
{
"text": "Do not stray into the dense trees and grass.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [
{
"style": "landmark_heavy",
"text": "Follow the <dirt_path> through the <trees> and <grass_patch>.",
"referenced_landmarks": [
"dirt_path",
"trees",
"grass_patch"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "MULTI-CLAUSE"
},
{
... | [
[
-0.2587670087814331,
0.5970748662948608,
-23.14155387878418
],
[
-0.031089799478650093,
0.6475905776023865,
-7.045348644256592
],
[
0.06283432990312576,
0.59781813621521,
-3.4571340084075928
],
[
0.3134894073009491,
0.6814086437225342,
25.967554092407227... | [
[
4,
0,
0
],
[
7,
1,
3
],
[
9,
1,
5
],
[
12,
1,
15
],
[
5,
0,
1
],
[
10,
1,
13
],
[
8,
0,
3
],
[
7,
1,
11
],
[
9,
1,
8
]
] | |
batch_20260503_204200 | BTqlANhpFjMEUm1SYcQJC5 | 5 | 60 | 69 | 2 | Follow the winding dirt path. | [
{
"text": "Follow the path as it curves right.",
"frames": [
0,
1,
2,
3
],
"instruction_type": "ROUTE"
},
{
"text": "Continue as the path curves left.",
"frames": [
4,
5
],
"instruction_type": "ROUTE"
},
{
"text": "Follow the path as it... | [
{
"primitive": "go straight",
"frames": [
0
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn right",
"frames": [
1
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
2
],
"instruction_type": "SPATIAL"
},
... | [] | [
{
"text": "Follow the <dirt_path> for the entire trajectory.",
"frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"referenced_landmarks": [
"dirt_path"
],
"instruction_type": "LANDMARK"
}
] | [] | [] | [
{
"style": "landmark_heavy",
"text": "Follow the <dirt_path> as it curves right, then left, then right again, and finally left.",
"referenced_landmarks": [
"dirt_path"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "MULTI-CLAUSE"
},
{
"styl... | [
[
-0.1267906129360199,
0.5881947875022888,
-7.851465225219727
],
[
0.4013496935367584,
0.7388594746589661,
35.1422004699707
],
[
-0.020713556557893753,
0.6110199093818665,
5.704983711242676
],
[
0.09585071355104446,
0.7322962284088135,
14.454761505126953
... | [
[
5,
0,
3
],
[
13,
1,
15
],
[
7,
1,
11
],
[
10,
1,
14
],
[
10,
0,
6
],
[
6,
1,
0
],
[
5,
0,
4
],
[
12,
0,
13
],
[
5,
1,
1
]
] | |
batch_20260503_204200 | BTqlANhpFjMEUm1SYcQJC5 | 6 | 75 | 84 | 2 | Follow the dirt road. | [
{
"text": "Go straight on the dirt road.",
"frames": [
0,
1,
2
],
"instruction_type": "ROUTE"
},
{
"text": "Make a slight right turn.",
"frames": [
3,
4
],
"instruction_type": "ROUTE"
},
{
"text": "Make a slight left turn.",
"frames": [
... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2
],
"instruction_type": "SPATIAL"
},
{
"primitive": "slight right",
"frames": [
3,
4
],
"instruction_type": "SPATIAL"
},
{
"primitive": "slight left",
"frames": [
5,
6
],
... | [] | [
{
"text": "Follow the <dirt_road>.",
"frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"referenced_landmarks": [
"dirt_road"
],
"instruction_type": "LANDMARK"
},
{
"text": "Turn slight right on the <dirt_road>.",
"frames... | [
{
"text": "Follow the clear path.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [
{
"text": "Do not encounter any obstacles.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [
{
"style": "landmark_heavy",
"text": "Follow the <dirt_road> through the surrounding <trees>.",
"referenced_landmarks": [
"dirt_road",
"trees"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "LANDMARK"
},
{
"style": "task_only",
"t... | [
[
-0.17439928650856018,
0.7269182205200195,
-7.472265720367432
],
[
0.036059148609638214,
0.5680158734321594,
0.8364391922950745
],
[
0.005139352288097143,
0.5043167471885681,
2.398402690887451
],
[
0.382152259349823,
0.7675368189811707,
29.51734733581543
... | [
[
5,
1,
3
],
[
9,
0,
8
],
[
8,
0,
9
],
[
13,
1,
15
],
[
7,
0,
6
],
[
4,
1,
0
],
[
6,
0,
2
],
[
9,
0,
9
],
[
9,
1,
12
]
] | |
batch_20260503_204200 | BTqlANhpFjMEUm1SYcQJC5 | 7 | 85 | 94 | 3 | Follow the forest path to its end. | [
{
"text": "Proceed along the forest path, making initial turns.",
"frames": [
0,
1,
2,
3,
4
],
"instruction_type": "ROUTE"
},
{
"text": "Continue forward and make a final turn.",
"frames": [
5,
6,
7
],
"instruction_type": "ROUTE"
... | [
{
"primitive": "go straight",
"frames": [
0
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn right",
"frames": [
1
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn left",
"frames": [
2
],
"instruction_type": "SPATIAL"
},
{... | [] | [] | [] | [] | [
{
"style": "task_only",
"text": "Follow the forest path to its end.",
"referenced_landmarks": [],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "GOAL"
},
{
"style": "spatial_only",
"text": "Go straight, then turn slightly right, followed by a sligh... | [
[
-0.008115901611745358,
0.5999532341957092,
3.8740439414978027
],
[
0.07307273149490356,
0.6608015894889832,
13.619397163391113
],
[
-0.21771346032619476,
0.5780657529830933,
-22.59023094177246
],
[
-0.01769302785396576,
0.5892359614372253,
-4.24316549301... | [
[
8,
1,
10
],
[
9,
1,
14
],
[
4,
0,
0
],
[
7,
0,
4
],
[
6,
0,
0
],
[
9,
0,
8
],
[
11,
0,
14
],
[
11,
1,
15
],
[
8,
1,
12
]
] | |
batch_20260503_204200 | BTqlANhpFjMEUm1SYcQJC5 | 8 | 95 | 104 | 2 | Follow the winding dirt road through the forest. | [
{
"text": "Navigate the initial slight left turn on the dirt road.",
"frames": [
0,
1,
2
],
"instruction_type": "ROUTE"
},
{
"text": "Make a slight right turn.",
"frames": [
3,
4
],
"instruction_type": "ROUTE"
},
{
"text": "Execute another sl... | [
{
"primitive": "turn left",
"frames": [
0,
1
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
2
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn right",
"frames": [
3,
4
],
"instruction_type": ... | [] | [
{
"text": "follow the <dirt_road>",
"frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"referenced_landmarks": [
"dirt_road"
],
"instruction_type": "LANDMARK"
},
{
"text": "pass by the <trees>",
"frames": [
0,
... | [] | [] | [
{
"style": "landmark_heavy",
"text": "Follow the <dirt_road> as it turns left, then right, then left again, continuing past the <trees>.",
"referenced_landmarks": [
"dirt_road",
"trees"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "MULTI-CL... | [
[
-0.0935966968536377,
0.5071467757225037,
-10.350312232971191
],
[
-0.20098011195659637,
0.526131272315979,
-28.645044326782227
],
[
0.07439529150724411,
0.5962803959846497,
-0.9293840527534485
],
[
0.3321010172367096,
0.6743393540382385,
27.8787593841552... | [
[
6,
0,
2
],
[
4,
0,
0
],
[
9,
0,
7
],
[
12,
1,
15
],
[
10,
1,
15
],
[
4,
0,
0
],
[
7,
0,
2
],
[
9,
0,
6
],
[
8,
0,
6
]
] | |
batch_20260503_204200 | BVPJjH7nRfWo7GlDhSnEAA | 0 | 32 | 41 | 3 | Navigate down the road and turn left. | [
{
"text": "Go straight along the road.",
"frames": [
0,
1,
2,
3,
4
],
"instruction_type": "ROUTE"
},
{
"text": "Turn left at the intersection.",
"frames": [
5,
6
],
"instruction_type": "ROUTE"
},
{
"text": "Continue straight after... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3,
4
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn left",
"frames": [
5,
6
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
7,
... | [
{
"text": "approach the <trees>",
"frames": [
3,
4,
5
],
"instruction_type": "LANDMARK"
},
{
"text": "pass the <house>",
"frames": [
6,
7,
8
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "approach the <trees> and turn left",
"frames": [
3,
4,
5,
6
],
"referenced_landmarks": [
"trees"
],
"instruction_type": "LANDMARK"
},
{
"text": "continue past the <house>",
"frames": [
7,
8,
9
],
"referenced_landm... | [] | [] | [
{
"style": "landmark_heavy",
"text": "Go straight past the <parked_car>s, then turn left near the <trees>, and continue past the <house>.",
"referenced_landmarks": [
"parked_car",
"trees",
"house"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction... | [
[
-0.5523061752319336,
6.820977210998535,
0.7072710990905762
],
[
-0.3661620020866394,
6.927929878234863,
2.1700329780578613
],
[
-0.21530835330486298,
5.751895904541016,
2.7457473278045654
],
[
-0.2767636477947235,
4.8094868659973145,
1.4167166948318481
... | [
[
2,
13,
8
],
[
3,
13,
9
],
[
4,
12,
10
],
[
4,
11,
9
],
[
2,
10,
5
],
[
0,
14,
1
],
[
0,
14,
0
],
[
1,
13,
5
],
[
2,
14,
7
]
] | |
batch_20260503_204200 | BVPJjH7nRfWo7GlDhSnEAA | 1 | 42 | 50 | 2 | Navigate to the end of the road after turning right. | [
{
"text": "Go straight towards the intersection",
"frames": [
0,
1,
2,
3
],
"instruction_type": "ROUTE"
},
{
"text": "Turn right at the intersection",
"frames": [
4,
5,
6
],
"instruction_type": "ROUTE"
},
{
"text": "Continue strai... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn right",
"frames": [
4,
5,
6
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
7,
... | [
{
"text": "approach the <street_sign>",
"frames": [
2,
3,
4
],
"instruction_type": "LANDMARK"
},
{
"text": "pass the <street_sign>",
"frames": [
5,
6
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "turn right at the <street_sign>",
"frames": [
3,
4,
5,
6
],
"referenced_landmarks": [
"street_sign"
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "drive on the clear road",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8
]
}
] | [
{
"text": "do not turn left",
"evidence_frames": [
3,
4,
5,
6
]
},
{
"text": "avoid pedestrians",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8
]
}
] | [
{
"style": "landmark_heavy",
"text": "Go straight past the <parked_car>, then turn right at the <street_sign>, and continue past the <utility_pole>.",
"referenced_landmarks": [
"parked_car",
"street_sign",
"utility_pole"
],
"referenced_preferences": [],
"referenced_negation... | [
[
-0.7931406497955322,
8.659646034240723,
-0.02225654013454914
],
[
-0.7278025150299072,
7.97368049621582,
0.4586603343486786
],
[
-0.7668018937110901,
10.595455169677734,
2.6785268783569336
],
[
-0.13936620950698853,
5.841022491455078,
4.666914939880371
... | [
[
1,
14,
7
],
[
1,
14,
8
],
[
1,
15,
10
],
[
5,
12,
11
],
[
13,
12,
13
],
[
14,
12,
13
],
[
13,
13,
13
],
[
11,
13,
12
]
] | |
batch_20260503_204200 | BVPJjH7nRfWo7GlDhSnEAA | 2 | 53 | 62 | 2 | Navigate through the residential street to the end. | [
{
"text": "Go straight on the initial street",
"frames": [
0,
1,
2,
3
],
"instruction_type": "ROUTE"
},
{
"text": "Turn right at the intersection",
"frames": [
4
],
"instruction_type": "ROUTE"
},
{
"text": "Continue straight on the new street... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn right",
"frames": [
4
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
5,
6
],
... | [
{
"text": "approach the house on the right",
"frames": [
3,
4
],
"instruction_type": "LANDMARK"
},
{
"text": "turn left at the street_sign",
"frames": [
7,
8
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "turn right past the house",
"frames": [
3,
4,
5
],
"referenced_landmarks": [
"house"
],
"instruction_type": "LANDMARK"
},
{
"text": "turn left at the street_sign",
"frames": [
7,
8
],
"referenced_landmarks": [
"street_s... | [] | [] | [
{
"style": "landmark_heavy",
"text": "Go straight past the initial <houses>, then turn right near the prominent <house> on the corner, and finally turn left at the <street_sign>.",
"referenced_landmarks": [
"house",
"street_sign"
],
"referenced_preferences": [],
"referenced_negat... | [
[
0.313263475894928,
8.171218872070312,
5.456838130950928
],
[
-0.26611003279685974,
9.895997047424316,
5.585477828979492
],
[
-0.31953316926956177,
8.460419654846191,
3.5651490688323975
],
[
-0.4557429850101471,
10.784269332885742,
3.686079263687134
],
... | [
[
12,
14,
11
],
[
4,
15,
11
],
[
3,
14,
10
],
[
2,
15,
10
],
[
9,
15,
13
],
[
2,
14,
10
],
[
0,
15,
7
],
[
0,
14,
0
],
[
0,
14,
0
]
] | |
batch_20260503_204200 | BVPJjH7nRfWo7GlDhSnEAA | 3 | 151 | 160 | 4 | Follow the road to the right turn. | [
{
"text": "Go straight on the road",
"frames": [
0,
1,
2,
3,
4,
5
],
"instruction_type": "ROUTE"
},
{
"text": "Turn right",
"frames": [
6,
7,
8,
9
],
"instruction_type": "ROUTE"
}
] | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3,
4,
5
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn right",
"frames": [
6,
7,
8,
9
],
"instruction_type": "SPATIAL"
}
] | [
{
"text": "approach the <orange_truck>",
"frames": [
3,
4,
5
],
"instruction_type": "LANDMARK"
},
{
"text": "pass the <guardrail>",
"frames": [
6,
7,
8
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "pass the <orange_truck>",
"frames": [
3,
4,
5
],
"referenced_landmarks": [
"orange_truck"
],
"instruction_type": "LANDMARK"
},
{
"text": "turn right past the <guardrail>",
"frames": [
6,
7,
8,
9
],
"referenced_lan... | [] | [] | [
{
"style": "landmark_heavy",
"text": "Go straight past the <orange_truck>, then turn right where you see the <guardrail>.",
"referenced_landmarks": [
"orange_truck",
"guardrail"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "MULTI-CLAUSE"
... | [
[
-0.6344704031944275,
11.878761291503906,
3.2485511302948
],
[
-1.7606478929519653,
16.612829208374023,
-1.3394173383712769
],
[
-0.8797830939292908,
12.679222106933594,
2.0502371788024902
],
[
-0.40899884700775146,
11.500144958496094,
4.794103145599365
... | [
[
2,
15,
10
],
[
0,
15,
6
],
[
1,
15,
9
],
[
3,
15,
11
],
[
2,
15,
10
],
[
0,
15,
10
],
[
14,
15,
14
],
[
14,
15,
14
],
[
15,
15,
14
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 0 | 20 | 29 | 3 | Reach the end of the wooded trail. | [
{
"text": "Follow the path as it curves left.",
"frames": [
0,
1,
2
],
"instruction_type": "ROUTE"
},
{
"text": "Turn left at the fork in the path.",
"frames": [
3,
4
],
"instruction_type": "ROUTE"
},
{
"text": "Continue straight on the path.... | [
{
"primitive": "turn left",
"frames": [
0,
1,
2
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn left",
"frames": [
3,
4
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
5,
6
],
"in... | [
{
"text": "approach the <fork_in_path>",
"frames": [
3,
4,
5
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "turn left at the <fork_in_path>",
"frames": [
3,
4,
5
],
"referenced_landmarks": [
"fork_in_path"
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "follow the unpaved path",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [] | [
{
"style": "landmark_heavy",
"text": "Turn left at the <fork_in_path> and continue straight on the <dirt_path>.",
"referenced_landmarks": [
"fork_in_path",
"dirt_path"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "MULTI-CLAUSE"
},
{
... | [
[
-0.09925159811973572,
0.9408198595046997,
-20.294124603271484
],
[
-0.10362701117992401,
0.44869524240493774,
-10.501062393188477
],
[
-0.3267000615596771,
0.6349146366119385,
-31.316865921020508
],
[
0.02664766274392605,
0.7706020474433899,
1.2095623016... | [
[
6,
1,
0
],
[
6,
0,
2
],
[
3,
1,
0
],
[
8,
1,
9
],
[
2,
1,
0
],
[
10,
1,
6
],
[
4,
5,
10
],
[
10,
1,
13
],
[
6,
2,
3
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 1 | 30 | 39 | 2 | Reach the end of the path. | [
{
"text": "Start by going forward on the path.",
"frames": [
0,
1,
2
],
"instruction_type": "ROUTE"
},
{
"text": "Navigate the first slight right turn.",
"frames": [
3
],
"instruction_type": "ROUTE"
},
{
"text": "Continue forward.",
"frames": [... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2
],
"instruction_type": "SPATIAL"
},
{
"primitive": "slight right",
"frames": [
3
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
4,
5
],
"instru... | [] | [
{
"text": "follow the <dirt_path>",
"frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"referenced_landmarks": [
"dirt_path"
],
"instruction_type": "LANDMARK"
},
{
"text": "approach the end of the <dirt_path>",
"frames": ... | [
{
"text": "stay on the clear path",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [] | [
{
"style": "landmark_heavy",
"text": "Follow the <dirt_path> through the <trees> and <foliage> until you reach the end.",
"referenced_landmarks": [
"dirt_path",
"trees",
"foliage"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "MULTI-CL... | [
[
0.12372726947069168,
1.016073226928711,
9.122325897216797
],
[
-0.016063807532191277,
0.8597695231437683,
-3.459099054336548
],
[
0.019432753324508667,
0.8267778158187866,
-6.731746673583984
],
[
0.15498340129852295,
0.5900874733924866,
10.02324771881103... | [
[
10,
2,
13
],
[
7,
1,
5
],
[
8,
1,
3
],
[
11,
0,
13
],
[
9,
1,
12
],
[
7,
1,
8
],
[
3,
3,
1
],
[
7,
0,
11
],
[
7,
4,
11
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 2 | 125 | 134 | 4 | Navigate through the wooded path, taking the right fork. | [
{
"text": "Go straight along the path",
"frames": [
0,
1,
2,
3
],
"instruction_type": "ROUTE"
},
{
"text": "Turn right at the fork",
"frames": [
4,
5,
6
],
"instruction_type": "ROUTE"
},
{
"text": "Continue on the right path",
... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn right",
"frames": [
4,
5,
6
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
7,
... | [
{
"text": "approach the <fork_in_path>",
"frames": [
3,
4,
5
],
"instruction_type": "LANDMARK"
},
{
"text": "follow the <right_path>",
"frames": [
6,
7,
8,
9
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "approach the <fork_in_path>",
"frames": [
3,
4,
5
],
"referenced_landmarks": [
"fork_in_path"
],
"instruction_type": "LANDMARK"
},
{
"text": "turn right at the <fork_in_path>",
"frames": [
4,
5,
6
],
"referenced_landmar... | [] | [
{
"text": "Do not take the left path",
"evidence_frames": [
3,
4,
5,
6
]
}
] | [
{
"style": "landmark_heavy",
"text": "Go straight, then turn right at the <fork_in_path>, and continue on the <right_path>.",
"referenced_landmarks": [
"fork_in_path",
"right_path"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "MULTI-CLAUSE"... | [
[
-0.1626894772052765,
1.6710041761398315,
-0.06821069121360779
],
[
-0.1292932629585266,
2.144803524017334,
1.610018014907837
],
[
-0.1304183304309845,
1.607423186302185,
-2.7564773559570312
],
[
0.06336266547441483,
1.9082094430923462,
5.380983829498291
... | [
[
5,
4,
7
],
[
5,
5,
9
],
[
5,
4,
5
],
[
9,
5,
11
],
[
9,
4,
10
],
[
5,
4,
3
],
[
11,
3,
13
],
[
3,
5,
4
],
[
7,
3,
7
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 3 | 223 | 232 | 2 | Follow the dirt path, curving to the right. | [
{
"text": "Go straight on the dirt path towards the bridge",
"frames": [
0,
1,
2,
3
],
"instruction_type": "ROUTE"
},
{
"text": "Curve right on the path",
"frames": [
4,
5,
6
],
"instruction_type": "ROUTE"
},
{
"text": "Curve righ... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn right",
"frames": [
4,
5,
6
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn right",
"frames": [
7,
... | [
{
"text": "pass the <logs>",
"frames": [
6,
7,
8
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "follow the <dirt_path>",
"frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"referenced_landmarks": [
"dirt_path"
],
"instruction_type": "LANDMARK"
},
{
"text": "pass the <logs>",
"frames": [
6,
7,
... | [] | [] | [
{
"style": "landmark_heavy",
"text": "Go straight on the <dirt_path>, then turn right past the <logs>.",
"referenced_landmarks": [
"dirt_path",
"logs"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "MULTI-CLAUSE"
},
{
"style": "task_o... | [
[
0.006694782059639692,
1.5643305778503418,
1.7746537923812866
],
[
0.019597552716732025,
1.5158556699752808,
-1.2040082216262817
],
[
0.07551747560501099,
1.398884892463684,
1.9826526641845703
],
[
0.19104799628257751,
1.4093483686447144,
9.707763671875
... | [
[
8,
4,
9
],
[
8,
3,
6
],
[
9,
3,
9
],
[
11,
3,
13
],
[
11,
3,
13
],
[
9,
2,
12
],
[
10,
2,
12
],
[
10,
2,
14
],
[
8,
2,
12
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 4 | 243 | 252 | 2 | Reach the paved road under the bridge. | [
{
"text": "Follow the dirt path",
"frames": [
0,
1,
2,
3
],
"instruction_type": "ROUTE"
},
{
"text": "Turn left onto the paved road",
"frames": [
4,
5
],
"instruction_type": "ROUTE"
},
{
"text": "Continue on the paved road towards the b... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn left",
"frames": [
4,
5
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
6,
7,
... | [
{
"text": "approach the <logs>",
"frames": [
2,
3,
4
],
"instruction_type": "LANDMARK"
},
{
"text": "turn onto the <paved_road>",
"frames": [
3,
4,
5
],
"instruction_type": "LANDMARK"
},
{
"text": "pass under the <bridge>",
"frames"... | [
{
"text": "pass the <logs> on your right",
"frames": [
3,
4,
5
],
"referenced_landmarks": [
"logs"
],
"instruction_type": "LANDMARK"
},
{
"text": "approach the <bridge>",
"frames": [
6,
7,
8,
9
],
"referenced_landmarks": [
... | [] | [] | [
{
"style": "landmark_heavy",
"text": "Go straight along the <dirt_path>, then turn left past the <logs> onto the <paved_road> towards the <bridge>.",
"referenced_landmarks": [
"dirt_path",
"logs",
"paved_road",
"bridge"
],
"referenced_preferences": [],
"referenced_neg... | [
[
-0.08284523338079453,
1.059425950050354,
-3.958009958267212
],
[
-0.0748167559504509,
1.0478057861328125,
-2.4083707332611084
],
[
-0.1435575932264328,
0.9668474793434143,
-9.709217071533203
],
[
0.01800784282386303,
1.003153681755066,
2.639237642288208
... | [
[
6,
2,
5
],
[
6,
2,
5
],
[
5,
1,
2
],
[
8,
2,
10
],
[
5,
1,
1
],
[
6,
1,
1
],
[
9,
0,
10
],
[
7,
0,
4
],
[
7,
0,
3
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 5 | 283 | 292 | 4 | Navigate through the underpass and continue on the path. | [
{
"text": "Approach the underpass and turn left",
"frames": [
0,
1,
2,
3
],
"instruction_type": "ROUTE"
},
{
"text": "Go straight under the underpass",
"frames": [
4,
5
],
"instruction_type": "ROUTE"
},
{
"text": "Turn right onto the ne... | [
{
"primitive": "go straight",
"frames": [
0
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn left",
"frames": [
1,
2,
3
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
4,
5
],
"instructi... | [
{
"text": "approach the <prohibitory_sign>",
"frames": [
0
],
"instruction_type": "LANDMARK"
},
{
"text": "turn left at the <underpass>",
"frames": [
1,
2,
3
],
"instruction_type": "LANDMARK"
},
{
"text": "pass the <car>",
"frames": [
4
... | [
{
"text": "turn left past the <prohibitory_sign>",
"frames": [
1,
2,
3
],
"referenced_landmarks": [
"prohibitory_sign"
],
"instruction_type": "LANDMARK"
},
{
"text": "pass through the <underpass>",
"frames": [
4,
5
],
"referenced_landma... | [
{
"text": "stay on the paved path",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [
{
"text": "do not enter the road",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [
{
"style": "landmark_heavy",
"text": "Go straight towards the <prohibitory_sign>, then turn left under the <underpass>, and finally turn right at the <bike_path_sign>.",
"referenced_landmarks": [
"prohibitory_sign",
"underpass",
"bike_path_sign"
],
"referenced_preferences": [],... | [
[
-0.0158984437584877,
1.8395498991012573,
-0.6692543029785156
],
[
-0.2363356053829193,
1.7445600032806396,
-11.698347091674805
],
[
-0.20449483394622803,
1.7212557792663574,
-12.963459968566895
],
[
-0.1781182736158371,
1.576712965965271,
-14.89612102508... | [
[
7,
4,
7
],
[
4,
4,
2
],
[
4,
4,
1
],
[
5,
4,
1
],
[
9,
4,
5
],
[
10,
3,
9
],
[
11,
4,
13
],
[
9,
3,
12
],
[
9,
3,
13
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 6 | 323 | 332 | 2 | Reach the area under the bridge. | [
{
"text": "Go straight towards the first intersection.",
"frames": [
0,
1,
2
],
"instruction_type": "ROUTE"
},
{
"text": "Turn left at the first intersection.",
"frames": [
3
],
"instruction_type": "ROUTE"
},
{
"text": "Go straight towards the seco... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn left",
"frames": [
3
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
4,
5
],
"instructi... | [
{
"text": "approach the <sign>",
"frames": [
0,
1,
2
],
"instruction_type": "LANDMARK"
},
{
"text": "turn left at the <first_intersection>",
"frames": [
2,
3,
4
],
"instruction_type": "LANDMARK"
},
{
"text": "turn right at the <second_i... | [
{
"text": "pass the <sign>",
"frames": [
0,
1,
2
],
"referenced_landmarks": [
"sign"
],
"instruction_type": "LANDMARK"
},
{
"text": "turn left at the <first_intersection>",
"frames": [
2,
3,
4
],
"referenced_landmarks": [
"f... | [
{
"text": "Stay on the paved path.",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [] | [
{
"style": "landmark_heavy",
"text": "Pass the <sign>, then turn left at the first <first_intersection>, and finally turn right at the second <second_intersection> to go under the <bridge_underpass>.",
"referenced_landmarks": [
"sign",
"first_intersection",
"second_intersection",
... | [
[
0.05724096670746803,
1.1239055395126343,
-3.3084094524383545
],
[
0.1320333331823349,
1.1320207118988037,
7.959430694580078
],
[
-0.14146052300930023,
1.1832072734832764,
-7.073527812957764
],
[
-0.19563914835453033,
1.0463309288024902,
-18.1731815338134... | [
[
9,
2,
5
],
[
10,
2,
12
],
[
5,
2,
3
],
[
4,
2,
1
],
[
9,
2,
3
],
[
12,
2,
15
],
[
12,
2,
15
],
[
9,
2,
14
],
[
7,
2,
12
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 7 | 373 | 382 | 4 | Navigate along the winding road. | [
{
"text": "Start by moving backward.",
"frames": [
0,
1
],
"instruction_type": "ROUTE"
},
{
"text": "Proceed straight forward along the road.",
"frames": [
2,
3,
4,
5
],
"instruction_type": "ROUTE"
},
{
"text": "Make a right turn.",
... | [
{
"primitive": "reverse",
"frames": [
0,
1
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
2,
3,
4,
5
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn right",
"frames": [
6
],
"ins... | [] | [
{
"text": "Follow the <road> as it curves right.",
"frames": [
2,
3,
4,
5,
6,
7,
8
],
"referenced_landmarks": [
"road"
],
"instruction_type": "LANDMARK"
}
] | [] | [] | [
{
"style": "landmark_heavy",
"text": "Start by reversing on the <road>, then go straight past the <trees>, and turn right following the curve of the <road>.",
"referenced_landmarks": [
"road",
"trees"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_t... | [
[
-0.2172011435031891,
4.787928581237793,
0.8630407452583313
],
[
0.08019580692052841,
-2.2761003971099854,
-7.42952299118042
],
[
0.4473692774772644,
4.631545066833496,
7.302987575531006
],
[
-0.019217269495129585,
2.4047863483428955,
1.1736586093902588
... | [
[
4,
11,
8
],
[
9,
0,
3
],
[
13,
11,
12
],
[
7,
6,
9
],
[
7,
6,
7
],
[
9,
6,
10
],
[
12,
6,
13
],
[
10,
5,
11
],
[
10,
10,
10
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 8 | 602 | 611 | 2 | Follow the dirt path through the wooded area. | [
{
"text": "Start by turning right onto the path.",
"frames": [
0
],
"instruction_type": "ROUTE"
},
{
"text": "Continue straight along the path.",
"frames": [
1,
2,
3
],
"instruction_type": "ROUTE"
},
{
"text": "Turn left at the fork in the path.",
... | [
{
"primitive": "turn right",
"frames": [
0
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
1,
2,
3
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn left",
"frames": [
4
],
"instruction_type": ... | [
{
"text": "approach the <orange_cone>",
"frames": [
0,
1,
2,
3
],
"instruction_type": "LANDMARK"
},
{
"text": "pass the <orange_cone>",
"frames": [
4,
5,
6
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "turn right onto the <dirt_path>",
"frames": [
0
],
"referenced_landmarks": [
"dirt_path"
],
"instruction_type": "LANDMARK"
},
{
"text": "turn left at the fork in the <dirt_path>",
"frames": [
4
],
"referenced_landmarks": [
"dirt_path"
... | [
{
"text": "stay on the main path",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [
{
"text": "do not take the path to the right",
"evidence_frames": [
2,
3,
4
]
}
] | [
{
"style": "landmark_heavy",
"text": "Turn right onto the <dirt_path>, then turn left at the fork, continuing past the <orange_cone>.",
"referenced_landmarks": [
"dirt_path",
"orange_cone"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "MULTI... | [
[
0.05710113048553467,
0.6503933668136597,
11.147322654724121
],
[
-0.036753762513399124,
0.7231565117835999,
5.045970916748047
],
[
-0.07159151136875153,
0.46494510769844055,
-3.378911018371582
],
[
-0.02775726467370987,
0.4166329503059387,
4.193874835968... | [
[
9,
1,
13
],
[
7,
1,
11
],
[
6,
0,
5
],
[
7,
0,
11
],
[
5,
0,
0
],
[
9,
0,
10
],
[
8,
0,
8
],
[
10,
2,
12
],
[
9,
0,
5
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 9 | 652 | 661 | 3 | Follow the road around the building. | [
{
"text": "Proceed straight along the road towards the building.",
"frames": [
0,
1,
2,
3
],
"instruction_type": "ROUTE"
},
{
"text": "Turn left around the building.",
"frames": [
4,
5,
6,
7,
8
],
"instruction_type": "ROUTE"
... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn left",
"frames": [
4,
5,
6,
7,
8
],
"instruction_type": "SPATIAL"
},
{
"primitive": "slight right",
"fra... | [
{
"text": "approach the <building>",
"frames": [
2,
3,
4
],
"instruction_type": "LANDMARK"
},
{
"text": "pass the <fence>",
"frames": [
4,
5,
6
],
"instruction_type": "LANDMARK"
},
{
"text": "continue past the <trees>",
"frames": [
... | [
{
"text": "approach the <building>",
"frames": [
0,
1,
2,
3,
4
],
"referenced_landmarks": [
"building"
],
"instruction_type": "LANDMARK"
},
{
"text": "turn left past the <fence>",
"frames": [
4,
5,
6,
7,
8
],
... | [] | [] | [
{
"style": "landmark_heavy",
"text": "Go straight towards the <building>, then turn left around it, passing the wooden <fence>.",
"referenced_landmarks": [
"building",
"fence"
],
"referenced_preferences": [],
"referenced_negations": [],
"instruction_type": "MULTI-CLAUSE"
},... | [
[
0.04522133991122246,
1.8408981561660767,
2.7782936096191406
],
[
0.05267980322241783,
1.681104302406311,
4.709746837615967
],
[
-0.012026496231555939,
1.4724860191345215,
1.1733143329620361
],
[
-0.001155050820671022,
2.5373239517211914,
1.12648606300354... | [
[
9,
4,
10
],
[
9,
4,
11
],
[
8,
3,
9
],
[
8,
6,
9
],
[
9,
0,
8
],
[
4,
6,
2
],
[
5,
2,
1
],
[
4,
1,
0
],
[
5,
1,
0
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 10 | 672 | 681 | 3 | Navigate the curve in the road. | [
{
"text": "Proceed straight past the construction barriers.",
"frames": [
0,
1,
2,
3,
4
],
"instruction_type": "ROUTE"
},
{
"text": "Turn right, following the curve of the road.",
"frames": [
5,
6,
7
],
"instruction_type": "ROUTE"
... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3,
4
],
"instruction_type": "SPATIAL"
},
{
"primitive": "turn right",
"frames": [
5,
6,
7
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
... | [
{
"text": "pass the <construction_barrier>",
"frames": [
2,
3,
4
],
"instruction_type": "LANDMARK"
},
{
"text": "approach the <building>",
"frames": [
4,
5,
6
],
"instruction_type": "LANDMARK"
}
] | [
{
"text": "pass the <construction_barrier>",
"frames": [
0,
1,
2,
3,
4
],
"referenced_landmarks": [
"construction_barrier"
],
"instruction_type": "LANDMARK"
},
{
"text": "turn right around the <building>",
"frames": [
5,
6,
7
... | [] | [] | [
{
"style": "landmark_heavy",
"text": "Go straight, passing the orange <construction_barrier>s on your right, then follow the <road> as it curves past the brick <building>.",
"referenced_landmarks": [
"construction_barrier",
"road",
"building"
],
"referenced_preferences": [],
... | [
[
0.043891578912734985,
1.2553516626358032,
-0.0605335459113121
],
[
-0.021218135952949524,
1.2909200191497803,
-2.6972432136535645
],
[
-0.07621685415506363,
1.2531973123550415,
-9.329622268676758
],
[
0.006401075981557369,
1.3266023397445679,
-3.78971743... | [
[
9,
2,
7
],
[
7,
3,
5
],
[
6,
2,
2
],
[
8,
3,
5
],
[
9,
2,
8
],
[
12,
3,
14
],
[
9,
3,
12
],
[
10,
3,
14
],
[
5,
3,
4
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 11 | 874 | 883 | 3 | Navigate along the dirt path past the parked cars. | [
{
"text": "Follow the path as it curves left past the brick building.",
"frames": [
0,
1,
2,
3,
4
],
"instruction_type": "ROUTE"
},
{
"text": "Continue along the path, passing the parked cars towards the end.",
"frames": [
5,
6,
7,
8,... | [
{
"primitive": "slight left",
"frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"instruction_type": "SPATIAL"
}
] | [
{
"text": "approach the brick_building",
"frames": [
0,
1,
2
],
"instruction_type": "LANDMARK"
},
{
"text": "pass the brick_building",
"frames": [
3,
4,
5
],
"instruction_type": "LANDMARK"
},
{
"text": "pass the parked_car",
"frames... | [
{
"text": "follow the path past the <brick_building>",
"frames": [
0,
1,
2,
3,
4
],
"referenced_landmarks": [
"brick_building"
],
"instruction_type": "LANDMARK"
},
{
"text": "continue past the <parked_car>",
"frames": [
5,
6,
... | [] | [] | [
{
"style": "landmark_heavy",
"text": "Follow the <dirt_path> as it curves left past the <brick_building>, then continue past the <parked_car> towards the <utility_pole>.",
"referenced_landmarks": [
"dirt_path",
"brick_building",
"parked_car",
"utility_pole"
],
"referenced... | [
[
-0.17739903926849365,
0.19495441019535065,
-40.30282211303711
],
[
0.19359570741653442,
0.6085454225540161,
12.574259757995605
],
[
-0.19366222620010376,
0.6169676780700684,
-19.779869079589844
],
[
0.07937054336071014,
0.7234037518501282,
-0.43431749939... | [
[
5,
0,
0
],
[
11,
1,
14
],
[
4,
1,
0
],
[
9,
1,
7
],
[
6,
1,
3
],
[
3,
2,
0
],
[
10,
2,
4
],
[
11,
3,
12
],
[
7,
4,
3
]
] | |
batch_20260503_204200 | BVTEfGnu5kuG8adTFSVvTK | 12 | 894 | 903 | 3 | Reach the end of the unpaved path. | [
{
"text": "Go straight past the parked car and house on the left.",
"frames": [
0,
1,
2
],
"instruction_type": "ROUTE"
},
{
"text": "Continue forward, passing the house on the right.",
"frames": [
3,
4
],
"instruction_type": "ROUTE"
},
{
"tex... | [
{
"primitive": "go straight",
"frames": [
0,
1,
2,
3,
4
],
"instruction_type": "SPATIAL"
},
{
"primitive": "slight right",
"frames": [
5,
6,
7,
8,
9
],
"instruction_type": "SPATIAL"
}
] | [
{
"text": "pass the <car>",
"frames": [
0,
1,
2
],
"instruction_type": "LANDMARK"
},
{
"text": "approach the <house_right>",
"frames": [
2,
3,
4
],
"instruction_type": "LANDMARK"
},
{
"text": "follow the <unpaved_road>",
"frames": [... | [
{
"text": "pass the <car>",
"frames": [
0,
1,
2
],
"referenced_landmarks": [
"car"
],
"instruction_type": "LANDMARK"
},
{
"text": "pass the <house_right>",
"frames": [
3,
4
],
"referenced_landmarks": [
"house_right"
],
"in... | [
{
"text": "stay on the unpaved road",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [] | [
{
"style": "landmark_heavy",
"text": "Go straight past the <car> and <house_left>, then continue along the <hedge> until the end of the path.",
"referenced_landmarks": [
"car",
"house_left",
"hedge"
],
"referenced_preferences": [],
"referenced_negations": [],
"instructi... | [
[
-0.032565973699092865,
1.8653745651245117,
1.4743812084197998
],
[
-0.023933321237564087,
1.8856337070465088,
2.22152042388916
],
[
-0.10016293823719025,
1.8534033298492432,
-0.28810980916023254
],
[
-0.19067798554897308,
2.020669937133789,
-4.9247417449... | [
[
7,
4,
9
],
[
7,
4,
9
],
[
6,
4,
7
],
[
5,
5,
4
],
[
4,
5,
2
],
[
12,
6,
13
],
[
4,
6,
4
],
[
7,
6,
6
],
[
7,
6,
6
]
] | |
batch_20260503_204200 | BVTfGH8SMp25o37Gayuvow | 0 | 0 | 9 | 4 | Reach the crosswalk. | [
{
"text": "Follow the sidewalk straight ahead.",
"frames": [
0,
1,
2,
3,
4,
5
],
"instruction_type": "ROUTE"
},
{
"text": "Make a left curve then a sharp right turn.",
"frames": [
6,
7
],
"instruction_type": "ROUTE"
},
{
"te... | [
{
"primitive": "stationary",
"frames": [
0
],
"instruction_type": "SPATIAL"
},
{
"primitive": "go straight",
"frames": [
1,
2,
3,
4,
5
],
"instruction_type": "SPATIAL"
},
{
"primitive": "slight left",
"frames": [
6
],
... | [
{
"text": "approach the <traffic_sign>",
"frames": [
3,
4,
5
],
"instruction_type": "LANDMARK"
},
{
"text": "pass the <building>",
"frames": [
4,
5
],
"instruction_type": "LANDMARK"
},
{
"text": "head towards the <overpass>",
"frames": [
... | [
{
"text": "pass the <building>",
"frames": [
4,
5
],
"referenced_landmarks": [
"building"
],
"instruction_type": "LANDMARK"
},
{
"text": "turn right past the <traffic_sign>",
"frames": [
6,
7
],
"referenced_landmarks": [
"traffic_sign"
... | [
{
"text": "follow the dedicated pedestrian path",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
]
}
] | [
{
"text": "do not enter the road",
"evidence_frames": [
0,
1,
2,
3,
4,
5,
6,
7
]
}
] | [
{
"style": "landmark_heavy",
"text": "Go straight past the <building> and the <traffic_sign>, then turn right under the <overpass> to reach the <crosswalk>.",
"referenced_landmarks": [
"building",
"traffic_sign",
"overpass",
"crosswalk"
],
"referenced_preferences": [],
... | [
[
-0.1969793289899826,
0.017136244103312492,
0.3835388720035553
],
[
0.39379167556762695,
1.8842973709106445,
6.932840824127197
],
[
0.2984149754047394,
4.476743698120117,
6.847722053527832
],
[
0.7086344957351685,
6.3267107009887695,
4.854414939880371
]... | [
[
4,
0,
8
],
[
13,
4,
12
],
[
12,
10,
12
],
[
14,
13,
11
],
[
13,
14,
5
],
[
15,
14,
11
],
[
7,
14,
1
],
[
15,
14,
14
],
[
15,
14,
11
]
] |
End of preview. Expand in Data Studio
nav3 training data
Materialized stage-07 output from the nav3 V2 pipeline. Each row is one priority-frame-anchored chunk of a Mapillary pedestrian/cyclist sequence, already paired with bucketized action tokens and the full 8-level Gemini instruction hierarchy (L1–L8).
Source batch: batch_20260503_204200
Columns
history_images_rgb/future_images_rgb— PIL images split at the priority frame.history_images_rgbincludes the priority frame as its last entry.history_actions/future_actions—(N, 3)arrays of(dx, dz, dyaw_deg)in cam-i frame.history_action_tokens/future_action_tokens— bucketized into the quantile bins shipped asaction_bins.jsonin this repo (n_bins = 16).task_instruction(L1, str) — trajectory-level goal.subtask_instructions(L2, list[str]) — ordered subtask segments, future-only.motion_instructions(L3, list[str]) — atomic motion primitives, future-only.pixel_atomic_instructions(L4, list[str]) — landmark-anchored pixel-level instructions, future-only.keypoint_atomic_instructions(L5, list[str]) — keypoint-trace-anchored instructions, future-only.preferences(L6, list[str]) — trajectory-level preferences.negations(L7, list[str]) — trajectory-level negations.combination_instructions(L8, list[{style, text}]) — compositional variations (up to 15 styles per chunk; null slots from Gemini are dropped).
Action bins
action_bins.json is a sidecar file with the quantile breakpoints used to
bucketize (dx, dz, dyaw_deg) into the integer action tokens. Load via
huggingface_hub.hf_hub_download(repo_id, "action_bins.json", repo_type="dataset").
- Downloads last month
- 10