Verify hosted English question rebuild and benchmark exclusion

english-task-verification.json

@@ -0,0 +1,66 @@
+{
+  "data_revision": "015600d0873de20a5d9f418ca412a05d627d906c",
+  "tasksource_commit": "a78550424f5c2c6dc5cc2dcdaf9db232604cdcb0",
+  "task_definitions_sha256": "bd3275f5fac8ecf5594939e40e1927b0097bbe782ea6d42007f4f4d1a75090e0",
+  "updated_parquet_sha256_verified": 35,
+  "other_parquet_unchanged": 56,
+  "vision_parquet_unchanged": 50,
+  "hosted_test_examples_checked": {
+    "quarel": 11,
+    "missing-item-prediction/contrastive": 11,
+    "goal-step-wikihow/order": 19
+  },
+  "examples": {
+    "goal-step-wikihow/order": {
+      "state": "Season Cast Iron in the Oven",
+      "question": "Which step should be done first to achieve this goal?",
+      "options": [
+        "Bake the cast iron in the oven for 1 hour.",
+        "Fit a piece of aluminum foil underneath the cast iron on the bottom rack."
+      ],
+      "target": [
+        0.0,
+        1.0
+      ]
+    },
+    "missing-item-prediction/contrastive": {
+      "state": "text_A: Magnesium, Beryllium, Helium, Silicon, Sulfur, Lithium, Phosphorus, Chlorine, Oxygen, Neon, Boron, Carbon, Fluorine, Nitrogen, Aluminium, Hydrogen\ntext_B: Sodium",
+      "question": "Does the list in text_A contain the item in text_B?",
+      "options": [
+        "no",
+        "yes"
+      ],
+      "target": [
+        1.0,
+        0.0
+      ]
+    },
+    "quarel": {
+      "state": "Joe and Mike are walking down the same street at the same pace. If Mike walks for one more hour than Joe, who travelled further?",
+      "question": "Which option correctly answers or completes the question?",
+      "options": [
+        "Mike.",
+        "Joe"
+      ],
+      "target": [
+        1.0,
+        0.0
+      ]
+    }
+  },
+  "rows": {
+    "default": {
+      "test": 15098,
+      "train": 1026061,
+      "validation": 14977
+    },
+    "full": {
+      "test": 15098,
+      "train": 2523746,
+      "validation": 14977
+    },
+    "filtered-full": {
+      "train": 1305072
+    }
+  }
+}