{
 "sets": {
  "test_count_fresh": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 600,
    "pictures": 600,
    "a": 0.74,
    "b": 0.7633333333333333,
    "difference": 0.023333333333333317,
    "ci95": [
     -0.0050000000000000044,
     0.05166666666666664
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 600,
    "pictures": 600,
    "a": 0.5763476151482702,
    "b": 0.5738475317419256,
    "difference": -0.0025000834063445554,
    "ci95": [
     -0.029590039246708333,
     0.02324226594506207
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "bench_countbench": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 491,
    "pictures": 491,
    "a": 0.1894093686354379,
    "b": 0.3564154786150713,
    "difference": 0.1670061099796334,
    "ci95": [
     0.12830957230142565,
     0.20977596741344195
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 491,
    "pictures": 491,
    "a": 2.249204785673313,
    "b": 1.6021778431421219,
    "difference": -0.647026942531191,
    "ci95": [
     -0.7236271559411285,
     -0.5697184849200367
    ],
    "resamples": 2000
   },
   "nll_warning": false
  },
  "test_vsr": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 904,
    "pictures": 552,
    "a": 0.6792035398230089,
    "b": 0.661504424778761,
    "difference": -0.017699115044247815,
    "ci95": [
     -0.03871681415929196,
     0.003293084522502765
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 904,
    "pictures": 552,
    "a": 0.6155330847335275,
    "b": 0.6197663897233163,
    "difference": 0.0042333049897888,
    "ci95": [
     -0.008103858831641997,
     0.016373944210066993
    ],
    "resamples": 2000
   },
   "held": false,
   "proven_worse": false,
   "nll_warning": false
  },
  "test_v7w": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 1000,
    "pictures": 1000,
    "a": 0.721,
    "b": 0.721,
    "difference": 0.0,
    "ci95": [
     -0.017000000000000015,
     0.018000000000000016
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 1000,
    "pictures": 1000,
    "a": 0.6838468317414087,
    "b": 0.6725771682747884,
    "difference": -0.011269663466620283,
    "ci95": [
     -0.03041221689888618,
     0.007412588911638762
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "bench_pope": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 8676,
    "pictures": 482,
    "a": 0.851083448593822,
    "b": 0.8495850622406639,
    "difference": -0.001498386353158132,
    "ci95": [
     -0.008529276164130906,
     0.005301982480405698
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 8676,
    "pictures": 482,
    "a": 0.3702472832864825,
    "b": 0.36794475743031013,
    "difference": -0.002302525856172355,
    "ci95": [
     -0.01004565387165457,
     0.005636645091598225
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "test_vqa_yesno": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 1000,
    "pictures": 984,
    "a": 0.71,
    "b": 0.704,
    "difference": -0.006000000000000005,
    "ci95": [
     -0.021956087824351322,
     0.01106639839034207
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 1000,
    "pictures": 984,
    "a": 0.5768279689372598,
    "b": 0.5741047062190436,
    "difference": -0.002723262718216213,
    "ci95": [
     -0.012621285357050205,
     0.0073198649324632825
    ],
    "resamples": 2000
   },
   "held": false,
   "proven_worse": false,
   "nll_warning": false
  },
  "test_vqa_choice": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 1420,
    "pictures": 1346,
    "a": 0.897887323943662,
    "b": 0.9091549295774648,
    "difference": 0.011267605633802802,
    "ci95": [
     0.0020862308762169546,
     0.022487702037948032
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 1420,
    "pictures": 1346,
    "a": 0.2706275980383607,
    "b": 0.2705435796140065,
    "difference": -8.401842435418194e-05,
    "ci95": [
     -0.011447355800969727,
     0.011158424151348323
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "test_gqa": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 992,
    "pictures": 882,
    "a": 0.7842741935483871,
    "b": 0.7903225806451613,
    "difference": 0.006048387096774133,
    "ci95": [
     -0.011033099297893645,
     0.02300000000000002
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 992,
    "pictures": 882,
    "a": 0.4089840430444238,
    "b": 0.4104783294623267,
    "difference": 0.0014942864179028903,
    "ci95": [
     -0.013564724731577937,
     0.016149505199867686
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "test_exist": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 1120,
    "pictures": 1099,
    "a": 0.9383928571428571,
    "b": 0.9473214285714285,
    "difference": 0.008928571428571397,
    "ci95": [
     -0.000889679715302516,
     0.018766756032171594
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 1120,
    "pictures": 1099,
    "a": 0.16199041568107028,
    "b": 0.16308653751641133,
    "difference": 0.0010961218353410518,
    "ci95": [
     -0.012942116093576261,
     0.01502907012673671
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "test_position": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 1274,
    "pictures": 1231,
    "a": 0.8720565149136578,
    "b": 0.8846153846153846,
    "difference": 0.012558869701726816,
    "ci95": [
     -0.0007886435331231345,
     0.026025236593059997
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 1274,
    "pictures": 1231,
    "a": 0.3045267949584121,
    "b": 0.2882323744865599,
    "difference": -0.0162944204718522,
    "ci95": [
     -0.03009423881355039,
     -0.003500244757008719
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "test_relation": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 1950,
    "pictures": 1453,
    "a": 0.7107692307692308,
    "b": 0.7548717948717949,
    "difference": 0.04410256410256408,
    "ci95": [
     0.028018339276617454,
     0.059034907597535935
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 1950,
    "pictures": 1453,
    "a": 0.5401162626494179,
    "b": 0.4896254970901639,
    "difference": -0.050490765559254025,
    "ci95": [
     -0.061521469864747935,
     -0.038556349244042165
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "test_size": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 1100,
    "pictures": 975,
    "a": 0.86,
    "b": 0.8718181818181818,
    "difference": 0.011818181818181839,
    "ci95": [
     -0.0009165902841430373,
     0.025711662075298514
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 1100,
    "pictures": 975,
    "a": 0.35032188023362587,
    "b": 0.3361044083603172,
    "difference": -0.014217471873308685,
    "ci95": [
     -0.028957271795469974,
     0.0003334966656227678
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "test_position_flip": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 640,
    "pictures": 636,
    "a": 0.85,
    "b": 0.846875,
    "difference": -0.0031249999999999334,
    "ci95": [
     -0.021943573667711602,
     0.014128728414442682
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 640,
    "pictures": 636,
    "a": 0.37288566299069764,
    "b": 0.35918046542178717,
    "difference": -0.013705197568910477,
    "ci95": [
     -0.029400569310068958,
     0.002968877802506875
    ],
    "resamples": 2000
   },
   "held": false,
   "proven_worse": false,
   "nll_warning": false
  },
  "test_relation_flip": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 1492,
    "pictures": 1192,
    "a": 0.6689008042895442,
    "b": 0.6977211796246648,
    "difference": 0.028820375335120607,
    "ci95": [
     0.009510869565217406,
     0.04778156996587024
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 1492,
    "pictures": 1192,
    "a": 0.585918810016869,
    "b": 0.5450049355513791,
    "difference": -0.040913874465489886,
    "ci95": [
     -0.052748318876262856,
     -0.029602205956100636
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "test_bind_gqa": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 1484,
    "pictures": 967,
    "a": 0.6785714285714286,
    "b": 0.6765498652291105,
    "difference": -0.0020215633423180668,
    "ci95": [
     -0.01720578114246385,
     0.013468013468013518
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 1484,
    "pictures": 967,
    "a": 0.56543531676673,
    "b": 0.5592760342365289,
    "difference": -0.006159282530201038,
    "ci95": [
     -0.015578151742373092,
     0.0028278953758936742
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "bench_lv_vqav2_yesno": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 4887,
    "pictures": 2069,
    "a": 0.724780028647432,
    "b": 0.7249846531614488,
    "difference": 0.0002046245140168157,
    "ci95": [
     -0.0075005067909993395,
     0.008213984835720334
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 4887,
    "pictures": 2069,
    "a": 0.551391599768774,
    "b": 0.5517217482239184,
    "difference": 0.0003301484551443412,
    "ci95": [
     -0.004196828299816491,
     0.005027468860727358
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "bench_lv_aokvqa": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 1138,
    "pictures": 1138,
    "a": 0.6256590509666081,
    "b": 0.6124780316344464,
    "difference": -0.013181019332161759,
    "ci95": [
     -0.02899824253075567,
     0.002636203866432374
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 1138,
    "pictures": 1138,
    "a": 0.9295007201330518,
    "b": 0.9357389801118688,
    "difference": 0.006238259978816996,
    "ci95": [
     -0.011132842065626924,
     0.023444144517427712
    ],
    "resamples": 2000
   },
   "held": false,
   "proven_worse": false,
   "nll_warning": false
  },
  "bench_lv_scienceqa": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 2097,
    "pictures": 2097,
    "a": 0.7663328564616119,
    "b": 0.7892226990939437,
    "difference": 0.022889842632331847,
    "ci95": [
     0.010968049594659024,
     0.03528850739151168
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 2097,
    "pictures": 2097,
    "a": 0.505658483494164,
    "b": 0.4709893216879221,
    "difference": -0.034669161806241855,
    "ci95": [
     -0.04485873743892388,
     -0.024565908726882524
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "bench_lv_pope": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 9000,
    "pictures": 500,
    "a": 0.8502222222222222,
    "b": 0.8481111111111111,
    "difference": -0.0021111111111110636,
    "ci95": [
     -0.008888888888888946,
     0.004555555555555535
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 9000,
    "pictures": 500,
    "a": 0.37292244276041664,
    "b": 0.3721321698679672,
    "difference": -0.0007902728924494284,
    "ci95": [
     -0.008944474453509432,
     0.007254876618201789
    ],
    "resamples": 2000
   },
   "held": true,
   "proven_worse": false,
   "nll_warning": false
  },
  "bench_whatsup": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 820,
    "pictures": 820,
    "a": 0.2841463414634146,
    "b": 0.2865853658536585,
    "difference": 0.0024390243902439046,
    "ci95": [
     -0.01341463414634142,
     0.018292682926829285
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 820,
    "pictures": 820,
    "a": 1.4723660060388828,
    "b": 1.4909160791754408,
    "difference": 0.018550073136557987,
    "ci95": [
     0.005068862787158679,
     0.031356396984290935
    ],
    "resamples": 2000
   },
   "nll_warning": false
  },
  "dev_mix": {
   "accuracy": {
    "metric": "accuracy",
    "questions": 5178,
    "pictures": 3087,
    "a": 0.753765932792584,
    "b": 0.7553109308613364,
    "difference": 0.0015449980687524123,
    "ci95": [
     -0.0056006179992273974,
     0.009378663540445475
    ],
    "resamples": 2000
   },
   "nll": {
    "metric": "nll",
    "questions": 5178,
    "pictures": 3087,
    "a": 0.5427544545511686,
    "b": 0.5446964962906559,
    "difference": 0.0019420417394873102,
    "ci95": [
     -0.0049800214856609015,
     0.00870632376742897
    ],
    "resamples": 2000
   },
   "nll_warning": false
  }
 },
 "primary": true,
 "protection": false,
 "not_held": [
  "test_vsr",
  "test_vqa_yesno",
  "test_position_flip",
  "bench_lv_aokvqa"
 ],
 "proven_worse": [],
 "warnings": [],
 "candidate": false
}