{"case_id": "eval-01", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 37.0, "source_country_logp": -0.3147105872631073, "target_country_logp": -7.00221061706543, "source_code_first_rank": 12422.0, "target_code_first_rank": 17839.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 11848.0, "target_country_rank": 214.5, "source_country_logp": -13.828615188598633, "target_country_logp": -7.615724563598633, "source_code_first_rank": 73770.0, "target_code_first_rank": 83657.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 1702.5, "source_country_logp": -4.828204154968262, "target_country_logp": -34.51570510864258, "source_code_first_rank": 3309.5, "target_code_first_rank": 133285.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 17588.5, "target_country_rank": 19974.0, "source_country_logp": -30.382015228271484, "target_country_logp": -30.663265228271484, "source_code_first_rank": 80098.5, "target_code_first_rank": 57298.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 86.5, "source_country_logp": -1.5602748394012451, "target_country_logp": -7.060275077819824, "source_code_first_rank": 18168.5, "target_code_first_rank": 9766.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 10983.0, "target_country_rank": 1399.0, "source_country_logp": -13.293344497680664, "target_country_logp": -9.826547622680664, "source_code_first_rank": 30901.0, "target_code_first_rank": 17172.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 677.5, "source_country_logp": -3.146188259124756, "target_country_logp": -27.396188735961914, "source_code_first_rank": 5369.0, "target_code_first_rank": 93585.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 14577.5, "target_country_rank": 60241.5, "source_country_logp": -19.352048873901367, "target_country_logp": -22.664548873901367, "source_code_first_rank": 6433.0, "target_code_first_rank": 11588.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 73.5, "source_country_logp": -0.9451602697372437, "target_country_logp": -7.070160388946533, "source_code_first_rank": 22384.0, "target_code_first_rank": 3052.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 25540.0, "target_country_rank": 20115.5, "source_country_logp": -15.551603317260742, "target_country_logp": -15.000822067260742, "source_code_first_rank": 6383.0, "target_code_first_rank": 18810.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 1212.5, "source_country_logp": -3.3856143951416016, "target_country_logp": -30.0731143951416, "source_code_first_rank": 5079.5, "target_code_first_rank": 87846.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 120974.5, "target_country_rank": 141661.5, "source_country_logp": -16.105623245239258, "target_country_logp": -17.855623245239258, "source_code_first_rank": 1410.0, "target_code_first_rank": 14375.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 15.5, "target_country_rank": 1527.5, "source_country_logp": -6.900780200958252, "target_country_logp": -14.713279724121094, "source_code_first_rank": 5510.0, "target_code_first_rank": 27247.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 249.0, "target_country_rank": 147535.0, "source_country_logp": -22.69126319885254, "target_country_logp": -46.56626510620117, "source_code_first_rank": 1.0, "target_code_first_rank": 145.5}], "actual_source_country_rank": 249.0, "actual_target_country_rank": 147535.0, "array_path": "results/readout-arrays/eval-01-main.npz"}
{"case_id": "eval-01", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 49.0, "source_country_logp": -0.39410021901130676, "target_country_logp": -6.894100189208984, "source_code_first_rank": 11854.0, "target_code_first_rank": 13654.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 12073.0, "target_country_rank": 566.0, "source_country_logp": -13.928335189819336, "target_country_logp": -8.861928939819336, "source_code_first_rank": 79374.5, "target_code_first_rank": 92263.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 1610.0, "source_country_logp": -4.70626163482666, "target_country_logp": -34.143760681152344, "source_code_first_rank": 5156.5, "target_code_first_rank": 135377.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 33994.5, "target_country_rank": 32459.5, "source_country_logp": -33.4565315246582, "target_country_logp": -33.3315315246582, "source_code_first_rank": 71231.0, "target_code_first_rank": 62513.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 238.5, "source_country_logp": -1.7808102369308472, "target_country_logp": -8.374560356140137, "source_code_first_rank": 17451.0, "target_code_first_rank": 11545.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 4346.5, "target_country_rank": 1497.5, "source_country_logp": -11.188188552856445, "target_country_logp": -9.656938552856445, "source_code_first_rank": 26566.0, "target_code_first_rank": 33252.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 977.0, "source_country_logp": -3.654458522796631, "target_country_logp": -28.40445899963379, "source_code_first_rank": 8918.0, "target_code_first_rank": 100274.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 8637.0, "target_country_rank": 40454.5, "source_country_logp": -21.22871208190918, "target_country_logp": -24.60371208190918, "source_code_first_rank": 10023.0, "target_code_first_rank": 18655.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 96.0, "source_country_logp": -0.9969416260719299, "target_country_logp": -7.434441566467285, "source_code_first_rank": 18176.0, "target_code_first_rank": 2734.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 24629.0, "target_country_rank": 23022.5, "source_country_logp": -16.84065818786621, "target_country_logp": -16.66878318786621, "source_code_first_rank": 20005.0, "target_code_first_rank": 20186.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.5, "target_country_rank": 1706.5, "source_country_logp": -3.805858612060547, "target_country_logp": -30.430858612060547, "source_code_first_rank": 7832.5, "target_code_first_rank": 88577.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 118391.0, "target_country_rank": 135439.0, "source_country_logp": -16.472143173217773, "target_country_logp": -17.659643173217773, "source_code_first_rank": 9663.5, "target_code_first_rank": 16381.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 18.0, "target_country_rank": 2465.5, "source_country_logp": -6.4904069900512695, "target_country_logp": -15.61540699005127, "source_code_first_rank": 4653.0, "target_code_first_rank": 27907.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 58.0, "target_country_rank": 123919.5, "source_country_logp": -24.875064849853516, "target_country_logp": -48.625064849853516, "source_code_first_rank": 2.0, "target_code_first_rank": 632.5}], "actual_source_country_rank": 58.0, "actual_target_country_rank": 123919.5, "array_path": "results/readout-arrays/eval-01-paraphrase.npz"}
{"case_id": "eval-02", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 6.0, "target_country_rank": 3085.5, "source_country_logp": -3.736684799194336, "target_country_logp": -11.674184799194336, "source_code_first_rank": 2361.0, "target_code_first_rank": 30667.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 14493.0, "target_country_rank": 314.0, "source_country_logp": -14.457666397094727, "target_country_logp": -8.240869522094727, "source_code_first_rank": 81592.0, "target_code_first_rank": 81069.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 6.0, "target_country_rank": 17513.0, "source_country_logp": -4.392697334289551, "target_country_logp": -28.267696380615234, "source_code_first_rank": 2546.5, "target_code_first_rank": 84690.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 23566.0, "target_country_rank": 25208.0, "source_country_logp": -30.257516860961914, "target_country_logp": -30.413766860961914, "source_code_first_rank": 84599.5, "target_code_first_rank": 56255.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 7.0, "target_country_rank": 536.0, "source_country_logp": -3.9445157051086426, "target_country_logp": -9.225765228271484, "source_code_first_rank": 1848.0, "target_code_first_rank": 5312.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 12406.5, "target_country_rank": 1606.0, "source_country_logp": -13.776079177856445, "target_country_logp": -10.178422927856445, "source_code_first_rank": 33565.5, "target_code_first_rank": 14817.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 6.0, "target_country_rank": 2819.0, "source_country_logp": -5.765507698059082, "target_country_logp": -21.3905086517334, "source_code_first_rank": 3495.5, "target_code_first_rank": 32384.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 16993.0, "target_country_rank": 59861.0, "source_country_logp": -19.60384750366211, "target_country_logp": -22.66634750366211, "source_code_first_rank": 6773.5, "target_code_first_rank": 11522.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 810.5, "source_country_logp": -2.1880171298980713, "target_country_logp": -10.063016891479492, "source_code_first_rank": 541.5, "target_code_first_rank": 515.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 29350.5, "target_country_rank": 15801.5, "source_country_logp": -15.645747184753418, "target_country_logp": -14.209223747253418, "source_code_first_rank": 4956.0, "target_code_first_rank": 19756.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 3359.5, "source_country_logp": -4.113725185394287, "target_country_logp": -27.426225662231445, "source_code_first_rank": 958.5, "target_code_first_rank": 28854.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 121431.0, "target_country_rank": 139504.5, "source_country_logp": -16.073699951171875, "target_country_logp": -17.573699951171875, "source_code_first_rank": 1011.0, "target_code_first_rank": 16935.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 14.0, "target_country_rank": 310.0, "source_country_logp": -8.01659870147705, "target_country_logp": -13.64159870147705, "source_code_first_rank": 1260.0, "target_code_first_rank": 38804.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 239.5, "target_country_rank": 144398.0, "source_country_logp": -21.657283782958984, "target_country_logp": -43.532283782958984, "source_code_first_rank": 1.0, "target_code_first_rank": 96.0}], "actual_source_country_rank": 239.5, "actual_target_country_rank": 144398.0, "array_path": "results/readout-arrays/eval-02-main.npz"}
{"case_id": "eval-02", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 28.0, "target_country_rank": 2379.0, "source_country_logp": -5.385872840881348, "target_country_logp": -11.260872840881348, "source_code_first_rank": 2945.0, "target_code_first_rank": 33002.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 12257.0, "target_country_rank": 635.5, "source_country_logp": -14.082839965820312, "target_country_logp": -9.086746215820312, "source_code_first_rank": 84538.0, "target_code_first_rank": 90212.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 6.5, "target_country_rank": 13006.0, "source_country_logp": -4.769664287567139, "target_country_logp": -26.082164764404297, "source_code_first_rank": 1880.0, "target_code_first_rank": 87091.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 34904.0, "target_country_rank": 36092.0, "source_country_logp": -33.213958740234375, "target_country_logp": -33.307708740234375, "source_code_first_rank": 66795.0, "target_code_first_rank": 60477.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 17.0, "target_country_rank": 519.5, "source_country_logp": -4.784332275390625, "target_country_logp": -9.503082275390625, "source_code_first_rank": 1640.0, "target_code_first_rank": 8128.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 3536.5, "target_country_rank": 1364.5, "source_country_logp": -10.896103858947754, "target_country_logp": -9.552353858947754, "source_code_first_rank": 29255.0, "target_code_first_rank": 32433.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 6.5, "target_country_rank": 1639.5, "source_country_logp": -5.187289237976074, "target_country_logp": -19.749788284301758, "source_code_first_rank": 2819.0, "target_code_first_rank": 31210.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 7324.0, "target_country_rank": 37276.0, "source_country_logp": -21.336986541748047, "target_country_logp": -24.805736541748047, "source_code_first_rank": 9331.0, "target_code_first_rank": 19557.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 4.5, "target_country_rank": 899.0, "source_country_logp": -2.8147525787353516, "target_country_logp": -10.283502578735352, "source_code_first_rank": 300.5, "target_code_first_rank": 518.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 26240.5, "target_country_rank": 22203.5, "source_country_logp": -16.717267990112305, "target_country_logp": -16.295392990112305, "source_code_first_rank": 21711.5, "target_code_first_rank": 24082.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 2730.0, "source_country_logp": -1.8036890029907227, "target_country_logp": -24.86618995666504, "source_code_first_rank": 873.0, "target_code_first_rank": 29651.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 118459.0, "target_country_rank": 133710.0, "source_country_logp": -16.054344177246094, "target_country_logp": -17.054344177246094, "source_code_first_rank": 8625.0, "target_code_first_rank": 19050.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 17.0, "target_country_rank": 273.5, "source_country_logp": -7.8305745124816895, "target_country_logp": -13.330574989318848, "source_code_first_rank": 708.0, "target_code_first_rank": 30725.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 48.0, "target_country_rank": 80134.5, "source_country_logp": -25.125028610229492, "target_country_logp": -46.17190170288086, "source_code_first_rank": 2.0, "target_code_first_rank": 476.5}], "actual_source_country_rank": 48.0, "actual_target_country_rank": 80134.5, "array_path": "results/readout-arrays/eval-02-paraphrase.npz"}
{"case_id": "eval-03", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 69.5, "source_country_logp": -0.31217047572135925, "target_country_logp": -9.562170028686523, "source_code_first_rank": 15511.0, "target_code_first_rank": 26164.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 179.5, "target_country_rank": 5716.5, "source_country_logp": -7.4296040534973145, "target_country_logp": -12.320229530334473, "source_code_first_rank": 89180.5, "target_code_first_rank": 50269.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 12.5, "target_country_rank": 520.0, "source_country_logp": -12.755165100097656, "target_country_logp": -28.380165100097656, "source_code_first_rank": 68431.5, "target_code_first_rank": 41816.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 17625.0, "target_country_rank": 9262.5, "source_country_logp": -29.609895706176758, "target_country_logp": -28.297395706176758, "source_code_first_rank": 54858.5, "target_code_first_rank": 32701.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 7.0, "source_country_logp": -0.4846648573875427, "target_country_logp": -5.1096649169921875, "source_code_first_rank": 6898.0, "target_code_first_rank": 27696.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 1294.5, "target_country_rank": 2614.0, "source_country_logp": -9.758652687072754, "target_country_logp": -10.821152687072754, "source_code_first_rank": 15010.0, "target_code_first_rank": 551.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 198.5, "source_country_logp": -6.253067970275879, "target_country_logp": -18.878068923950195, "source_code_first_rank": 28436.0, "target_code_first_rank": 40080.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 54305.0, "target_country_rank": 9242.5, "source_country_logp": -22.486791610717773, "target_country_logp": -18.580541610717773, "source_code_first_rank": 10506.0, "target_code_first_rank": 14931.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 18.0, "source_country_logp": -0.5780290365219116, "target_country_logp": -5.640529155731201, "source_code_first_rank": 1604.5, "target_code_first_rank": 18828.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 14943.5, "target_country_rank": 24380.0, "source_country_logp": -14.119660377502441, "target_country_logp": -15.201691627502441, "source_code_first_rank": 14627.5, "target_code_first_rank": 280.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 451.0, "source_country_logp": -4.0385942459106445, "target_country_logp": -21.91359519958496, "source_code_first_rank": 18537.0, "target_code_first_rank": 49462.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 139252.0, "target_country_rank": 133715.5, "source_country_logp": -17.763347625732422, "target_country_logp": -17.200847625732422, "source_code_first_rank": 11319.5, "target_code_first_rank": 18147.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 8.0, "target_country_rank": 386.0, "source_country_logp": -5.266289234161377, "target_country_logp": -13.328789710998535, "source_code_first_rank": 20642.5, "target_code_first_rank": 5381.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 1293.0, "target_country_rank": 104408.5, "source_country_logp": -27.217777252197266, "target_country_logp": -40.452152252197266, "source_code_first_rank": 1.0, "target_code_first_rank": 29.0}], "actual_source_country_rank": 1293.0, "actual_target_country_rank": 104408.5, "array_path": "results/readout-arrays/eval-03-main.npz"}
{"case_id": "eval-03", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 120.0, "source_country_logp": -0.37929031252861023, "target_country_logp": -9.754290580749512, "source_code_first_rank": 10751.5, "target_code_first_rank": 17493.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 462.5, "target_country_rank": 9277.0, "source_country_logp": -8.599967956542969, "target_country_logp": -13.342155456542969, "source_code_first_rank": 99885.0, "target_code_first_rank": 58849.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 11.0, "target_country_rank": 459.0, "source_country_logp": -12.753162384033203, "target_country_logp": -27.753162384033203, "source_code_first_rank": 63693.0, "target_code_first_rank": 37263.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 25996.0, "target_country_rank": 20675.0, "source_country_logp": -32.57698440551758, "target_country_logp": -32.01448440551758, "source_code_first_rank": 59034.0, "target_code_first_rank": 30325.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 25.0, "source_country_logp": -1.1405435800552368, "target_country_logp": -5.640543460845947, "source_code_first_rank": 5431.0, "target_code_first_rank": 25672.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 1411.5, "target_country_rank": 1363.0, "source_country_logp": -9.614522933959961, "target_country_logp": -9.552022933959961, "source_code_first_rank": 30261.5, "target_code_first_rank": 7186.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 218.5, "source_country_logp": -6.752109050750732, "target_country_logp": -18.12710952758789, "source_code_first_rank": 25849.0, "target_code_first_rank": 38187.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 35824.0, "target_country_rank": 8371.0, "source_country_logp": -24.699228286743164, "target_country_logp": -21.542978286743164, "source_code_first_rank": 16518.0, "target_code_first_rank": 24769.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 23.5, "source_country_logp": -1.0066827535629272, "target_country_logp": -5.506682872772217, "source_code_first_rank": 1347.5, "target_code_first_rank": 22011.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 18615.0, "target_country_rank": 10986.5, "source_country_logp": -16.140832901000977, "target_country_logp": -14.99630069732666, "source_code_first_rank": 21467.5, "target_code_first_rank": 1094.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 515.5, "source_country_logp": -5.2584710121154785, "target_country_logp": -22.38347053527832, "source_code_first_rank": 16993.0, "target_code_first_rank": 51588.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 132771.5, "target_country_rank": 131263.0, "source_country_logp": -17.32692527770996, "target_country_logp": -17.20192527770996, "source_code_first_rank": 16096.5, "target_code_first_rank": 35471.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 18.0, "target_country_rank": 483.5, "source_country_logp": -5.9913129806518555, "target_country_logp": -13.366312980651855, "source_code_first_rank": 21346.5, "target_code_first_rank": 7386.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 206.5, "target_country_rank": 14287.0, "source_country_logp": -27.625, "target_country_logp": -39.0625, "source_code_first_rank": 5.5, "target_code_first_rank": 146.5}], "actual_source_country_rank": 206.5, "actual_target_country_rank": 14287.0, "array_path": "results/readout-arrays/eval-03-paraphrase.npz"}
{"case_id": "eval-04", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 581.5, "source_country_logp": -0.39769914746284485, "target_country_logp": -11.585199356079102, "source_code_first_rank": 4861.0, "target_code_first_rank": 20109.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 236.0, "target_country_rank": 6968.5, "source_country_logp": -7.751914024353027, "target_country_logp": -12.759726524353027, "source_code_first_rank": 87333.5, "target_code_first_rank": 53428.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 8.0, "target_country_rank": 2714.5, "source_country_logp": -9.302313804626465, "target_country_logp": -27.48981475830078, "source_code_first_rank": 43117.5, "target_code_first_rank": 55858.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 18922.5, "target_country_rank": 10174.5, "source_country_logp": -30.27252197265625, "target_country_logp": -28.96002197265625, "source_code_first_rank": 56324.5, "target_code_first_rank": 34025.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 145.5, "source_country_logp": -0.8419966101646423, "target_country_logp": -8.591996192932129, "source_code_first_rank": 1213.0, "target_code_first_rank": 28893.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 1334.5, "target_country_rank": 2804.5, "source_country_logp": -9.869580268859863, "target_country_logp": -11.041455268859863, "source_code_first_rank": 15889.5, "target_code_first_rank": 715.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 770.5, "source_country_logp": -5.444947242736816, "target_country_logp": -21.257448196411133, "source_code_first_rank": 22725.5, "target_code_first_rank": 63336.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 60200.0, "target_country_rank": 10768.0, "source_country_logp": -22.488134384155273, "target_country_logp": -18.550634384155273, "source_code_first_rank": 10479.5, "target_code_first_rank": 14914.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 266.0, "source_country_logp": -1.2574422359466553, "target_country_logp": -9.101192474365234, "source_code_first_rank": 1866.5, "target_code_first_rank": 25512.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 17806.5, "target_country_rank": 31036.0, "source_country_logp": -14.618865966796875, "target_country_logp": -15.958709716796875, "source_code_first_rank": 17160.5, "target_code_first_rank": 352.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 4.0, "target_country_rank": 1953.5, "source_country_logp": -4.5572509765625, "target_country_logp": -26.8697509765625, "source_code_first_rank": 15865.5, "target_code_first_rank": 71114.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 140843.5, "target_country_rank": 136228.0, "source_country_logp": -17.858930587768555, "target_country_logp": -17.358930587768555, "source_code_first_rank": 13538.5, "target_code_first_rank": 16914.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 12.0, "target_country_rank": 2747.5, "source_country_logp": -6.918797492980957, "target_country_logp": -19.293798446655273, "source_code_first_rank": 8926.0, "target_code_first_rank": 8218.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 1486.5, "target_country_rank": 122787.0, "source_country_logp": -27.757043838500977, "target_country_logp": -41.30392074584961, "source_code_first_rank": 1.0, "target_code_first_rank": 32.0}], "actual_source_country_rank": 1486.5, "actual_target_country_rank": 122787.0, "array_path": "results/readout-arrays/eval-04-main.npz"}
{"case_id": "eval-04", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 1184.0, "source_country_logp": -0.502359926700592, "target_country_logp": -12.252360343933105, "source_code_first_rank": 4921.5, "target_code_first_rank": 15476.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 441.5, "target_country_rank": 8931.5, "source_country_logp": -8.496031761169434, "target_country_logp": -13.277281761169434, "source_code_first_rank": 91337.0, "target_code_first_rank": 58893.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 8.0, "target_country_rank": 2963.0, "source_country_logp": -9.776039123535156, "target_country_logp": -27.932289123535156, "source_code_first_rank": 42335.5, "target_code_first_rank": 57590.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 30413.5, "target_country_rank": 22390.5, "source_country_logp": -33.07859802246094, "target_country_logp": -32.29734802246094, "source_code_first_rank": 60379.0, "target_code_first_rank": 32312.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 289.5, "source_country_logp": -1.451220154762268, "target_country_logp": -9.638720512390137, "source_code_first_rank": 1670.0, "target_code_first_rank": 29899.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 1207.0, "target_country_rank": 1259.0, "source_country_logp": -9.358642578125, "target_country_logp": -9.421142578125, "source_code_first_rank": 32065.0, "target_code_first_rank": 6157.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 1037.0, "source_country_logp": -5.723191261291504, "target_country_logp": -22.223190307617188, "source_code_first_rank": 22524.0, "target_code_first_rank": 65931.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 33644.5, "target_country_rank": 7412.5, "source_country_logp": -23.641342163085938, "target_country_logp": -20.516342163085938, "source_code_first_rank": 18087.0, "target_code_first_rank": 26210.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.5, "target_country_rank": 234.0, "source_country_logp": -1.8376086950302124, "target_country_logp": -9.368858337402344, "source_code_first_rank": 1851.0, "target_code_first_rank": 26598.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 21675.5, "target_country_rank": 13701.0, "source_country_logp": -16.40302276611328, "target_country_logp": -15.348335266113281, "source_code_first_rank": 23982.0, "target_code_first_rank": 809.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 4.0, "target_country_rank": 2027.5, "source_country_logp": -3.9810078144073486, "target_country_logp": -25.793508529663086, "source_code_first_rank": 14542.0, "target_code_first_rank": 65376.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 135382.0, "target_country_rank": 135382.0, "source_country_logp": -17.169570922851562, "target_country_logp": -17.169570922851562, "source_code_first_rank": 17565.0, "target_code_first_rank": 32635.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 19.0, "target_country_rank": 2718.5, "source_country_logp": -7.382572650909424, "target_country_logp": -19.382572174072266, "source_code_first_rank": 10568.0, "target_code_first_rank": 10852.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 185.0, "target_country_rank": 26800.5, "source_country_logp": -26.250028610229492, "target_country_logp": -39.71877670288086, "source_code_first_rank": 2.0, "target_code_first_rank": 129.0}], "actual_source_country_rank": 185.0, "actual_target_country_rank": 26800.5, "array_path": "results/readout-arrays/eval-04-paraphrase.npz"}
{"case_id": "eval-05", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 441.5, "source_country_logp": -1.5131430625915527, "target_country_logp": -9.075643539428711, "source_code_first_rank": 87253.0, "target_code_first_rank": 40890.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 4143.5, "target_country_rank": 9116.5, "source_country_logp": -11.708401679992676, "target_country_logp": -13.216214179992676, "source_code_first_rank": 53891.0, "target_code_first_rank": 19133.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 2730.5, "source_country_logp": -12.250197410583496, "target_country_logp": -35.68769836425781, "source_code_first_rank": 101052.5, "target_code_first_rank": 43432.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 3329.0, "target_country_rank": 83789.5, "source_country_logp": -28.742969512939453, "target_country_logp": -37.05156326293945, "source_code_first_rank": 30424.5, "target_code_first_rank": 45322.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 1940.0, "source_country_logp": -1.4695007801055908, "target_country_logp": -11.360125541687012, "source_code_first_rank": 61609.0, "target_code_first_rank": 32387.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 2285.5, "target_country_rank": 695.5, "source_country_logp": -10.537945747375488, "target_country_logp": -8.803570747375488, "source_code_first_rank": 1654.0, "target_code_first_rank": 860.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.5, "target_country_rank": 1693.0, "source_country_logp": -8.500603675842285, "target_country_logp": -27.31310272216797, "source_code_first_rank": 100807.0, "target_code_first_rank": 44815.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 5709.5, "target_country_rank": 25003.5, "source_country_logp": -18.241008758544922, "target_country_logp": -20.975383758544922, "source_code_first_rank": 16046.0, "target_code_first_rank": 44477.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 1650.5, "source_country_logp": -2.3784806728363037, "target_country_logp": -10.878480911254883, "source_code_first_rank": 57785.0, "target_code_first_rank": 12413.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 15855.0, "target_country_rank": 1494.0, "source_country_logp": -14.120662689208984, "target_country_logp": -10.190486907958984, "source_code_first_rank": 455.5, "target_code_first_rank": 519.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 7.0, "target_country_rank": 3092.0, "source_country_logp": -6.768617153167725, "target_country_logp": -25.581117630004883, "source_code_first_rank": 93444.0, "target_code_first_rank": 44625.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 123202.5, "target_country_rank": 92514.0, "source_country_logp": -16.59193992614746, "target_country_logp": -15.091939926147461, "source_code_first_rank": 18955.5, "target_code_first_rank": 31032.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 25.0, "target_country_rank": 1701.5, "source_country_logp": -7.488949298858643, "target_country_logp": -15.176448822021484, "source_code_first_rank": 3136.5, "target_code_first_rank": 3630.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 275.0, "target_country_rank": 112233.5, "source_country_logp": -25.312664031982422, "target_country_logp": -44.87516403198242, "source_code_first_rank": 1.0, "target_code_first_rank": 29.0}], "actual_source_country_rank": 275.0, "actual_target_country_rank": 112233.5, "array_path": "results/readout-arrays/eval-05-main.npz"}
{"case_id": "eval-05", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 1826.5, "source_country_logp": -1.842360019683838, "target_country_logp": -10.71735954284668, "source_code_first_rank": 88666.0, "target_code_first_rank": 37008.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 8164.5, "target_country_rank": 12588.5, "source_country_logp": -12.994893074035645, "target_country_logp": -13.899189949035645, "source_code_first_rank": 61253.0, "target_code_first_rank": 23522.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 3175.5, "source_country_logp": -12.000173568725586, "target_country_logp": -36.25017547607422, "source_code_first_rank": 102341.5, "target_code_first_rank": 48671.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 15807.5, "target_country_rank": 116836.5, "source_country_logp": -32.06510925292969, "target_country_logp": -39.22135925292969, "source_code_first_rank": 32221.5, "target_code_first_rank": 45833.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 6184.0, "source_country_logp": -2.002880096435547, "target_country_logp": -13.182567596435547, "source_code_first_rank": 71753.5, "target_code_first_rank": 34738.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 1387.5, "target_country_rank": 2453.0, "source_country_logp": -9.594123840332031, "target_country_logp": -10.375373840332031, "source_code_first_rank": 7861.5, "target_code_first_rank": 7474.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 3130.0, "source_country_logp": -8.000874519348145, "target_country_logp": -27.50087547302246, "source_code_first_rank": 106314.5, "target_code_first_rank": 52949.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 6804.5, "target_country_rank": 27351.5, "source_country_logp": -20.833520889282227, "target_country_logp": -23.646020889282227, "source_code_first_rank": 26601.0, "target_code_first_rank": 70113.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 2288.5, "source_country_logp": -2.2695090770721436, "target_country_logp": -11.644509315490723, "source_code_first_rank": 60464.5, "target_code_first_rank": 13096.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 8912.5, "target_country_rank": 3527.0, "source_country_logp": -14.396852493286133, "target_country_logp": -12.754274368286133, "source_code_first_rank": 763.5, "target_code_first_rank": 723.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.5, "target_country_rank": 5507.0, "source_country_logp": -6.381282329559326, "target_country_logp": -28.068782806396484, "source_code_first_rank": 100891.0, "target_code_first_rank": 55114.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 128643.5, "target_country_rank": 105752.0, "source_country_logp": -16.909221649169922, "target_country_logp": -15.659221649169922, "source_code_first_rank": 32816.5, "target_code_first_rank": 42786.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 23.0, "target_country_rank": 3000.5, "source_country_logp": -7.283448696136475, "target_country_logp": -16.408449172973633, "source_code_first_rank": 3340.5, "target_code_first_rank": 3997.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 212.5, "target_country_rank": 17368.5, "source_country_logp": -27.125003814697266, "target_country_logp": -40.890628814697266, "source_code_first_rank": 4.0, "target_code_first_rank": 19.0}], "actual_source_country_rank": 212.5, "actual_target_country_rank": 17368.5, "array_path": "results/readout-arrays/eval-05-paraphrase.npz"}
{"case_id": "eval-06", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 522.0, "source_country_logp": -2.1546170711517334, "target_country_logp": -9.185867309570312, "source_code_first_rank": 68648.0, "target_code_first_rank": 33228.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 4796.5, "target_country_rank": 10545.0, "source_country_logp": -12.072135925292969, "target_country_logp": -13.630729675292969, "source_code_first_rank": 53031.5, "target_code_first_rank": 17709.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 3.5, "target_country_rank": 2943.5, "source_country_logp": -2.926470994949341, "target_country_logp": -21.676471710205078, "source_code_first_rank": 67612.0, "target_code_first_rank": 53143.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 4882.5, "target_country_rank": 93469.0, "source_country_logp": -29.107311248779297, "target_country_logp": -37.2010612487793, "source_code_first_rank": 31118.0, "target_code_first_rank": 46359.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 1771.5, "source_country_logp": -1.9962613582611084, "target_country_logp": -12.199386596679688, "source_code_first_rank": 11050.0, "target_code_first_rank": 8227.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 2731.0, "target_country_rank": 733.5, "source_country_logp": -10.903998374938965, "target_country_logp": -8.966498374938965, "source_code_first_rank": 1122.0, "target_code_first_rank": 629.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 1.5, "target_country_rank": 2011.0, "source_country_logp": -1.5818169116973877, "target_country_logp": -16.456817626953125, "source_code_first_rank": 66745.0, "target_code_first_rank": 49511.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 7456.0, "target_country_rank": 26346.5, "source_country_logp": -19.176633834838867, "target_country_logp": -21.582883834838867, "source_code_first_rank": 14505.0, "target_code_first_rank": 41606.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 1021.0, "source_country_logp": -1.8981549739837646, "target_country_logp": -11.366905212402344, "source_code_first_rank": 7286.5, "target_code_first_rank": 3259.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 19158.5, "target_country_rank": 1131.5, "source_country_logp": -14.524718284606934, "target_country_logp": -9.766905784606934, "source_code_first_rank": 528.5, "target_code_first_rank": 456.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 12.0, "target_country_rank": 8640.5, "source_country_logp": -9.230219841003418, "target_country_logp": -27.7927188873291, "source_code_first_rank": 85772.5, "target_code_first_rank": 69374.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 125682.5, "target_country_rank": 85840.0, "source_country_logp": -16.806413650512695, "target_country_logp": -14.868913650512695, "source_code_first_rank": 17478.5, "target_code_first_rank": 27806.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 23.0, "target_country_rank": 3935.0, "source_country_logp": -8.465636253356934, "target_country_logp": -18.028135299682617, "source_code_first_rank": 3648.5, "target_code_first_rank": 979.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 218.5, "target_country_rank": 127846.0, "source_country_logp": -29.187517166137695, "target_country_logp": -50.12501907348633, "source_code_first_rank": 1.0, "target_code_first_rank": 131.0}], "actual_source_country_rank": 218.5, "actual_target_country_rank": 127846.0, "array_path": "results/readout-arrays/eval-06-main.npz"}
{"case_id": "eval-06", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 3.5, "target_country_rank": 1769.5, "source_country_logp": -3.303464412689209, "target_country_logp": -11.28783893585205, "source_code_first_rank": 67968.0, "target_code_first_rank": 33065.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 8092.0, "target_country_rank": 13222.0, "source_country_logp": -13.097559928894043, "target_country_logp": -14.138575553894043, "source_code_first_rank": 64553.5, "target_code_first_rank": 24445.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 3296.0, "source_country_logp": -2.9276645183563232, "target_country_logp": -22.365163803100586, "source_code_first_rank": 66754.5, "target_code_first_rank": 57678.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 12866.0, "target_country_rank": 114656.5, "source_country_logp": -32.05906295776367, "target_country_logp": -39.68406295776367, "source_code_first_rank": 30736.5, "target_code_first_rank": 45308.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 2842.0, "source_country_logp": -2.8825459480285645, "target_country_logp": -12.952858924865723, "source_code_first_rank": 11771.0, "target_code_first_rank": 10359.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 1424.0, "target_country_rank": 2589.0, "source_country_logp": -9.662240982055664, "target_country_logp": -10.505990982055664, "source_code_first_rank": 8156.0, "target_code_first_rank": 7848.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 2539.0, "source_country_logp": -1.3125312328338623, "target_country_logp": -17.375030517578125, "source_code_first_rank": 60381.0, "target_code_first_rank": 48544.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 6063.0, "target_country_rank": 25515.5, "source_country_logp": -21.141883850097656, "target_country_logp": -24.016883850097656, "source_code_first_rank": 24825.0, "target_code_first_rank": 67484.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 1149.5, "source_country_logp": -2.5318076610565186, "target_country_logp": -11.969307899475098, "source_code_first_rank": 7859.0, "target_code_first_rank": 3679.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 13683.5, "target_country_rank": 3359.0, "source_country_logp": -15.072574615478516, "target_country_logp": -12.521793365478516, "source_code_first_rank": 1076.0, "target_code_first_rank": 808.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 6.0, "target_country_rank": 9317.0, "source_country_logp": -3.389662027359009, "target_country_logp": -23.82716178894043, "source_code_first_rank": 76342.0, "target_code_first_rank": 71815.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 133880.5, "target_country_rank": 107464.0, "source_country_logp": -17.023029327392578, "target_country_logp": -15.460529327392578, "source_code_first_rank": 34646.0, "target_code_first_rank": 44337.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 34.5, "target_country_rank": 9059.5, "source_country_logp": -8.54429817199707, "target_country_logp": -19.16929817199707, "source_code_first_rank": 3394.5, "target_code_first_rank": 1144.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 324.5, "target_country_rank": 41874.0, "source_country_logp": -32.5, "target_country_logp": -47.40234375, "source_code_first_rank": 5.0, "target_code_first_rank": 102.0}], "actual_source_country_rank": 324.5, "actual_target_country_rank": 41874.0, "array_path": "results/readout-arrays/eval-06-paraphrase.npz"}
{"case_id": "eval-07", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 7.0, "target_country_rank": 374.0, "source_country_logp": -3.8504772186279297, "target_country_logp": -8.16297721862793, "source_code_first_rank": 56744.0, "target_code_first_rank": 119831.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 12683.0, "target_country_rank": 25643.0, "source_country_logp": -14.050474166870117, "target_country_logp": -15.718442916870117, "source_code_first_rank": 13286.5, "target_code_first_rank": 117922.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 4.0, "target_country_rank": 57.0, "source_country_logp": -4.164070129394531, "target_country_logp": -10.039070129394531, "source_code_first_rank": 118824.0, "target_code_first_rank": 127689.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 106583.0, "target_country_rank": 50606.5, "source_country_logp": -35.71897506713867, "target_country_logp": -32.53147506713867, "source_code_first_rank": 49105.5, "target_code_first_rank": 109298.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 61.5, "target_country_rank": 15.0, "source_country_logp": -8.235733985900879, "target_country_logp": -6.954483985900879, "source_code_first_rank": 53432.5, "target_code_first_rank": 123768.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 1054.0, "target_country_rank": 5553.0, "source_country_logp": -9.540994644165039, "target_country_logp": -12.181619644165039, "source_code_first_rank": 437.0, "target_code_first_rank": 115570.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 7.0, "target_country_rank": 58.0, "source_country_logp": -4.270725727081299, "target_country_logp": -8.020726203918457, "source_code_first_rank": 104923.5, "target_code_first_rank": 107251.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 32432.0, "target_country_rank": 23559.0, "source_country_logp": -19.723421096801758, "target_country_logp": -19.035921096801758, "source_code_first_rank": 44304.0, "target_code_first_rank": 21428.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 24.0, "target_country_rank": 15.0, "source_country_logp": -8.151082038879395, "target_country_logp": -7.5260820388793945, "source_code_first_rank": 17071.0, "target_code_first_rank": 100257.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 2698.0, "target_country_rank": 14320.0, "source_country_logp": -11.12103271484375, "target_country_logp": -14.08953857421875, "source_code_first_rank": 396.0, "target_code_first_rank": 67061.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 14.0, "target_country_rank": 52.0, "source_country_logp": -8.432653427124023, "target_country_logp": -12.182653427124023, "source_code_first_rank": 98136.0, "target_code_first_rank": 55501.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 95230.0, "target_country_rank": 93690.0, "source_country_logp": -15.084134101867676, "target_country_logp": -15.021634101867676, "source_code_first_rank": 26294.5, "target_code_first_rank": 2387.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 16.5, "target_country_rank": 419.5, "source_country_logp": -6.800332546234131, "target_country_logp": -13.987833023071289, "source_code_first_rank": 1471.5, "target_code_first_rank": 72690.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 342.0, "target_country_rank": 9528.0, "source_country_logp": -24.56284523010254, "target_country_logp": -33.40659713745117, "source_code_first_rank": 1.0, "target_code_first_rank": 27.5}], "actual_source_country_rank": 342.0, "actual_target_country_rank": 9528.0, "array_path": "results/readout-arrays/eval-07-main.npz"}
{"case_id": "eval-07", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 76.0, "target_country_rank": 585.0, "source_country_logp": -6.377695560455322, "target_country_logp": -8.815195083618164, "source_code_first_rank": 70171.5, "target_code_first_rank": 102184.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 14513.0, "target_country_rank": 27869.5, "source_country_logp": -14.345271110534668, "target_country_logp": -15.944148063659668, "source_code_first_rank": 22852.0, "target_code_first_rank": 113172.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 9.0, "target_country_rank": 72.0, "source_country_logp": -5.166375637054443, "target_country_logp": -10.291376113891602, "source_code_first_rank": 127530.0, "target_code_first_rank": 132788.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 122061.5, "target_country_rank": 70680.0, "source_country_logp": -39.044281005859375, "target_country_logp": -35.794281005859375, "source_code_first_rank": 47332.5, "target_code_first_rank": 104162.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 112.5, "target_country_rank": 29.0, "source_country_logp": -9.608076095581055, "target_country_logp": -7.920576095581055, "source_code_first_rank": 50494.0, "target_code_first_rank": 126776.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 2851.0, "target_country_rank": 6197.5, "source_country_logp": -10.598158836364746, "target_country_logp": -11.840346336364746, "source_code_first_rank": 7306.0, "target_code_first_rank": 119183.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 15.5, "target_country_rank": 67.5, "source_country_logp": -5.2461395263671875, "target_country_logp": -8.371139526367188, "source_code_first_rank": 115503.5, "target_code_first_rank": 110351.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 27875.0, "target_country_rank": 17414.5, "source_country_logp": -23.19463348388672, "target_country_logp": -22.16338348388672, "source_code_first_rank": 70893.0, "target_code_first_rank": 48740.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 24.0, "target_country_rank": 12.0, "source_country_logp": -8.585184097290039, "target_country_logp": -7.741434574127197, "source_code_first_rank": 19264.0, "target_code_first_rank": 68188.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 3893.0, "target_country_rank": 9591.0, "source_country_logp": -13.023428916931152, "target_country_logp": -14.602042198181152, "source_code_first_rank": 1218.0, "target_code_first_rank": 39414.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 28.5, "source_country_logp": -3.4540138244628906, "target_country_logp": -7.579013824462891, "source_code_first_rank": 103668.0, "target_code_first_rank": 42617.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 109516.5, "target_country_rank": 100481.0, "source_country_logp": -15.933348655700684, "target_country_logp": -15.558348655700684, "source_code_first_rank": 50211.0, "target_code_first_rank": 5802.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 21.0, "target_country_rank": 521.0, "source_country_logp": -6.5377044677734375, "target_country_logp": -14.225204467773438, "source_code_first_rank": 1674.5, "target_code_first_rank": 84327.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 80.0, "target_country_rank": 2590.0, "source_country_logp": -23.37502098083496, "target_country_logp": -32.81251907348633, "source_code_first_rank": 4.0, "target_code_first_rank": 49.5}], "actual_source_country_rank": 80.0, "actual_target_country_rank": 2590.0, "array_path": "results/readout-arrays/eval-07-paraphrase.npz"}
{"case_id": "eval-08", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 153.0, "source_country_logp": -1.419278621673584, "target_country_logp": -7.388028621673584, "source_code_first_rank": 47345.0, "target_code_first_rank": 68216.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 12878.5, "target_country_rank": 24337.0, "source_country_logp": -14.069215774536133, "target_country_logp": -15.561403274536133, "source_code_first_rank": 13459.0, "target_code_first_rank": 117104.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 6.0, "target_country_rank": 22.0, "source_country_logp": -3.030710220336914, "target_country_logp": -8.905710220336914, "source_code_first_rank": 25755.0, "target_code_first_rank": 5211.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 103287.5, "target_country_rank": 47392.5, "source_country_logp": -35.614315032958984, "target_country_logp": -32.426815032958984, "source_code_first_rank": 50479.0, "target_code_first_rank": 110572.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 12.0, "target_country_rank": 44.5, "source_country_logp": -4.966514587402344, "target_country_logp": -6.716514587402344, "source_code_first_rank": 17020.5, "target_code_first_rank": 44298.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 882.5, "target_country_rank": 4636.0, "source_country_logp": -9.30946159362793, "target_country_logp": -11.85633659362793, "source_code_first_rank": 622.0, "target_code_first_rank": 113810.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 16.0, "source_country_logp": -1.8443223237991333, "target_country_logp": -4.969322204589844, "source_code_first_rank": 25494.5, "target_code_first_rank": 7568.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 30032.0, "target_country_rank": 23364.0, "source_country_logp": -20.026643753051758, "target_country_logp": -19.495393753051758, "source_code_first_rank": 50985.0, "target_code_first_rank": 27568.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 13.0, "source_country_logp": -2.4661762714385986, "target_country_logp": -5.0286760330200195, "source_code_first_rank": 5667.5, "target_code_first_rank": 35419.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 1960.0, "target_country_rank": 13223.5, "source_country_logp": -10.863922119140625, "target_country_logp": -14.165435791015625, "source_code_first_rank": 441.5, "target_code_first_rank": 69376.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 25.0, "source_country_logp": -4.135900020599365, "target_country_logp": -9.885899543762207, "source_code_first_rank": 31183.0, "target_code_first_rank": 3068.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 95565.0, "target_country_rank": 98152.5, "source_country_logp": -14.94176197052002, "target_country_logp": -15.00426197052002, "source_code_first_rank": 34739.5, "target_code_first_rank": 2913.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 6.0, "target_country_rank": 91.5, "source_country_logp": -4.587220668792725, "target_country_logp": -12.524720191955566, "source_code_first_rank": 1685.5, "target_code_first_rank": 33751.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 243.0, "target_country_rank": 18458.5, "source_country_logp": -25.000091552734375, "target_country_logp": -36.437591552734375, "source_code_first_rank": 1.0, "target_code_first_rank": 86.5}], "actual_source_country_rank": 243.0, "actual_target_country_rank": 18458.5, "array_path": "results/readout-arrays/eval-08-main.npz"}
{"case_id": "eval-08", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 175.0, "source_country_logp": -2.985994338989258, "target_country_logp": -7.267244338989258, "source_code_first_rank": 51198.5, "target_code_first_rank": 54553.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 14796.5, "target_country_rank": 26872.0, "source_country_logp": -14.36595630645752, "target_country_logp": -15.82201099395752, "source_code_first_rank": 24757.0, "target_code_first_rank": 114150.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 6.0, "target_country_rank": 32.0, "source_country_logp": -3.890406608581543, "target_country_logp": -8.515406608581543, "source_code_first_rank": 31474.5, "target_code_first_rank": 5104.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 120816.0, "target_country_rank": 67715.5, "source_country_logp": -39.01369094848633, "target_country_logp": -35.70119094848633, "source_code_first_rank": 45225.0, "target_code_first_rank": 104338.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 17.0, "target_country_rank": 90.5, "source_country_logp": -6.503506660461426, "target_country_logp": -8.284756660461426, "source_code_first_rank": 17626.0, "target_code_first_rank": 51729.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 2696.5, "target_country_rank": 6124.5, "source_country_logp": -10.514753341674805, "target_country_logp": -11.811628341674805, "source_code_first_rank": 8808.0, "target_code_first_rank": 117084.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 4.0, "target_country_rank": 20.5, "source_country_logp": -2.610553503036499, "target_country_logp": -5.235553741455078, "source_code_first_rank": 31261.0, "target_code_first_rank": 8207.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 27878.5, "target_country_rank": 18414.5, "source_country_logp": -23.787784576416016, "target_country_logp": -22.850284576416016, "source_code_first_rank": 70833.5, "target_code_first_rank": 54140.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 13.0, "source_country_logp": -2.8570308685302734, "target_country_logp": -5.357030868530273, "source_code_first_rank": 6112.5, "target_code_first_rank": 31379.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 3493.0, "target_country_rank": 9911.5, "source_country_logp": -12.837571144104004, "target_country_logp": -14.675950050354004, "source_code_first_rank": 1197.0, "target_code_first_rank": 42251.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 13.0, "source_country_logp": -2.2659757137298584, "target_country_logp": -8.015975952148438, "source_code_first_rank": 43838.0, "target_code_first_rank": 2893.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 107403.5, "target_country_rank": 99666.0, "source_country_logp": -15.64238166809082, "target_country_logp": -15.32988166809082, "source_code_first_rank": 51097.5, "target_code_first_rank": 5098.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 9.0, "target_country_rank": 118.5, "source_country_logp": -5.481373310089111, "target_country_logp": -13.16887378692627, "source_code_first_rank": 3631.5, "target_code_first_rank": 57443.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 91.5, "target_country_rank": 4637.5, "source_country_logp": -20.413984298706055, "target_country_logp": -31.445234298706055, "source_code_first_rank": 2.0, "target_code_first_rank": 81.0}], "actual_source_country_rank": 91.5, "actual_target_country_rank": 4637.5, "array_path": "results/readout-arrays/eval-08-paraphrase.npz"}
{"case_id": "eval-09", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 19.5, "target_country_rank": 4071.5, "source_country_logp": -4.856130599975586, "target_country_logp": -12.176443099975586, "source_code_first_rank": 115167.5, "target_code_first_rank": 12331.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 24262.0, "target_country_rank": 6775.5, "source_country_logp": -15.542325019836426, "target_country_logp": -12.729825019836426, "source_code_first_rank": 118383.5, "target_code_first_rank": 79802.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 19949.0, "source_country_logp": -1.2441174983978271, "target_country_logp": -29.822242736816406, "source_code_first_rank": 38444.0, "target_code_first_rank": 94111.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 44494.0, "target_country_rank": 26837.0, "source_country_logp": -32.887779235839844, "target_country_logp": -31.48152732849121, "source_code_first_rank": 110162.0, "target_code_first_rank": 28237.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 2.5, "target_country_rank": 2761.0, "source_country_logp": -2.714583396911621, "target_country_logp": -11.948958396911621, "source_code_first_rank": 81737.5, "target_code_first_rank": 7413.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 5064.0, "target_country_rank": 5111.0, "source_country_logp": -11.937919616699219, "target_country_logp": -11.953544616699219, "source_code_first_rank": 115082.5, "target_code_first_rank": 1506.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 7625.5, "source_country_logp": -1.645362377166748, "target_country_logp": -23.395362854003906, "source_code_first_rank": 31362.0, "target_code_first_rank": 102191.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 23086.0, "target_country_rank": 14154.0, "source_country_logp": -21.0407772064209, "target_country_logp": -20.0720272064209, "source_code_first_rank": 24197.5, "target_code_first_rank": 27239.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 3.5, "target_country_rank": 856.5, "source_country_logp": -2.398062229156494, "target_country_logp": -10.273061752319336, "source_code_first_rank": 87235.5, "target_code_first_rank": 1598.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 10250.0, "target_country_rank": 29008.0, "source_country_logp": -13.478124618530273, "target_country_logp": -15.720312118530273, "source_code_first_rank": 60246.5, "target_code_first_rank": 8303.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 9624.5, "source_country_logp": -1.3855769634246826, "target_country_logp": -27.573076248168945, "source_code_first_rank": 7949.0, "target_code_first_rank": 108472.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 85726.0, "target_country_rank": 106261.0, "source_country_logp": -14.576993942260742, "target_country_logp": -15.326993942260742, "source_code_first_rank": 2525.0, "target_code_first_rank": 40919.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 6.0, "target_country_rank": 2722.0, "source_country_logp": -6.1609673500061035, "target_country_logp": -20.348466873168945, "source_code_first_rank": 28754.0, "target_code_first_rank": 20130.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 155.5, "target_country_rank": 147347.0, "source_country_logp": -17.34800148010254, "target_country_logp": -40.41050338745117, "source_code_first_rank": 1.0, "target_code_first_rank": 390.0}], "actual_source_country_rank": 155.5, "actual_target_country_rank": 147347.0, "array_path": "results/readout-arrays/eval-09-main.npz"}
{"case_id": "eval-09", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 9.0, "target_country_rank": 7165.5, "source_country_logp": -4.189716815948486, "target_country_logp": -13.244403839111328, "source_code_first_rank": 102690.0, "target_code_first_rank": 10360.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 26668.5, "target_country_rank": 6626.0, "source_country_logp": -15.78148365020752, "target_country_logp": -12.66429615020752, "source_code_first_rank": 114725.5, "target_code_first_rank": 68676.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 33814.5, "source_country_logp": -2.0776610374450684, "target_country_logp": -31.392114639282227, "source_code_first_rank": 33494.5, "target_code_first_rank": 96565.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 58218.5, "target_country_rank": 40685.0, "source_country_logp": -35.46105194091797, "target_country_logp": -34.30480194091797, "source_code_first_rank": 103367.0, "target_code_first_rank": 22955.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 3.0, "target_country_rank": 5459.0, "source_country_logp": -2.7218360900878906, "target_country_logp": -13.11246109008789, "source_code_first_rank": 77712.0, "target_code_first_rank": 6643.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 5982.5, "target_country_rank": 4512.0, "source_country_logp": -11.745022773742676, "target_country_logp": -11.268460273742676, "source_code_first_rank": 117012.0, "target_code_first_rank": 14576.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 3.5, "target_country_rank": 9096.5, "source_country_logp": -2.1222965717315674, "target_country_logp": -24.934797286987305, "source_code_first_rank": 29925.0, "target_code_first_rank": 105310.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 15000.5, "target_country_rank": 12066.5, "source_country_logp": -22.48717498779297, "target_country_logp": -22.04967498779297, "source_code_first_rank": 49823.0, "target_code_first_rank": 40947.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 1116.0, "source_country_logp": -2.043098211288452, "target_country_logp": -10.293098449707031, "source_code_first_rank": 75492.5, "target_code_first_rank": 1232.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 9963.5, "target_country_rank": 17590.5, "source_country_logp": -14.548879623413086, "target_country_logp": -15.759817123413086, "source_code_first_rank": 48761.5, "target_code_first_rank": 2976.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 16079.0, "source_country_logp": -1.8564698696136475, "target_country_logp": -28.528345108032227, "source_code_first_rank": 8013.0, "target_code_first_rank": 113305.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 97627.0, "target_country_rank": 109472.0, "source_country_logp": -15.268967628479004, "target_country_logp": -15.706467628479004, "source_code_first_rank": 6135.0, "target_code_first_rank": 43697.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 4.5, "target_country_rank": 3308.5, "source_country_logp": -6.020038604736328, "target_country_logp": -20.395038604736328, "source_code_first_rank": 21328.0, "target_code_first_rank": 19016.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 42.0, "target_country_rank": 109081.5, "source_country_logp": -27.625, "target_country_logp": -51.140625, "source_code_first_rank": 9.0, "target_code_first_rank": 445.5}], "actual_source_country_rank": 42.0, "actual_target_country_rank": 109081.5, "array_path": "results/readout-arrays/eval-09-paraphrase.npz"}
{"case_id": "eval-10", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 570.5, "source_country_logp": -0.5541080832481384, "target_country_logp": -10.179107666015625, "source_code_first_rank": 85480.0, "target_code_first_rank": 14578.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 24874.5, "target_country_rank": 7448.5, "source_country_logp": -15.652143478393555, "target_country_logp": -12.964643478393555, "source_code_first_rank": 117596.0, "target_code_first_rank": 85130.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 2119.5, "source_country_logp": -0.7978227734565735, "target_country_logp": -28.297822952270508, "source_code_first_rank": 16602.5, "target_code_first_rank": 66064.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 48651.0, "target_country_rank": 28634.0, "source_country_logp": -32.61250686645508, "target_country_logp": -31.112506866455078, "source_code_first_rank": 109875.5, "target_code_first_rank": 28283.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 250.0, "source_country_logp": -1.114964246749878, "target_country_logp": -8.489964485168457, "source_code_first_rank": 114266.5, "target_code_first_rank": 9754.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 5466.5, "target_country_rank": 5021.0, "source_country_logp": -12.145798683166504, "target_country_logp": -11.993454933166504, "source_code_first_rank": 111562.0, "target_code_first_rank": 1724.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 1415.5, "source_country_logp": -1.1159617900848389, "target_country_logp": -22.4284610748291, "source_code_first_rank": 26687.5, "target_code_first_rank": 53524.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 27352.5, "target_country_rank": 14216.5, "source_country_logp": -19.474225997924805, "target_country_logp": -18.192975997924805, "source_code_first_rank": 22783.5, "target_code_first_rank": 27757.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 106.5, "source_country_logp": -1.4979478120803833, "target_country_logp": -8.716697692871094, "source_code_first_rank": 63567.0, "target_code_first_rank": 1631.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 11226.0, "target_country_rank": 26128.5, "source_country_logp": -13.500691413879395, "target_country_logp": -15.305378913879395, "source_code_first_rank": 70502.0, "target_code_first_rank": 7254.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 1710.0, "source_country_logp": -2.893054485321045, "target_country_logp": -25.143054962158203, "source_code_first_rank": 6275.5, "target_code_first_rank": 36323.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 83181.0, "target_country_rank": 103935.5, "source_country_logp": -14.52826976776123, "target_country_logp": -15.27826976776123, "source_code_first_rank": 3716.5, "target_code_first_rank": 34576.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 8.0, "target_country_rank": 4831.0, "source_country_logp": -5.246536731719971, "target_country_logp": -18.715286254882812, "source_code_first_rank": 5193.5, "target_code_first_rank": 12656.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 134.5, "target_country_rank": 138123.5, "source_country_logp": -16.276704788208008, "target_country_logp": -37.901702880859375, "source_code_first_rank": 2.0, "target_code_first_rank": 427.5}], "actual_source_country_rank": 134.5, "actual_target_country_rank": 138123.5, "array_path": "results/readout-arrays/eval-10-main.npz"}
{"case_id": "eval-10", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 1227.0, "source_country_logp": -0.6541546583175659, "target_country_logp": -10.997904777526855, "source_code_first_rank": 79198.0, "target_code_first_rank": 18978.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 26721.5, "target_country_rank": 6837.0, "source_country_logp": -15.883295059204102, "target_country_logp": -12.805170059204102, "source_code_first_rank": 113190.5, "target_code_first_rank": 68232.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 3.5, "target_country_rank": 4482.5, "source_country_logp": -1.76593816280365, "target_country_logp": -29.04718780517578, "source_code_first_rank": 13931.5, "target_code_first_rank": 75872.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 64043.0, "target_country_rank": 43378.0, "source_country_logp": -35.65581512451172, "target_country_logp": -34.34331512451172, "source_code_first_rank": 104433.5, "target_code_first_rank": 24625.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 141.0, "source_country_logp": -1.075538158416748, "target_country_logp": -7.919288158416748, "source_code_first_rank": 126486.0, "target_code_first_rank": 9287.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 5758.5, "target_country_rank": 5019.5, "source_country_logp": -11.690752983093262, "target_country_logp": -11.464190483093262, "source_code_first_rank": 117353.0, "target_code_first_rank": 15866.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 1423.0, "source_country_logp": -1.9116730690002441, "target_country_logp": -22.786672592163086, "source_code_first_rank": 31838.5, "target_code_first_rank": 62209.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 16517.5, "target_country_rank": 13945.5, "source_country_logp": -21.972450256347656, "target_country_logp": -21.628700256347656, "source_code_first_rank": 50556.5, "target_code_first_rank": 41939.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 67.5, "source_country_logp": -1.6215035915374756, "target_country_logp": -8.090253829956055, "source_code_first_rank": 59206.5, "target_code_first_rank": 879.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 8185.0, "target_country_rank": 16620.0, "source_country_logp": -14.30862808227539, "target_country_logp": -15.78323745727539, "source_code_first_rank": 44174.0, "target_code_first_rank": 2752.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 1255.5, "source_country_logp": -3.647866725921631, "target_country_logp": -24.77286720275879, "source_code_first_rank": 5576.0, "target_code_first_rank": 36462.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 92499.5, "target_country_rank": 107457.5, "source_country_logp": -15.173030853271484, "target_country_logp": -15.673030853271484, "source_code_first_rank": 5842.5, "target_code_first_rank": 42018.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 7.5, "target_country_rank": 4197.5, "source_country_logp": -4.7713541984558105, "target_country_logp": -18.55260467529297, "source_code_first_rank": 4916.5, "target_code_first_rank": 18216.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 24.5, "target_country_rank": 63670.5, "source_country_logp": -25.625, "target_country_logp": -47.12109375, "source_code_first_rank": 12.0, "target_code_first_rank": 343.0}], "actual_source_country_rank": 24.5, "actual_target_country_rank": 63670.5, "array_path": "results/readout-arrays/eval-10-paraphrase.npz"}
{"case_id": "eval-11", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 45.0, "source_country_logp": -2.4062724113464355, "target_country_logp": -5.9687724113464355, "source_code_first_rank": 45274.5, "target_code_first_rank": 48820.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 5297.0, "target_country_rank": 28056.0, "source_country_logp": -12.303651809692383, "target_country_logp": -15.967714309692383, "source_code_first_rank": 79115.0, "target_code_first_rank": 126070.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 308.5, "source_country_logp": -5.8910112380981445, "target_country_logp": -25.14101219177246, "source_code_first_rank": 61659.5, "target_code_first_rank": 79880.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 23630.5, "target_country_rank": 5548.5, "source_country_logp": -30.73795509338379, "target_country_logp": -27.75358009338379, "source_code_first_rank": 26950.0, "target_code_first_rank": 28011.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 3.0, "target_country_rank": 35.0, "source_country_logp": -3.4080686569213867, "target_country_logp": -6.095568656921387, "source_code_first_rank": 16174.0, "target_code_first_rank": 69950.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 4506.5, "target_country_rank": 3399.5, "source_country_logp": -11.7152681350708, "target_country_logp": -11.2543306350708, "source_code_first_rank": 1954.0, "target_code_first_rank": 146689.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.5, "target_country_rank": 229.5, "source_country_logp": -1.7335939407348633, "target_country_logp": -18.858592987060547, "source_code_first_rank": 65220.0, "target_code_first_rank": 59862.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 12012.0, "target_country_rank": 17854.0, "source_country_logp": -18.05698013305664, "target_country_logp": -18.77573013305664, "source_code_first_rank": 31666.5, "target_code_first_rank": 67159.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 142.0, "source_country_logp": -2.4524877071380615, "target_country_logp": -8.17123794555664, "source_code_first_rank": 5146.5, "target_code_first_rank": 107777.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 23145.5, "target_country_rank": 6222.0, "source_country_logp": -15.1839599609375, "target_country_logp": -12.5745849609375, "source_code_first_rank": 7770.5, "target_code_first_rank": 135816.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 323.0, "source_country_logp": -1.8557021617889404, "target_country_logp": -21.355701446533203, "source_code_first_rank": 77985.5, "target_code_first_rank": 52064.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 98944.0, "target_country_rank": 53718.0, "source_country_logp": -15.074972152709961, "target_country_logp": -13.481222152709961, "source_code_first_rank": 36376.0, "target_code_first_rank": 38529.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 21.0, "target_country_rank": 1175.5, "source_country_logp": -7.775784969329834, "target_country_logp": -15.525784492492676, "source_code_first_rank": 1663.5, "target_code_first_rank": 117922.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 230.5, "target_country_rank": 113349.5, "source_country_logp": -19.944229125976562, "target_country_logp": -39.41297912597656, "source_code_first_rank": 1.0, "target_code_first_rank": 297.5}], "actual_source_country_rank": 230.5, "actual_target_country_rank": 113349.5, "array_path": "results/readout-arrays/eval-11-main.npz"}
{"case_id": "eval-11", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 3.5, "target_country_rank": 78.0, "source_country_logp": -2.849733591079712, "target_country_logp": -6.724733352661133, "source_code_first_rank": 50925.5, "target_code_first_rank": 50692.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 6132.0, "target_country_rank": 37348.5, "source_country_logp": -12.561302185058594, "target_country_logp": -16.795677185058594, "source_code_first_rank": 71345.5, "target_code_first_rank": 130378.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 324.0, "source_country_logp": -5.394008159637451, "target_country_logp": -25.14400863647461, "source_code_first_rank": 68809.5, "target_code_first_rank": 75506.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 48483.5, "target_country_rank": 13815.5, "source_country_logp": -34.14922332763672, "target_country_logp": -30.96172332763672, "source_code_first_rank": 26523.5, "target_code_first_rank": 31416.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 6.0, "target_country_rank": 47.5, "source_country_logp": -3.789405345916748, "target_country_logp": -6.226905345916748, "source_code_first_rank": 18658.0, "target_code_first_rank": 79309.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 4153.0, "target_country_rank": 1434.5, "source_country_logp": -11.179892539978027, "target_country_logp": -9.648642539978027, "source_code_first_rank": 16615.0, "target_code_first_rank": 143921.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 297.5, "source_country_logp": -2.2242133617401123, "target_country_logp": -19.849212646484375, "source_code_first_rank": 76742.5, "target_code_first_rank": 62515.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 15593.0, "target_country_rank": 8964.5, "source_country_logp": -21.72799301147461, "target_country_logp": -20.66549301147461, "source_code_first_rank": 44006.5, "target_code_first_rank": 61021.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 119.5, "source_country_logp": -3.0116324424743652, "target_country_logp": -8.230382919311523, "source_code_first_rank": 4340.0, "target_code_first_rank": 108007.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 15036.0, "target_country_rank": 4088.0, "source_country_logp": -15.665885925292969, "target_country_logp": -13.220573425292969, "source_code_first_rank": 3015.0, "target_code_first_rank": 147129.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.5, "target_country_rank": 361.0, "source_country_logp": -2.4503941535949707, "target_country_logp": -22.700393676757812, "source_code_first_rank": 83361.0, "target_code_first_rank": 50928.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 99947.0, "target_country_rank": 45843.0, "source_country_logp": -15.483030319213867, "target_country_logp": -13.733030319213867, "source_code_first_rank": 43305.0, "target_code_first_rank": 57806.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 18.0, "target_country_rank": 1229.5, "source_country_logp": -6.616858005523682, "target_country_logp": -15.86685848236084, "source_code_first_rank": 1384.5, "target_code_first_rank": 114761.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 57.0, "target_country_rank": 9647.0, "source_country_logp": -23.8750057220459, "target_country_logp": -38.437503814697266, "source_code_first_rank": 3.0, "target_code_first_rank": 933.0}], "actual_source_country_rank": 57.0, "actual_target_country_rank": 9647.0, "array_path": "results/readout-arrays/eval-11-paraphrase.npz"}
{"case_id": "eval-12", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 33.0, "source_country_logp": -3.11181378364563, "target_country_logp": -6.736814022064209, "source_code_first_rank": 33505.5, "target_code_first_rank": 55877.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 6454.5, "target_country_rank": 30837.0, "source_country_logp": -12.641526222229004, "target_country_logp": -16.172775268554688, "source_code_first_rank": 83343.0, "target_code_first_rank": 125381.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 581.0, "source_country_logp": -3.1059489250183105, "target_country_logp": -19.66844940185547, "source_code_first_rank": 118802.5, "target_code_first_rank": 119514.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 24715.0, "target_country_rank": 5340.0, "source_country_logp": -30.924062728881836, "target_country_logp": -27.736562728881836, "source_code_first_rank": 28730.5, "target_code_first_rank": 26020.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 49.5, "source_country_logp": -3.0695178508758545, "target_country_logp": -6.444518089294434, "source_code_first_rank": 11059.5, "target_code_first_rank": 72114.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 4681.5, "target_country_rank": 3474.5, "source_country_logp": -11.881502151489258, "target_country_logp": -11.373689651489258, "source_code_first_rank": 2311.0, "target_code_first_rank": 147110.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 958.0, "source_country_logp": -1.7152214050292969, "target_country_logp": -17.715221405029297, "source_code_first_rank": 112171.5, "target_code_first_rank": 107214.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 13965.5, "target_country_rank": 18368.0, "source_country_logp": -17.83192253112793, "target_country_logp": -18.33192253112793, "source_code_first_rank": 32298.5, "target_code_first_rank": 64894.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 100.0, "source_country_logp": -1.5832757949829102, "target_country_logp": -7.48952579498291, "source_code_first_rank": 5925.0, "target_code_first_rank": 95131.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 26489.0, "target_country_rank": 6327.0, "source_country_logp": -15.613324165344238, "target_country_logp": -12.675824165344238, "source_code_first_rank": 8618.5, "target_code_first_rank": 128206.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 555.5, "source_country_logp": -0.8188551068305969, "target_country_logp": -19.19385528564453, "source_code_first_rank": 107373.5, "target_code_first_rank": 99920.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 99797.0, "target_country_rank": 51554.0, "source_country_logp": -14.990001678466797, "target_country_logp": -13.302501678466797, "source_code_first_rank": 35000.5, "target_code_first_rank": 34279.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 11.5, "target_country_rank": 1062.0, "source_country_logp": -7.47197961807251, "target_country_logp": -16.15947914123535, "source_code_first_rank": 1365.0, "target_code_first_rank": 122362.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 272.0, "target_country_rank": 105232.0, "source_country_logp": -19.56403923034668, "target_country_logp": -37.61091613769531, "source_code_first_rank": 1.0, "target_code_first_rank": 272.0}], "actual_source_country_rank": 272.0, "actual_target_country_rank": 105232.0, "array_path": "results/readout-arrays/eval-12-main.npz"}
{"case_id": "eval-12", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 4.0, "target_country_rank": 63.0, "source_country_logp": -3.3278799057006836, "target_country_logp": -7.327879905700684, "source_code_first_rank": 20926.0, "target_code_first_rank": 50762.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 6189.0, "target_country_rank": 38838.0, "source_country_logp": -12.597536087036133, "target_country_logp": -16.949098587036133, "source_code_first_rank": 72890.5, "target_code_first_rank": 130428.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 566.0, "source_country_logp": -2.4368748664855957, "target_country_logp": -20.124374389648438, "source_code_first_rank": 113025.5, "target_code_first_rank": 114883.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 48988.0, "target_country_rank": 15048.0, "source_country_logp": -34.28828048706055, "target_country_logp": -31.257030487060547, "source_code_first_rank": 26855.5, "target_code_first_rank": 31105.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 32.5, "source_country_logp": -3.166428565979004, "target_country_logp": -5.728928565979004, "source_code_first_rank": 9219.5, "target_code_first_rank": 64217.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 4465.5, "target_country_rank": 1247.5, "source_country_logp": -11.289691925048828, "target_country_logp": -9.438129425048828, "source_code_first_rank": 17019.5, "target_code_first_rank": 144603.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 858.0, "source_country_logp": -1.2652647495269775, "target_country_logp": -18.0777645111084, "source_code_first_rank": 113629.5, "target_code_first_rank": 104213.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 15896.0, "target_country_rank": 9330.0, "source_country_logp": -21.780426025390625, "target_country_logp": -20.749176025390625, "source_code_first_rank": 42645.0, "target_code_first_rank": 60118.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 76.5, "source_country_logp": -1.9939584732055664, "target_country_logp": -7.087708473205566, "source_code_first_rank": 4749.5, "target_code_first_rank": 89890.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 16331.0, "target_country_rank": 5216.5, "source_country_logp": -15.758788108825684, "target_country_logp": -13.571288108825684, "source_code_first_rank": 2745.5, "target_code_first_rank": 146664.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 467.0, "source_country_logp": -0.8568569421768188, "target_country_logp": -19.106857299804688, "source_code_first_rank": 105710.5, "target_code_first_rank": 99530.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 104089.0, "target_country_rank": 51909.0, "source_country_logp": -15.35113525390625, "target_country_logp": -13.66363525390625, "source_code_first_rank": 44775.0, "target_code_first_rank": 56676.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 13.0, "target_country_rank": 823.0, "source_country_logp": -7.038978099822998, "target_country_logp": -15.726478576660156, "source_code_first_rank": 1557.5, "target_code_first_rank": 120265.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 308.5, "target_country_rank": 13397.5, "source_country_logp": -26.187541961669922, "target_country_logp": -37.18754196166992, "source_code_first_rank": 8.0, "target_code_first_rank": 815.5}], "actual_source_country_rank": 308.5, "actual_target_country_rank": 13397.5, "array_path": "results/readout-arrays/eval-12-paraphrase.npz"}
{"case_id": "eval-13", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 3.0, "target_country_rank": 5678.0, "source_country_logp": -2.4366087913513184, "target_country_logp": -13.850671768188477, "source_code_first_rank": 9118.5, "target_code_first_rank": 63646.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 28633.0, "target_country_rank": 4918.0, "source_country_logp": -15.974218368530273, "target_country_logp": -12.130468368530273, "source_code_first_rank": 124310.0, "target_code_first_rank": 11124.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 8.0, "target_country_rank": 4232.5, "source_country_logp": -10.755674362182617, "target_country_logp": -32.66192626953125, "source_code_first_rank": 94463.5, "target_code_first_rank": 150833.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 5716.0, "target_country_rank": 34462.0, "source_country_logp": -27.682693481445312, "target_country_logp": -31.526443481445312, "source_code_first_rank": 25742.0, "target_code_first_rank": 64529.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 10.5, "target_country_rank": 1516.0, "source_country_logp": -4.833642959594727, "target_country_logp": -12.083642959594727, "source_code_first_rank": 39837.5, "target_code_first_rank": 70761.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 3394.0, "target_country_rank": 11587.5, "source_country_logp": -11.282779693603516, "target_country_logp": -13.493717193603516, "source_code_first_rank": 147392.5, "target_code_first_rank": 19948.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 1667.0, "source_country_logp": -6.638590335845947, "target_country_logp": -23.57608985900879, "source_code_first_rank": 72526.5, "target_code_first_rank": 138665.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 16967.0, "target_country_rank": 57867.0, "source_country_logp": -19.893489837646484, "target_country_logp": -22.893489837646484, "source_code_first_rank": 70109.0, "target_code_first_rank": 37532.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 9.0, "target_country_rank": 671.0, "source_country_logp": -4.915929317474365, "target_country_logp": -10.447178840637207, "source_code_first_rank": 31282.0, "target_code_first_rank": 37900.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 5843.5, "target_country_rank": 7696.5, "source_country_logp": -12.402071952819824, "target_country_logp": -12.878634452819824, "source_code_first_rank": 136356.0, "target_code_first_rank": 4624.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 4.0, "target_country_rank": 2636.5, "source_country_logp": -6.078693866729736, "target_country_logp": -29.453693389892578, "source_code_first_rank": 50945.5, "target_code_first_rank": 143209.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 47715.0, "target_country_rank": 102984.0, "source_country_logp": -13.241632461547852, "target_country_logp": -15.210382461547852, "source_code_first_rank": 38943.0, "target_code_first_rank": 51875.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 12.0, "target_country_rank": 5203.5, "source_country_logp": -6.264729976654053, "target_country_logp": -17.76473045349121, "source_code_first_rank": 21590.0, "target_code_first_rank": 116921.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 189.0, "target_country_rank": 88909.0, "source_country_logp": -16.418176651000977, "target_country_logp": -33.84005355834961, "source_code_first_rank": 3.0, "target_code_first_rank": 113.0}], "actual_source_country_rank": 189.0, "actual_target_country_rank": 88909.0, "array_path": "results/readout-arrays/eval-13-main.npz"}
{"case_id": "eval-13", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 6.0, "target_country_rank": 10391.5, "source_country_logp": -3.8174662590026855, "target_country_logp": -14.841270446777344, "source_code_first_rank": 11634.5, "target_code_first_rank": 64830.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 35684.0, "target_country_rank": 6413.5, "source_country_logp": -16.639503479003906, "target_country_logp": -12.631690979003906, "source_code_first_rank": 129569.5, "target_code_first_rank": 12786.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 9.0, "target_country_rank": 3188.0, "source_country_logp": -11.50436019897461, "target_country_logp": -31.06686019897461, "source_code_first_rank": 101155.5, "target_code_first_rank": 150733.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 13056.5, "target_country_rank": 52058.0, "source_country_logp": -30.69415855407715, "target_country_logp": -34.225406646728516, "source_code_first_rank": 29377.0, "target_code_first_rank": 54102.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 10.5, "target_country_rank": 2588.5, "source_country_logp": -5.442424297332764, "target_country_logp": -13.231486320495605, "source_code_first_rank": 41599.5, "target_code_first_rank": 66208.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 1209.5, "target_country_rank": 12554.0, "source_country_logp": -9.337752342224121, "target_country_logp": -13.064314842224121, "source_code_first_rank": 143991.5, "target_code_first_rank": 14484.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 6.0, "target_country_rank": 2296.5, "source_country_logp": -7.63709831237793, "target_country_logp": -24.51209831237793, "source_code_first_rank": 80038.5, "target_code_first_rank": 139176.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 7996.0, "target_country_rank": 40057.5, "source_country_logp": -20.925647735595703, "target_country_logp": -24.363147735595703, "source_code_first_rank": 61546.5, "target_code_first_rank": 44943.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 13.0, "target_country_rank": 890.5, "source_country_logp": -5.716221332550049, "target_country_logp": -11.309971809387207, "source_code_first_rank": 30295.0, "target_code_first_rank": 42918.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 3695.0, "target_country_rank": 15803.5, "source_country_logp": -13.026381492614746, "target_country_logp": -15.737318992614746, "source_code_first_rank": 146774.5, "target_code_first_rank": 4175.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 5.5, "target_country_rank": 3171.0, "source_country_logp": -7.289622783660889, "target_country_logp": -30.227123260498047, "source_code_first_rank": 55242.5, "target_code_first_rank": 143984.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 45161.5, "target_country_rank": 124659.0, "source_country_logp": -13.705719947814941, "target_country_logp": -16.705718994140625, "source_code_first_rank": 57876.0, "target_code_first_rank": 64409.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 14.0, "target_country_rank": 6202.0, "source_country_logp": -5.284550189971924, "target_country_logp": -18.284549713134766, "source_code_first_rank": 29452.0, "target_code_first_rank": 109737.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 46.0, "target_country_rank": 21214.0, "source_country_logp": -21.625036239624023, "target_country_logp": -37.343788146972656, "source_code_first_rank": 26.5, "target_code_first_rank": 646.5}], "actual_source_country_rank": 46.0, "actual_target_country_rank": 21214.0, "array_path": "results/readout-arrays/eval-13-paraphrase.npz"}
{"case_id": "eval-14", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 17.0, "target_country_rank": 765.5, "source_country_logp": -4.662764072418213, "target_country_logp": -10.006514549255371, "source_code_first_rank": 3314.0, "target_code_first_rank": 67466.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 30364.5, "target_country_rank": 4394.5, "source_country_logp": -16.1372127532959, "target_country_logp": -11.949711799621582, "source_code_first_rank": 121443.0, "target_code_first_rank": 10759.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 9.0, "target_country_rank": 215.5, "source_country_logp": -7.374580383300781, "target_country_logp": -14.124580383300781, "source_code_first_rank": 123868.5, "target_code_first_rank": 142811.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 5597.0, "target_country_rank": 30273.5, "source_country_logp": -27.669189453125, "target_country_logp": -31.294189453125, "source_code_first_rank": 27786.5, "target_code_first_rank": 63009.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 196.5, "target_country_rank": 1286.5, "source_country_logp": -8.36581802368164, "target_country_logp": -10.91269302368164, "source_code_first_rank": 17383.0, "target_code_first_rank": 68401.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 3374.5, "target_country_rank": 10852.5, "source_country_logp": -11.332540512084961, "target_country_logp": -13.453634262084961, "source_code_first_rank": 146931.0, "target_code_first_rank": 18953.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 18.0, "target_country_rank": 267.5, "source_country_logp": -5.216092109680176, "target_country_logp": -9.966092109680176, "source_code_first_rank": 104896.5, "target_code_first_rank": 114762.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 16190.0, "target_country_rank": 51992.0, "source_country_logp": -19.877788543701172, "target_country_logp": -22.627788543701172, "source_code_first_rank": 67676.0, "target_code_first_rank": 37583.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 67.5, "target_country_rank": 437.0, "source_country_logp": -7.271231174468994, "target_country_logp": -9.771230697631836, "source_code_first_rank": 23787.5, "target_code_first_rank": 14767.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 5585.5, "target_country_rank": 6469.0, "source_country_logp": -12.217474937438965, "target_country_logp": -12.451849937438965, "source_code_first_rank": 127237.0, "target_code_first_rank": 5111.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 9.0, "target_country_rank": 744.5, "source_country_logp": -5.2625203132629395, "target_country_logp": -17.51251983642578, "source_code_first_rank": 56156.5, "target_code_first_rank": 112559.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 42396.5, "target_country_rank": 81168.0, "source_country_logp": -13.093005180358887, "target_country_logp": -14.530505180358887, "source_code_first_rank": 37827.5, "target_code_first_rank": 54544.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 9.0, "target_country_rank": 7922.5, "source_country_logp": -6.134131908416748, "target_country_logp": -18.852882385253906, "source_code_first_rank": 54475.0, "target_code_first_rank": 81196.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 332.5, "target_country_rank": 58924.0, "source_country_logp": -19.288389205932617, "target_country_logp": -33.48370361328125, "source_code_first_rank": 3.0, "target_code_first_rank": 133.5}], "actual_source_country_rank": 332.5, "actual_target_country_rank": 58924.0, "array_path": "results/readout-arrays/eval-14-main.npz"}
{"case_id": "eval-14", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 18.0, "target_country_rank": 1469.5, "source_country_logp": -4.533614635467529, "target_country_logp": -11.017989158630371, "source_code_first_rank": 3167.0, "target_code_first_rank": 84002.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 35948.0, "target_country_rank": 4745.5, "source_country_logp": -16.73822784423828, "target_country_logp": -12.144477844238281, "source_code_first_rank": 126262.0, "target_code_first_rank": 13624.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 12.5, "target_country_rank": 226.5, "source_country_logp": -8.794493675231934, "target_country_logp": -15.794493675231934, "source_code_first_rank": 118631.5, "target_code_first_rank": 143904.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 10311.0, "target_country_rank": 42126.0, "source_country_logp": -31.20941734313965, "target_country_logp": -34.756290435791016, "source_code_first_rank": 30648.0, "target_code_first_rank": 53518.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 108.5, "target_country_rank": 1340.5, "source_country_logp": -7.848965167999268, "target_country_logp": -11.223965644836426, "source_code_first_rank": 19371.5, "target_code_first_rank": 82510.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 1291.5, "target_country_rank": 10968.0, "source_country_logp": -9.464435577392578, "target_country_logp": -12.827716827392578, "source_code_first_rank": 144400.5, "target_code_first_rank": 16113.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 24.0, "target_country_rank": 316.5, "source_country_logp": -6.175207614898682, "target_country_logp": -10.800207138061523, "source_code_first_rank": 107649.0, "target_code_first_rank": 121186.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 7140.5, "target_country_rank": 31721.5, "source_country_logp": -20.754249572753906, "target_country_logp": -23.816749572753906, "source_code_first_rank": 62429.5, "target_code_first_rank": 45419.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 72.0, "target_country_rank": 547.5, "source_country_logp": -7.4457902908325195, "target_country_logp": -9.97704029083252, "source_code_first_rank": 19253.5, "target_code_first_rank": 18052.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 4610.0, "target_country_rank": 16326.0, "source_country_logp": -13.065882682800293, "target_country_logp": -15.472132682800293, "source_code_first_rank": 145245.5, "target_code_first_rank": 4153.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 6.0, "target_country_rank": 916.0, "source_country_logp": -7.1667704582214355, "target_country_logp": -20.041770935058594, "source_code_first_rank": 52663.5, "target_code_first_rank": 112946.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 45463.5, "target_country_rank": 122517.0, "source_country_logp": -13.635255813598633, "target_country_logp": -16.416505813598633, "source_code_first_rank": 55696.5, "target_code_first_rank": 67322.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 14.0, "target_country_rank": 8091.5, "source_country_logp": -6.285018444061279, "target_country_logp": -19.160018920898438, "source_code_first_rank": 66370.0, "target_code_first_rank": 88523.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 22.5, "target_country_rank": 9607.5, "source_country_logp": -22.250011444091797, "target_country_logp": -38.6250114440918, "source_code_first_rank": 124.0, "target_code_first_rank": 873.0}], "actual_source_country_rank": 22.5, "actual_target_country_rank": 9607.5, "array_path": "results/readout-arrays/eval-14-paraphrase.npz"}
{"case_id": "eval-15", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 7.0, "target_country_rank": 390.0, "source_country_logp": -3.761343479156494, "target_country_logp": -8.605093955993652, "source_code_first_rank": 101451.5, "target_code_first_rank": 18877.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 4837.5, "target_country_rank": 25849.5, "source_country_logp": -12.097787857055664, "target_country_logp": -15.707162857055664, "source_code_first_rank": 10427.5, "target_code_first_rank": 22140.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 73.0, "source_country_logp": -8.250320434570312, "target_country_logp": -29.000320434570312, "source_code_first_rank": 97533.5, "target_code_first_rank": 55578.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 32686.0, "target_country_rank": 18608.0, "source_country_logp": -32.16609191894531, "target_country_logp": -30.79109001159668, "source_code_first_rank": 63103.5, "target_code_first_rank": 39867.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 8.5, "target_country_rank": 209.0, "source_country_logp": -4.3780694007873535, "target_country_logp": -8.721818923950195, "source_code_first_rank": 92754.0, "target_code_first_rank": 9863.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 12702.0, "target_country_rank": 19051.0, "source_country_logp": -13.563886642456055, "target_country_logp": -14.431074142456055, "source_code_first_rank": 17596.0, "target_code_first_rank": 3207.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 57.5, "source_country_logp": -5.255311012268066, "target_country_logp": -23.380311965942383, "source_code_first_rank": 56712.0, "target_code_first_rank": 73927.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 57417.0, "target_country_rank": 20909.0, "source_country_logp": -23.081119537353516, "target_country_logp": -20.581119537353516, "source_code_first_rank": 42010.0, "target_code_first_rank": 45012.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 6.0, "target_country_rank": 277.5, "source_country_logp": -3.972461700439453, "target_country_logp": -9.097461700439453, "source_code_first_rank": 74603.0, "target_code_first_rank": 4996.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 8984.0, "target_country_rank": 34258.0, "source_country_logp": -13.377045631408691, "target_country_logp": -16.310640335083008, "source_code_first_rank": 5198.5, "target_code_first_rank": 1522.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 56.0, "source_country_logp": -2.579714775085449, "target_country_logp": -18.704713821411133, "source_code_first_rank": 77147.0, "target_code_first_rank": 90527.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 102975.0, "target_country_rank": 108472.0, "source_country_logp": -15.160894393920898, "target_country_logp": -15.410894393920898, "source_code_first_rank": 55664.0, "target_code_first_rank": 33932.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 8.5, "target_country_rank": 253.0, "source_country_logp": -5.5864057540893555, "target_country_logp": -13.211405754089355, "source_code_first_rank": 3358.5, "target_code_first_rank": 5717.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 30.5, "target_country_rank": 39969.5, "source_country_logp": -14.281204223632812, "target_country_logp": -32.86128234863281, "source_code_first_rank": 1.0, "target_code_first_rank": 157.0}], "actual_source_country_rank": 30.5, "actual_target_country_rank": 39969.5, "array_path": "results/readout-arrays/eval-15-main.npz"}
{"case_id": "eval-15", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 29.0, "target_country_rank": 281.5, "source_country_logp": -5.247045993804932, "target_country_logp": -7.965795993804932, "source_code_first_rank": 100855.5, "target_code_first_rank": 24388.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 6178.0, "target_country_rank": 25076.5, "source_country_logp": -12.55206298828125, "target_country_logp": -15.63800048828125, "source_code_first_rank": 12684.0, "target_code_first_rank": 22097.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 72.5, "source_country_logp": -9.250123977661133, "target_country_logp": -29.250123977661133, "source_code_first_rank": 102353.0, "target_code_first_rank": 63394.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 50709.0, "target_country_rank": 30760.5, "source_country_logp": -34.55720138549805, "target_country_logp": -33.08845138549805, "source_code_first_rank": 53687.5, "target_code_first_rank": 35329.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 10.5, "target_country_rank": 217.0, "source_country_logp": -5.464393138885498, "target_country_logp": -9.058143615722656, "source_code_first_rank": 94234.0, "target_code_first_rank": 8639.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 12870.0, "target_country_rank": 21059.5, "source_country_logp": -13.159390449523926, "target_country_logp": -14.207242012023926, "source_code_first_rank": 13870.0, "target_code_first_rank": 25270.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 46.0, "source_country_logp": -6.002535343170166, "target_country_logp": -22.377534866333008, "source_code_first_rank": 59199.0, "target_code_first_rank": 80659.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 40189.5, "target_country_rank": 15716.0, "source_country_logp": -24.953964233398438, "target_country_logp": -22.766464233398438, "source_code_first_rank": 45050.5, "target_code_first_rank": 52657.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 12.0, "target_country_rank": 297.5, "source_country_logp": -5.40818452835083, "target_country_logp": -9.626934051513672, "source_code_first_rank": 74561.5, "target_code_first_rank": 5241.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 14776.0, "target_country_rank": 23088.5, "source_country_logp": -15.603446960449219, "target_country_logp": -16.64641571044922, "source_code_first_rank": 3674.5, "target_code_first_rank": 3521.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 50.5, "source_country_logp": -2.5796854496002197, "target_country_logp": -17.95468521118164, "source_code_first_rank": 83423.0, "target_code_first_rank": 92521.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 122056.5, "target_country_rank": 108721.5, "source_country_logp": -16.534547805786133, "target_country_logp": -15.847047805786133, "source_code_first_rank": 60323.5, "target_code_first_rank": 44919.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 11.5, "target_country_rank": 234.5, "source_country_logp": -5.534170627593994, "target_country_logp": -13.034170150756836, "source_code_first_rank": 2939.0, "target_code_first_rank": 4290.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 27.5, "target_country_rank": 4629.5, "source_country_logp": -20.12502670288086, "target_country_logp": -34.12502670288086, "source_code_first_rank": 2.0, "target_code_first_rank": 163.0}], "actual_source_country_rank": 27.5, "actual_target_country_rank": 4629.5, "array_path": "results/readout-arrays/eval-15-paraphrase.npz"}
{"case_id": "eval-16", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 7.0, "target_country_rank": 536.0, "source_country_logp": -3.4105803966522217, "target_country_logp": -9.4105806350708, "source_code_first_rank": 46480.0, "target_code_first_rank": 59075.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 4552.0, "target_country_rank": 25404.5, "source_country_logp": -12.082377433776855, "target_country_logp": -15.746439933776855, "source_code_first_rank": 11522.5, "target_code_first_rank": 23150.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 130.5, "source_country_logp": -0.8882595300674438, "target_country_logp": -21.263259887695312, "source_code_first_rank": 39919.0, "target_code_first_rank": 31096.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 27980.0, "target_country_rank": 16912.5, "source_country_logp": -31.20744514465332, "target_country_logp": -30.01994514465332, "source_code_first_rank": 61438.5, "target_code_first_rank": 41601.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 12.0, "target_country_rank": 584.5, "source_country_logp": -4.509275436401367, "target_country_logp": -9.728025436401367, "source_code_first_rank": 38826.5, "target_code_first_rank": 23978.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 11525.0, "target_country_rank": 16841.5, "source_country_logp": -13.610093116760254, "target_country_logp": -14.416733741760254, "source_code_first_rank": 17712.0, "target_code_first_rank": 3198.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 179.0, "source_country_logp": -0.9873285293579102, "target_country_logp": -18.737329483032227, "source_code_first_rank": 40150.5, "target_code_first_rank": 62099.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 59477.0, "target_country_rank": 21386.0, "source_country_logp": -22.826684951782227, "target_country_logp": -20.295434951782227, "source_code_first_rank": 38119.5, "target_code_first_rank": 36717.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 6.0, "target_country_rank": 388.0, "source_country_logp": -3.457857370376587, "target_country_logp": -9.239107131958008, "source_code_first_rank": 13777.5, "target_code_first_rank": 10739.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 7434.0, "target_country_rank": 40148.5, "source_country_logp": -12.761456489562988, "target_country_logp": -16.480205535888672, "source_code_first_rank": 6054.5, "target_code_first_rank": 2004.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 91.5, "source_country_logp": -0.07475990802049637, "target_country_logp": -17.32476043701172, "source_code_first_rank": 54088.0, "target_code_first_rank": 76006.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 94947.5, "target_country_rank": 110058.5, "source_country_logp": -14.956417083740234, "target_country_logp": -15.581417083740234, "source_code_first_rank": 56459.5, "target_code_first_rank": 30884.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 14.5, "target_country_rank": 3450.5, "source_country_logp": -7.181321620941162, "target_country_logp": -17.36882209777832, "source_code_first_rank": 1578.5, "target_code_first_rank": 9256.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 54.5, "target_country_rank": 34783.5, "source_country_logp": -15.268829345703125, "target_country_logp": -31.636016845703125, "source_code_first_rank": 1.0, "target_code_first_rank": 129.0}], "actual_source_country_rank": 54.5, "actual_target_country_rank": 34783.5, "array_path": "results/readout-arrays/eval-16-main.npz"}
{"case_id": "eval-16", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 7.0, "target_country_rank": 821.0, "source_country_logp": -3.2940971851348877, "target_country_logp": -9.919096946716309, "source_code_first_rank": 42837.0, "target_code_first_rank": 59004.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 5441.0, "target_country_rank": 24004.5, "source_country_logp": -12.391505241394043, "target_country_logp": -15.614161491394043, "source_code_first_rank": 12650.0, "target_code_first_rank": 22138.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 169.5, "source_country_logp": -0.9052921533584595, "target_country_logp": -21.780292510986328, "source_code_first_rank": 35971.5, "target_code_first_rank": 33788.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 43252.5, "target_country_rank": 25735.5, "source_country_logp": -34.458343505859375, "target_country_logp": -33.020843505859375, "source_code_first_rank": 49638.0, "target_code_first_rank": 34220.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 16.0, "target_country_rank": 1035.0, "source_country_logp": -4.627047538757324, "target_country_logp": -10.564547538757324, "source_code_first_rank": 35520.0, "target_code_first_rank": 18841.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 11702.0, "target_country_rank": 16057.5, "source_country_logp": -12.980838775634766, "target_country_logp": -13.600467681884766, "source_code_first_rank": 16327.0, "target_code_first_rank": 28136.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 280.5, "source_country_logp": -1.0035783052444458, "target_country_logp": -18.753578186035156, "source_code_first_rank": 38021.0, "target_code_first_rank": 68197.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 32497.5, "target_country_rank": 12101.0, "source_country_logp": -24.242076873779297, "target_country_logp": -22.085826873779297, "source_code_first_rank": 45934.0, "target_code_first_rank": 51904.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 4.0, "target_country_rank": 501.0, "source_country_logp": -3.6007368564605713, "target_country_logp": -9.725736618041992, "source_code_first_rank": 15161.0, "target_code_first_rank": 10153.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 12369.0, "target_country_rank": 24463.5, "source_country_logp": -14.876944541931152, "target_country_logp": -16.43163299560547, "source_code_first_rank": 3896.0, "target_code_first_rank": 4050.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 116.5, "source_country_logp": -0.07403706759214401, "target_country_logp": -17.824037551879883, "source_code_first_rank": 54520.5, "target_code_first_rank": 79832.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 118626.0, "target_country_rank": 109686.5, "source_country_logp": -15.94565486907959, "target_country_logp": -15.50815486907959, "source_code_first_rank": 65242.5, "target_code_first_rank": 42847.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 22.0, "target_country_rank": 5557.0, "source_country_logp": -7.050554275512695, "target_country_logp": -18.519304275512695, "source_code_first_rank": 1347.5, "target_code_first_rank": 9841.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 25.5, "target_country_rank": 5127.0, "source_country_logp": -25.375, "target_country_logp": -38.75, "source_code_first_rank": 5.0, "target_code_first_rank": 112.0}], "actual_source_country_rank": 25.5, "actual_target_country_rank": 5127.0, "array_path": "results/readout-arrays/eval-16-paraphrase.npz"}
{"case_id": "eval-17", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 223.5, "source_country_logp": -1.6203866004943848, "target_country_logp": -8.807886123657227, "source_code_first_rank": 49054.5, "target_code_first_rank": 99247.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 22150.0, "target_country_rank": 10908.0, "source_country_logp": -15.254105567932129, "target_country_logp": -13.636918067932129, "source_code_first_rank": 19682.5, "target_code_first_rank": 31404.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 7.0, "target_country_rank": 1187.0, "source_country_logp": -4.006846904754639, "target_country_logp": -17.194347381591797, "source_code_first_rank": 29531.0, "target_code_first_rank": 43634.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 11952.0, "target_country_rank": 9023.5, "source_country_logp": -30.450773239135742, "target_country_logp": -29.857023239135742, "source_code_first_rank": 37946.0, "target_code_first_rank": 92965.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 11.0, "target_country_rank": 178.0, "source_country_logp": -3.814882278442383, "target_country_logp": -7.939882278442383, "source_code_first_rank": 30435.0, "target_code_first_rank": 106719.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 19964.0, "target_country_rank": 7091.0, "source_country_logp": -14.733049392700195, "target_country_logp": -12.619768142700195, "source_code_first_rank": 2718.0, "target_code_first_rank": 8886.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 22.5, "target_country_rank": 1592.0, "source_country_logp": -6.910376071929932, "target_country_logp": -20.035375595092773, "source_code_first_rank": 58354.5, "target_code_first_rank": 42238.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 15590.0, "target_country_rank": 8451.5, "source_country_logp": -19.949256896972656, "target_country_logp": -18.839881896972656, "source_code_first_rank": 36302.0, "target_code_first_rank": 33342.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 7.0, "target_country_rank": 3164.5, "source_country_logp": -4.129035949707031, "target_country_logp": -12.316535949707031, "source_code_first_rank": 6018.0, "target_code_first_rank": 101340.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 37512.5, "target_country_rank": 76365.0, "source_country_logp": -16.195587158203125, "target_country_logp": -18.633087158203125, "source_code_first_rank": 1250.5, "target_code_first_rank": 16555.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 27.0, "target_country_rank": 3827.5, "source_country_logp": -8.416110038757324, "target_country_logp": -24.22861099243164, "source_code_first_rank": 43554.5, "target_code_first_rank": 29944.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 106280.0, "target_country_rank": 111326.0, "source_country_logp": -15.573625564575195, "target_country_logp": -15.823625564575195, "source_code_first_rank": 29368.5, "target_code_first_rank": 3478.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 22.5, "target_country_rank": 9516.0, "source_country_logp": -7.762992858886719, "target_country_logp": -19.01299285888672, "source_code_first_rank": 1719.0, "target_code_first_rank": 91125.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 203.5, "target_country_rank": 148369.5, "source_country_logp": -19.714630126953125, "target_country_logp": -42.777130126953125, "source_code_first_rank": 1.0, "target_code_first_rank": 272.5}], "actual_source_country_rank": 203.5, "actual_target_country_rank": 148369.5, "array_path": "results/readout-arrays/eval-17-main.npz"}
{"case_id": "eval-17", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 506.5, "source_country_logp": -1.1027110815048218, "target_country_logp": -9.915210723876953, "source_code_first_rank": 55613.0, "target_code_first_rank": 99107.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 18657.5, "target_country_rank": 12727.0, "source_country_logp": -14.844072341918945, "target_country_logp": -13.976884841918945, "source_code_first_rank": 19014.0, "target_code_first_rank": 36730.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 7.0, "target_country_rank": 1261.0, "source_country_logp": -3.7073750495910645, "target_country_logp": -18.207374572753906, "source_code_first_rank": 30121.0, "target_code_first_rank": 45028.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 22716.5, "target_country_rank": 18691.0, "source_country_logp": -32.93857192993164, "target_country_logp": -32.46982192993164, "source_code_first_rank": 35156.5, "target_code_first_rank": 98564.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 6.0, "target_country_rank": 310.5, "source_country_logp": -3.568021774291992, "target_country_logp": -8.536771774291992, "source_code_first_rank": 25521.5, "target_code_first_rank": 101512.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 16891.0, "target_country_rank": 10910.0, "source_country_logp": -13.777938842773438, "target_country_logp": -12.912460327148438, "source_code_first_rank": 23752.5, "target_code_first_rank": 21274.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 33.0, "target_country_rank": 2125.0, "source_country_logp": -8.589794158935547, "target_country_logp": -23.339794158935547, "source_code_first_rank": 60695.0, "target_code_first_rank": 43081.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 12750.5, "target_country_rank": 5933.5, "source_country_logp": -22.45830726623535, "target_country_logp": -20.98955726623535, "source_code_first_rank": 52994.5, "target_code_first_rank": 60534.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 3.0, "target_country_rank": 1921.5, "source_country_logp": -3.3536248207092285, "target_country_logp": -11.291125297546387, "source_code_first_rank": 5192.5, "target_code_first_rank": 99413.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 24335.0, "target_country_rank": 30239.5, "source_country_logp": -16.617469787597656, "target_country_logp": -17.179969787597656, "source_code_first_rank": 3322.0, "target_code_first_rank": 17938.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 16.0, "target_country_rank": 4691.0, "source_country_logp": -5.8859477043151855, "target_country_logp": -24.260948181152344, "source_code_first_rank": 43807.5, "target_code_first_rank": 33352.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 109715.0, "target_country_rank": 99177.5, "source_country_logp": -16.391462326049805, "target_country_logp": -15.953962326049805, "source_code_first_rank": 52038.0, "target_code_first_rank": 20136.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 27.0, "target_country_rank": 11115.5, "source_country_logp": -7.357054233551025, "target_country_logp": -19.482053756713867, "source_code_first_rank": 1763.5, "target_code_first_rank": 91490.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 63.0, "target_country_rank": 62946.0, "source_country_logp": -23.1250057220459, "target_country_logp": -43.648441314697266, "source_code_first_rank": 6.5, "target_code_first_rank": 210.0}], "actual_source_country_rank": 63.0, "actual_target_country_rank": 62946.0, "array_path": "results/readout-arrays/eval-17-paraphrase.npz"}
{"case_id": "eval-18", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 955.0, "target_country_rank": 694.5, "source_country_logp": -10.793733596801758, "target_country_logp": -10.293733596801758, "source_code_first_rank": 9390.0, "target_code_first_rank": 62983.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 24776.5, "target_country_rank": 14433.5, "source_country_logp": -15.633256912231445, "target_country_logp": -14.335893630981445, "source_code_first_rank": 22451.5, "target_code_first_rank": 30545.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 955.0, "target_country_rank": 31597.5, "source_country_logp": -20.014841079711914, "target_country_logp": -29.733591079711914, "source_code_first_rank": 19056.5, "target_code_first_rank": 91001.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 17668.0, "target_country_rank": 13382.5, "source_country_logp": -30.48004913330078, "target_country_logp": -29.88629913330078, "source_code_first_rank": 41037.5, "target_code_first_rank": 93753.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 679.5, "target_country_rank": 1266.0, "source_country_logp": -10.34268856048584, "target_country_logp": -11.38956356048584, "source_code_first_rank": 13517.0, "target_code_first_rank": 20783.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 15530.5, "target_country_rank": 8871.0, "source_country_logp": -14.049881935119629, "target_country_logp": -12.942948341369629, "source_code_first_rank": 4125.5, "target_code_first_rank": 8882.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 105.0, "target_country_rank": 5907.5, "source_country_logp": -7.714974403381348, "target_country_logp": -15.777474403381348, "source_code_first_rank": 20324.0, "target_code_first_rank": 60337.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 20400.5, "target_country_rank": 12767.0, "source_country_logp": -19.82774543762207, "target_country_logp": -18.95274543762207, "source_code_first_rank": 44276.0, "target_code_first_rank": 40203.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 879.5, "target_country_rank": 4560.0, "source_country_logp": -11.355470657348633, "target_country_logp": -14.230470657348633, "source_code_first_rank": 1966.5, "target_code_first_rank": 19177.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 26319.0, "target_country_rank": 63218.5, "source_country_logp": -15.539552688598633, "target_country_logp": -18.172365188598633, "source_code_first_rank": 1352.0, "target_code_first_rank": 17856.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 69.5, "target_country_rank": 6410.0, "source_country_logp": -11.239944458007812, "target_country_logp": -20.489944458007812, "source_code_first_rank": 13588.5, "target_code_first_rank": 64698.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 106606.0, "target_country_rank": 109269.0, "source_country_logp": -15.328030586242676, "target_country_logp": -15.453030586242676, "source_code_first_rank": 30962.5, "target_code_first_rank": 4791.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 653.5, "target_country_rank": 15311.5, "source_country_logp": -13.597610473632812, "target_country_logp": -19.675735473632812, "source_code_first_rank": 344.5, "target_code_first_rank": 6670.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 3653.0, "target_country_rank": 120516.5, "source_country_logp": -25.252531051635742, "target_country_logp": -36.940032958984375, "source_code_first_rank": 3.0, "target_code_first_rank": 76.0}], "actual_source_country_rank": 3653.0, "actual_target_country_rank": 120516.5, "array_path": "results/readout-arrays/eval-18-main.npz"}
{"case_id": "eval-18", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 253.0, "target_country_rank": 1229.0, "source_country_logp": -9.050524711608887, "target_country_logp": -11.597399711608887, "source_code_first_rank": 5708.0, "target_code_first_rank": 52605.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 27206.5, "target_country_rank": 18926.0, "source_country_logp": -15.895938873291016, "target_country_logp": -14.977970123291016, "source_code_first_rank": 22548.0, "target_code_first_rank": 32801.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 392.0, "target_country_rank": 27885.0, "source_country_logp": -16.660293579101562, "target_country_logp": -28.660293579101562, "source_code_first_rank": 13149.5, "target_code_first_rank": 74943.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 29495.0, "target_country_rank": 24653.5, "source_country_logp": -33.324039459228516, "target_country_logp": -32.855289459228516, "source_code_first_rank": 35759.5, "target_code_first_rank": 95010.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 209.0, "target_country_rank": 2032.0, "source_country_logp": -8.506763458251953, "target_country_logp": -12.069263458251953, "source_code_first_rank": 13726.5, "target_code_first_rank": 16409.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 15795.5, "target_country_rank": 11021.0, "source_country_logp": -13.544836044311523, "target_country_logp": -12.839757919311523, "source_code_first_rank": 26031.5, "target_code_first_rank": 16276.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 19.0, "target_country_rank": 7545.5, "source_country_logp": -5.326873302459717, "target_country_logp": -17.826873779296875, "source_code_first_rank": 20421.5, "target_code_first_rank": 43444.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 14219.5, "target_country_rank": 8243.5, "source_country_logp": -22.76641845703125, "target_country_logp": -21.70391845703125, "source_code_first_rank": 54928.0, "target_code_first_rank": 55990.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 247.5, "target_country_rank": 6100.0, "source_country_logp": -9.83053970336914, "target_country_logp": -15.22116470336914, "source_code_first_rank": 1944.5, "target_code_first_rank": 14803.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 15440.5, "target_country_rank": 25484.0, "source_country_logp": -15.261366844177246, "target_country_logp": -16.45667839050293, "source_code_first_rank": 2487.0, "target_code_first_rank": 13341.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 7.0, "target_country_rank": 3298.5, "source_country_logp": -5.612184047698975, "target_country_logp": -18.737184524536133, "source_code_first_rank": 9295.0, "target_code_first_rank": 42210.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 90816.0, "target_country_rank": 87251.5, "source_country_logp": -15.258779525756836, "target_country_logp": -15.133779525756836, "source_code_first_rank": 38775.0, "target_code_first_rank": 14781.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 192.5, "target_country_rank": 23540.5, "source_country_logp": -12.507408142089844, "target_country_logp": -21.913658142089844, "source_code_first_rank": 371.5, "target_code_first_rank": 8947.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 73.5, "target_country_rank": 55366.0, "source_country_logp": -22.375045776367188, "target_country_logp": -42.21684265136719, "source_code_first_rank": 4.0, "target_code_first_rank": 111.0}], "actual_source_country_rank": 73.5, "actual_target_country_rank": 55366.0, "array_path": "results/readout-arrays/eval-18-paraphrase.npz"}
{"case_id": "eval-19", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 7.0, "target_country_rank": 396.0, "source_country_logp": -3.7014427185058594, "target_country_logp": -8.67019271850586, "source_code_first_rank": 116478.0, "target_code_first_rank": 42605.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 10719.0, "target_country_rank": 41772.0, "source_country_logp": -13.713432312011719, "target_country_logp": -17.15874481201172, "source_code_first_rank": 32330.0, "target_code_first_rank": 39955.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 49.0, "source_country_logp": -3.7349631786346436, "target_country_logp": -13.234963417053223, "source_code_first_rank": 87130.5, "target_code_first_rank": 39763.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 9758.0, "target_country_rank": 9534.5, "source_country_logp": -29.13713836669922, "target_country_logp": -29.09026336669922, "source_code_first_rank": 94131.0, "target_code_first_rank": 59192.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 14.0, "target_country_rank": 141.5, "source_country_logp": -4.824333190917969, "target_country_logp": -7.605583190917969, "source_code_first_rank": 105645.0, "target_code_first_rank": 7248.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 7729.0, "target_country_rank": 37077.5, "source_country_logp": -12.694645881652832, "target_country_logp": -16.24152183532715, "source_code_first_rank": 9573.5, "target_code_first_rank": 12605.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.5, "target_country_rank": 15.5, "source_country_logp": -1.7268589735031128, "target_country_logp": -10.351859092712402, "source_code_first_rank": 63867.0, "target_code_first_rank": 35630.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 9621.0, "target_country_rank": 24608.5, "source_country_logp": -19.174915313720703, "target_country_logp": -20.987415313720703, "source_code_first_rank": 36345.5, "target_code_first_rank": 51628.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 14.5, "target_country_rank": 209.5, "source_country_logp": -4.537763595581055, "target_country_logp": -8.037763595581055, "source_code_first_rank": 75248.5, "target_code_first_rank": 5439.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 58634.0, "target_country_rank": 83593.0, "source_country_logp": -17.89377212524414, "target_country_logp": -19.36252212524414, "source_code_first_rank": 17346.5, "target_code_first_rank": 2188.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 16.5, "source_country_logp": -4.037631511688232, "target_country_logp": -12.037631034851074, "source_code_first_rank": 39072.5, "target_code_first_rank": 56052.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 105501.0, "target_country_rank": 89851.0, "source_country_logp": -15.32310676574707, "target_country_logp": -14.76060676574707, "source_code_first_rank": 4184.5, "target_code_first_rank": 38088.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 10.0, "target_country_rank": 362.0, "source_country_logp": -6.356386184692383, "target_country_logp": -14.356386184692383, "source_code_first_rank": 4135.0, "target_code_first_rank": 7115.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 383.0, "target_country_rank": 110644.5, "source_country_logp": -20.362754821777344, "target_country_logp": -36.847129821777344, "source_code_first_rank": 1.0, "target_code_first_rank": 67.0}], "actual_source_country_rank": 383.0, "actual_target_country_rank": 110644.5, "array_path": "results/readout-arrays/eval-19-main.npz"}
{"case_id": "eval-19", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 21.0, "target_country_rank": 471.5, "source_country_logp": -4.909214496612549, "target_country_logp": -9.06546401977539, "source_code_first_rank": 122094.0, "target_code_first_rank": 27099.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 14055.0, "target_country_rank": 50561.0, "source_country_logp": -14.315247535705566, "target_country_logp": -17.84356689453125, "source_code_first_rank": 34361.5, "target_code_first_rank": 36395.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 6.0, "target_country_rank": 52.0, "source_country_logp": -5.031610488891602, "target_country_logp": -13.031610488891602, "source_code_first_rank": 98878.0, "target_code_first_rank": 41088.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 20202.0, "target_country_rank": 16415.0, "source_country_logp": -31.848190307617188, "target_country_logp": -31.379440307617188, "source_code_first_rank": 97301.0, "target_code_first_rank": 46386.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 17.5, "target_country_rank": 102.5, "source_country_logp": -5.237242698669434, "target_country_logp": -7.424742698669434, "source_code_first_rank": 94909.5, "target_code_first_rank": 4852.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 10503.0, "target_country_rank": 39953.0, "source_country_logp": -12.749468803405762, "target_country_logp": -15.798296928405762, "source_code_first_rank": 19203.0, "target_code_first_rank": 28011.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 14.5, "source_country_logp": -2.2068169116973877, "target_country_logp": -9.706816673278809, "source_code_first_rank": 70109.5, "target_code_first_rank": 36726.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 6801.5, "target_country_rank": 12772.0, "source_country_logp": -20.46208381652832, "target_country_logp": -21.61833381652832, "source_code_first_rank": 57253.5, "target_code_first_rank": 59505.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 8.0, "target_country_rank": 170.0, "source_country_logp": -3.885162591934204, "target_country_logp": -7.822662353515625, "source_code_first_rank": 79465.0, "target_code_first_rank": 4292.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 30656.5, "target_country_rank": 71769.0, "source_country_logp": -17.245662689208984, "target_country_logp": -20.058162689208984, "source_code_first_rank": 14848.0, "target_code_first_rank": 3571.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 10.0, "source_country_logp": -4.272812843322754, "target_country_logp": -12.272812843322754, "source_code_first_rank": 47888.5, "target_code_first_rank": 60018.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 94561.5, "target_country_rank": 91061.5, "source_country_logp": -15.390909194946289, "target_country_logp": -15.265909194946289, "source_code_first_rank": 16256.0, "target_code_first_rank": 49453.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 12.0, "target_country_rank": 617.0, "source_country_logp": -6.538231372833252, "target_country_logp": -16.100730895996094, "source_code_first_rank": 4530.0, "target_code_first_rank": 9714.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 165.0, "target_country_rank": 49167.0, "source_country_logp": -27.5, "target_country_logp": -43.40625, "source_code_first_rank": 2.0, "target_code_first_rank": 71.0}], "actual_source_country_rank": 165.0, "actual_target_country_rank": 49167.0, "array_path": "results/readout-arrays/eval-19-paraphrase.npz"}
{"case_id": "eval-20", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 40.0, "target_country_rank": 434.0, "source_country_logp": -5.611531734466553, "target_country_logp": -9.267782211303711, "source_code_first_rank": 47249.5, "target_code_first_rank": 13268.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 11386.5, "target_country_rank": 44152.0, "source_country_logp": -13.908517837524414, "target_country_logp": -17.396799087524414, "source_code_first_rank": 29359.0, "target_code_first_rank": 37524.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 77.0, "source_country_logp": -4.92825984954834, "target_country_logp": -13.17825984954834, "source_code_first_rank": 37732.5, "target_code_first_rank": 21634.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 9333.0, "target_country_rank": 8566.0, "source_country_logp": -29.212495803833008, "target_country_logp": -29.040620803833008, "source_code_first_rank": 93865.5, "target_code_first_rank": 57988.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 15.5, "target_country_rank": 83.0, "source_country_logp": -5.513378620147705, "target_country_logp": -7.513378620147705, "source_code_first_rank": 68853.5, "target_code_first_rank": 3415.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 7962.0, "target_country_rank": 36855.5, "source_country_logp": -12.88714599609375, "target_country_logp": -16.38909912109375, "source_code_first_rank": 8913.5, "target_code_first_rank": 12167.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 111.0, "source_country_logp": -3.814192771911621, "target_country_logp": -10.189192771911621, "source_code_first_rank": 36091.5, "target_code_first_rank": 17997.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 8410.5, "target_country_rank": 20364.0, "source_country_logp": -18.662702560424805, "target_country_logp": -20.334577560424805, "source_code_first_rank": 33795.0, "target_code_first_rank": 43748.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 14.5, "target_country_rank": 60.5, "source_country_logp": -6.1496734619140625, "target_country_logp": -7.8684234619140625, "source_code_first_rank": 31232.0, "target_code_first_rank": 1403.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 64872.5, "target_country_rank": 85698.0, "source_country_logp": -18.204439163208008, "target_country_logp": -19.423189163208008, "source_code_first_rank": 16985.5, "target_code_first_rank": 2415.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.5, "target_country_rank": 32.5, "source_country_logp": -5.395864963531494, "target_country_logp": -11.520864486694336, "source_code_first_rank": 15771.5, "target_code_first_rank": 18615.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 102312.5, "target_country_rank": 86355.0, "source_country_logp": -15.264104843139648, "target_country_logp": -14.701604843139648, "source_code_first_rank": 2925.0, "target_code_first_rank": 33576.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 8.0, "target_country_rank": 339.5, "source_country_logp": -5.9938249588012695, "target_country_logp": -14.36882495880127, "source_code_first_rank": 2653.0, "target_code_first_rank": 2063.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 349.5, "target_country_rank": 90017.5, "source_country_logp": -21.069717407226562, "target_country_logp": -36.42518615722656, "source_code_first_rank": 1.0, "target_code_first_rank": 71.0}], "actual_source_country_rank": 349.5, "actual_target_country_rank": 90017.5, "array_path": "results/readout-arrays/eval-20-main.npz"}
{"case_id": "eval-20", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 53.5, "target_country_rank": 579.5, "source_country_logp": -6.705186367034912, "target_country_logp": -10.20518684387207, "source_code_first_rank": 59394.5, "target_code_first_rank": 13514.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 14565.0, "target_country_rank": 51103.0, "source_country_logp": -14.478291511535645, "target_country_logp": -17.97829246520996, "source_code_first_rank": 34264.0, "target_code_first_rank": 36873.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 4.0, "target_country_rank": 90.0, "source_country_logp": -5.426670074462891, "target_country_logp": -12.80167007446289, "source_code_first_rank": 44315.0, "target_code_first_rank": 20870.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 19096.5, "target_country_rank": 14603.5, "source_country_logp": -31.776926040649414, "target_country_logp": -31.183176040649414, "source_code_first_rank": 98874.5, "target_code_first_rank": 47551.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 13.0, "target_country_rank": 67.0, "source_country_logp": -5.740644454956055, "target_country_logp": -7.584394454956055, "source_code_first_rank": 69606.0, "target_code_first_rank": 4082.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 10119.0, "target_country_rank": 38182.5, "source_country_logp": -12.701801300048828, "target_country_logp": -15.686176300048828, "source_code_first_rank": 19208.5, "target_code_first_rank": 31436.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 8.5, "target_country_rank": 131.5, "source_country_logp": -4.725548267364502, "target_country_logp": -10.85054874420166, "source_code_first_rank": 38546.5, "target_code_first_rank": 20868.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 5772.0, "target_country_rank": 10431.5, "source_country_logp": -20.707136154174805, "target_country_logp": -21.785261154174805, "source_code_first_rank": 57031.5, "target_code_first_rank": 58128.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 7.0, "target_country_rank": 43.0, "source_country_logp": -5.53542423248291, "target_country_logp": -7.41042423248291, "source_code_first_rank": 41684.0, "target_code_first_rank": 1676.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 30043.5, "target_country_rank": 70507.0, "source_country_logp": -17.008726119995117, "target_country_logp": -19.805601119995117, "source_code_first_rank": 16962.5, "target_code_first_rank": 4195.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 21.5, "source_country_logp": -5.884310722351074, "target_country_logp": -12.384310722351074, "source_code_first_rank": 17769.0, "target_code_first_rank": 18692.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 97140.0, "target_country_rank": 88966.0, "source_country_logp": -15.286269187927246, "target_country_logp": -15.036269187927246, "source_code_first_rank": 16079.5, "target_code_first_rank": 49639.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 14.0, "target_country_rank": 601.5, "source_country_logp": -6.490467548370361, "target_country_logp": -15.05296802520752, "source_code_first_rank": 3033.0, "target_code_first_rank": 2282.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 79.5, "target_country_rank": 32520.0, "source_country_logp": -24.125022888183594, "target_country_logp": -40.593772888183594, "source_code_first_rank": 2.0, "target_code_first_rank": 60.5}], "actual_source_country_rank": 79.5, "actual_target_country_rank": 32520.0, "array_path": "results/readout-arrays/eval-20-paraphrase.npz"}
{"case_id": "eval-21", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 5.5, "target_country_rank": 1869.0, "source_country_logp": -3.724733829498291, "target_country_logp": -10.818483352661133, "source_code_first_rank": 17538.0, "target_code_first_rank": 16676.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 40046.5, "target_country_rank": 4304.0, "source_country_logp": -17.05103302001953, "target_country_logp": -11.941658020019531, "source_code_first_rank": 40474.0, "target_code_first_rank": 64040.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.5, "target_country_rank": 7957.5, "source_country_logp": -7.251553058624268, "target_country_logp": -34.62655258178711, "source_code_first_rank": 91456.0, "target_code_first_rank": 131256.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 9432.0, "target_country_rank": 1702.0, "source_country_logp": -29.047372817993164, "target_country_logp": -25.793466567993164, "source_code_first_rank": 59290.5, "target_code_first_rank": 30023.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 8.5, "target_country_rank": 762.0, "source_country_logp": -5.664945602416992, "target_country_logp": -11.602445602416992, "source_code_first_rank": 10110.0, "target_code_first_rank": 27273.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 36698.5, "target_country_rank": 6781.5, "source_country_logp": -16.207738876342773, "target_country_logp": -12.453832626342773, "source_code_first_rank": 13513.5, "target_code_first_rank": 17854.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 14084.0, "source_country_logp": -5.008310794830322, "target_country_logp": -31.195810317993164, "source_code_first_rank": 86166.5, "target_code_first_rank": 78914.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 23864.0, "target_country_rank": 3018.0, "source_country_logp": -20.320289611816406, "target_country_logp": -16.679664611816406, "source_code_first_rank": 54778.0, "target_code_first_rank": 5137.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 9.0, "target_country_rank": 407.5, "source_country_logp": -4.222443103790283, "target_country_logp": -9.159943580627441, "source_code_first_rank": 6282.0, "target_code_first_rank": 17523.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 86584.0, "target_country_rank": 20803.0, "source_country_logp": -19.56589126586914, "target_country_logp": -15.019017219543457, "source_code_first_rank": 2488.5, "target_code_first_rank": 4159.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 13355.0, "source_country_logp": -5.256354808807373, "target_country_logp": -34.69385528564453, "source_code_first_rank": 94932.0, "target_code_first_rank": 68345.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 88292.0, "target_country_rank": 97275.0, "source_country_logp": -14.677213668823242, "target_country_logp": -14.989713668823242, "source_code_first_rank": 36475.0, "target_code_first_rank": 6206.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 12.0, "target_country_rank": 5414.5, "source_country_logp": -6.68635368347168, "target_country_logp": -18.37385368347168, "source_code_first_rank": 163.0, "target_code_first_rank": 107823.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 53.5, "target_country_rank": 76035.0, "source_country_logp": -23.000022888183594, "target_country_logp": -44.898460388183594, "source_code_first_rank": 1.0, "target_code_first_rank": 838.0}], "actual_source_country_rank": 53.5, "actual_target_country_rank": 76035.0, "array_path": "results/readout-arrays/eval-21-main.npz"}
{"case_id": "eval-21", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 9.0, "target_country_rank": 3870.0, "source_country_logp": -4.314038276672363, "target_country_logp": -12.306225776672363, "source_code_first_rank": 12740.5, "target_code_first_rank": 18810.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 50312.5, "target_country_rank": 3862.0, "source_country_logp": -17.766189575195312, "target_country_logp": -11.719315528869629, "source_code_first_rank": 37250.5, "target_code_first_rank": 60145.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.5, "target_country_rank": 9254.0, "source_country_logp": -8.25059986114502, "target_country_logp": -36.0630989074707, "source_code_first_rank": 103601.5, "target_code_first_rank": 136010.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 15658.0, "target_country_rank": 3038.0, "source_country_logp": -31.310033798217773, "target_country_logp": -27.872533798217773, "source_code_first_rank": 47346.0, "target_code_first_rank": 31439.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 9.0, "target_country_rank": 1025.5, "source_country_logp": -6.487523078918457, "target_country_logp": -12.440648078918457, "source_code_first_rank": 10112.0, "target_code_first_rank": 31964.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 38058.0, "target_country_rank": 7618.0, "source_country_logp": -15.672316551208496, "target_country_logp": -12.180129051208496, "source_code_first_rank": 27680.0, "target_code_first_rank": 30122.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 15501.5, "source_country_logp": -5.505501747131348, "target_country_logp": -31.34925079345703, "source_code_first_rank": 92708.0, "target_code_first_rank": 94338.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 12465.0, "target_country_rank": 1608.0, "source_country_logp": -21.93686866760254, "target_country_logp": -18.35093116760254, "source_code_first_rank": 59462.0, "target_code_first_rank": 10077.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 9.0, "target_country_rank": 357.0, "source_country_logp": -4.883947849273682, "target_country_logp": -9.383947372436523, "source_code_first_rank": 5358.5, "target_code_first_rank": 21372.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 71376.5, "target_country_rank": 27317.5, "source_country_logp": -20.06340789794922, "target_country_logp": -16.96965789794922, "source_code_first_rank": 3733.0, "target_code_first_rank": 6426.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 14394.0, "source_country_logp": -5.754614353179932, "target_country_logp": -35.754615783691406, "source_code_first_rank": 99314.0, "target_code_first_rank": 74343.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 90023.0, "target_country_rank": 98008.5, "source_country_logp": -15.112394332885742, "target_country_logp": -15.362394332885742, "source_code_first_rank": 49963.0, "target_code_first_rank": 13281.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 13.5, "target_country_rank": 5207.5, "source_country_logp": -6.725768566131592, "target_country_logp": -18.60076904296875, "source_code_first_rank": 106.5, "target_code_first_rank": 122210.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 79.5, "target_country_rank": 11591.5, "source_country_logp": -24.750003814697266, "target_country_logp": -38.312503814697266, "source_code_first_rank": 4.5, "target_code_first_rank": 485.5}], "actual_source_country_rank": 79.5, "actual_target_country_rank": 11591.5, "array_path": "results/readout-arrays/eval-21-paraphrase.npz"}
{"case_id": "eval-22", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 28.0, "target_country_rank": 585.5, "source_country_logp": -5.84706974029541, "target_country_logp": -10.15956974029541, "source_code_first_rank": 1626.0, "target_code_first_rank": 14742.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 42930.5, "target_country_rank": 4875.0, "source_country_logp": -17.28496742248535, "target_country_logp": -12.238093376159668, "source_code_first_rank": 42313.0, "target_code_first_rank": 61739.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 6691.5, "source_country_logp": -1.2219603061676025, "target_country_logp": -18.221960067749023, "source_code_first_rank": 102890.5, "target_code_first_rank": 113650.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 7490.0, "target_country_rank": 1329.0, "source_country_logp": -28.844058990478516, "target_country_logp": -25.523746490478516, "source_code_first_rank": 59271.0, "target_code_first_rank": 28735.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 13.5, "target_country_rank": 457.5, "source_country_logp": -4.774291038513184, "target_country_logp": -9.336791038513184, "source_code_first_rank": 3027.5, "target_code_first_rank": 10534.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 32262.5, "target_country_rank": 6531.0, "source_country_logp": -15.931488990783691, "target_country_logp": -12.458832740783691, "source_code_first_rank": 13319.0, "target_code_first_rank": 15080.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 7513.5, "source_country_logp": -1.3968424797058105, "target_country_logp": -17.02184295654297, "source_code_first_rank": 54536.0, "target_code_first_rank": 44886.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 22625.5, "target_country_rank": 2870.5, "source_country_logp": -19.529203414916992, "target_country_logp": -16.029203414916992, "source_code_first_rank": 47686.0, "target_code_first_rank": 4634.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 7.0, "target_country_rank": 429.0, "source_country_logp": -3.3546948432922363, "target_country_logp": -8.760944366455078, "source_code_first_rank": 1147.0, "target_code_first_rank": 9079.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 81834.5, "target_country_rank": 18179.5, "source_country_logp": -18.945112228393555, "target_country_logp": -14.425580978393555, "source_code_first_rank": 2815.5, "target_code_first_rank": 4401.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 2424.5, "source_country_logp": -1.5588669776916504, "target_country_logp": -24.246366500854492, "source_code_first_rank": 78828.0, "target_code_first_rank": 47960.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 75298.0, "target_country_rank": 86439.5, "source_country_logp": -14.256781578063965, "target_country_logp": -14.663031578063965, "source_code_first_rank": 32036.0, "target_code_first_rank": 4987.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 8.0, "target_country_rank": 1592.0, "source_country_logp": -6.852316856384277, "target_country_logp": -17.227317810058594, "source_code_first_rank": 421.0, "target_code_first_rank": 56860.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 197.0, "target_country_rank": 122382.5, "source_country_logp": -23.823619842529297, "target_country_logp": -42.9329948425293, "source_code_first_rank": 1.0, "target_code_first_rank": 612.0}], "actual_source_country_rank": 197.0, "actual_target_country_rank": 122382.5, "array_path": "results/readout-arrays/eval-22-main.npz"}
{"case_id": "eval-22", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 123.5, "target_country_rank": 873.5, "source_country_logp": -7.743922710418701, "target_country_logp": -10.61892318725586, "source_code_first_rank": 1078.0, "target_code_first_rank": 29927.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 51591.0, "target_country_rank": 3586.5, "source_country_logp": -17.981586456298828, "target_country_logp": -11.731586456298828, "source_code_first_rank": 38937.5, "target_code_first_rank": 61401.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 6.0, "target_country_rank": 3315.0, "source_country_logp": -3.234523296356201, "target_country_logp": -15.10952377319336, "source_code_first_rank": 90200.5, "target_code_first_rank": 115283.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 13750.0, "target_country_rank": 2428.0, "source_country_logp": -31.24440574645996, "target_country_logp": -27.61940574645996, "source_code_first_rank": 46774.5, "target_code_first_rank": 27427.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 13.0, "target_country_rank": 480.0, "source_country_logp": -4.815619468688965, "target_country_logp": -9.003119468688965, "source_code_first_rank": 4605.0, "target_code_first_rank": 15765.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 34308.0, "target_country_rank": 6837.5, "source_country_logp": -15.32572078704834, "target_country_logp": -11.94290828704834, "source_code_first_rank": 32150.0, "target_code_first_rank": 27220.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 4.0, "target_country_rank": 4510.0, "source_country_logp": -2.9978690147399902, "target_country_logp": -15.560369491577148, "source_code_first_rank": 53909.5, "target_code_first_rank": 69966.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 9512.0, "target_country_rank": 1251.0, "source_country_logp": -21.483642578125, "target_country_logp": -17.913330078125, "source_code_first_rank": 57696.5, "target_code_first_rank": 9655.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 5.0, "target_country_rank": 403.0, "source_country_logp": -3.623922824859619, "target_country_logp": -8.342672348022461, "source_code_first_rank": 1261.5, "target_code_first_rank": 14385.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 62507.5, "target_country_rank": 25701.0, "source_country_logp": -19.24683952331543, "target_country_logp": -16.51246452331543, "source_code_first_rank": 3604.0, "target_code_first_rank": 4445.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 1236.5, "source_country_logp": -2.946101665496826, "target_country_logp": -22.196102142333984, "source_code_first_rank": 74432.0, "target_code_first_rank": 60689.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 72595.0, "target_country_rank": 88732.5, "source_country_logp": -14.331995964050293, "target_country_logp": -14.863245964050293, "source_code_first_rank": 40535.5, "target_code_first_rank": 8164.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 24.0, "target_country_rank": 1441.5, "source_country_logp": -8.594003677368164, "target_country_logp": -16.406503677368164, "source_code_first_rank": 658.0, "target_code_first_rank": 45281.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 2280.5, "target_country_rank": 35271.0, "source_country_logp": -30.312501907348633, "target_country_logp": -38.437503814697266, "source_code_first_rank": 14.0, "target_code_first_rank": 484.0}], "actual_source_country_rank": 2280.5, "actual_target_country_rank": 35271.0, "array_path": "results/readout-arrays/eval-22-paraphrase.npz"}
{"case_id": "eval-23", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 15277.0, "source_country_logp": -1.3377704620361328, "target_country_logp": -16.419801712036133, "source_code_first_rank": 13513.5, "target_code_first_rank": 23133.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 4910.0, "target_country_rank": 45330.0, "source_country_logp": -12.197721481323242, "target_country_logp": -17.432096481323242, "source_code_first_rank": 64492.5, "target_code_first_rank": 3977.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 6.5, "target_country_rank": 1156.5, "source_country_logp": -4.337258338928223, "target_country_logp": -22.212257385253906, "source_code_first_rank": 92366.0, "target_code_first_rank": 111882.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 1319.0, "target_country_rank": 6229.0, "source_country_logp": -25.06023406982422, "target_country_logp": -27.93059539794922, "source_code_first_rank": 28727.5, "target_code_first_rank": 94517.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 4.0, "target_country_rank": 6246.5, "source_country_logp": -2.7392311096191406, "target_country_logp": -13.60837173461914, "source_code_first_rank": 20702.5, "target_code_first_rank": 30280.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 6942.5, "target_country_rank": 15390.0, "source_country_logp": -12.623908042907715, "target_country_logp": -14.202033042907715, "source_code_first_rank": 16733.5, "target_code_first_rank": 5840.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 9.0, "target_country_rank": 1091.0, "source_country_logp": -4.3142571449279785, "target_country_logp": -15.18925666809082, "source_code_first_rank": 50724.0, "target_code_first_rank": 102125.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 2530.0, "target_country_rank": 10320.0, "source_country_logp": -16.326250076293945, "target_country_logp": -18.584062576293945, "source_code_first_rank": 4584.5, "target_code_first_rank": 66856.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 5360.5, "source_country_logp": -1.44800865650177, "target_country_logp": -13.17066478729248, "source_code_first_rank": 11762.0, "target_code_first_rank": 9287.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 16881.5, "target_country_rank": 25301.0, "source_country_logp": -14.305597305297852, "target_country_logp": -15.231378555297852, "source_code_first_rank": 3868.5, "target_code_first_rank": 2346.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 8.0, "target_country_rank": 644.5, "source_country_logp": -5.755030632019043, "target_country_logp": -21.31753158569336, "source_code_first_rank": 28305.0, "target_code_first_rank": 54991.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 85457.0, "target_country_rank": 83955.0, "source_country_logp": -14.637526512145996, "target_country_logp": -14.575026512145996, "source_code_first_rank": 4688.5, "target_code_first_rank": 48638.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 16.5, "target_country_rank": 2806.0, "source_country_logp": -8.30475902557373, "target_country_logp": -17.054759979248047, "source_code_first_rank": 12052.0, "target_code_first_rank": 65145.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 382.0, "target_country_rank": 81952.0, "source_country_logp": -17.980422973632812, "target_country_logp": -33.08979797363281, "source_code_first_rank": 1.0, "target_code_first_rank": 65.0}], "actual_source_country_rank": 382.0, "actual_target_country_rank": 81952.0, "array_path": "results/readout-arrays/eval-23-main.npz"}
{"case_id": "eval-23", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 23128.5, "source_country_logp": -1.360863447189331, "target_country_logp": -17.212425231933594, "source_code_first_rank": 19070.0, "target_code_first_rank": 32086.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 3882.0, "target_country_rank": 37776.0, "source_country_logp": -11.771900177001953, "target_country_logp": -16.850025177001953, "source_code_first_rank": 65280.0, "target_code_first_rank": 2619.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 7.5, "target_country_rank": 2145.5, "source_country_logp": -4.886598587036133, "target_country_logp": -23.511598587036133, "source_code_first_rank": 99300.0, "target_code_first_rank": 111161.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 2291.0, "target_country_rank": 7932.5, "source_country_logp": -27.845130920410156, "target_country_logp": -30.415443420410156, "source_code_first_rank": 28340.0, "target_code_first_rank": 83464.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 2.5, "target_country_rank": 8763.5, "source_country_logp": -2.531123638153076, "target_country_logp": -14.105831146240234, "source_code_first_rank": 20153.0, "target_code_first_rank": 30794.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 8899.0, "target_country_rank": 17378.5, "source_country_logp": -12.39892578125, "target_country_logp": -13.68798828125, "source_code_first_rank": 29286.5, "target_code_first_rank": 3756.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 8.0, "target_country_rank": 1161.5, "source_country_logp": -4.618989944458008, "target_country_logp": -14.868989944458008, "source_code_first_rank": 63054.0, "target_code_first_rank": 101165.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 1252.5, "target_country_rank": 5266.0, "source_country_logp": -18.499467849731445, "target_country_logp": -20.995561599731445, "source_code_first_rank": 9578.5, "target_code_first_rank": 56669.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 4146.0, "source_country_logp": -1.6351271867752075, "target_country_logp": -12.572627067565918, "source_code_first_rank": 13445.5, "target_code_first_rank": 11090.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 31262.5, "target_country_rank": 26425.0, "source_country_logp": -17.128210067749023, "target_country_logp": -16.690710067749023, "source_code_first_rank": 4289.5, "target_code_first_rank": 3340.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 8.0, "target_country_rank": 489.5, "source_country_logp": -6.546471118927002, "target_country_logp": -20.546470642089844, "source_code_first_rank": 29558.5, "target_code_first_rank": 56922.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 92865.5, "target_country_rank": 101763.0, "source_country_logp": -15.254343032836914, "target_country_logp": -15.504343032836914, "source_code_first_rank": 11409.5, "target_code_first_rank": 77857.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 21.0, "target_country_rank": 3634.0, "source_country_logp": -7.851508140563965, "target_country_logp": -16.91400718688965, "source_code_first_rank": 11862.0, "target_code_first_rank": 75756.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 192.5, "target_country_rank": 5983.0, "source_country_logp": -29.125, "target_country_logp": -39.3125, "source_code_first_rank": 28.0, "target_code_first_rank": 255.5}], "actual_source_country_rank": 192.5, "actual_target_country_rank": 5983.0, "array_path": "results/readout-arrays/eval-23-paraphrase.npz"}
{"case_id": "eval-24", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 27.0, "target_country_rank": 3353.0, "source_country_logp": -5.208775520324707, "target_country_logp": -12.255650520324707, "source_code_first_rank": 56074.0, "target_code_first_rank": 63198.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 4590.5, "target_country_rank": 46024.5, "source_country_logp": -12.050694465637207, "target_country_logp": -17.42569351196289, "source_code_first_rank": 62505.0, "target_code_first_rank": 3766.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 31.0, "target_country_rank": 7571.5, "source_country_logp": -7.661301612854004, "target_country_logp": -26.536300659179688, "source_code_first_rank": 50436.0, "target_code_first_rank": 57541.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 1424.0, "target_country_rank": 5972.0, "source_country_logp": -25.460386276245117, "target_country_logp": -28.157651901245117, "source_code_first_rank": 28127.5, "target_code_first_rank": 95099.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 63.0, "target_country_rank": 6142.5, "source_country_logp": -6.931648254394531, "target_country_logp": -13.650398254394531, "source_code_first_rank": 63216.5, "target_code_first_rank": 90780.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 6191.0, "target_country_rank": 16022.5, "source_country_logp": -12.355365753173828, "target_country_logp": -14.214740753173828, "source_code_first_rank": 17318.0, "target_code_first_rank": 6836.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 75.0, "target_country_rank": 10620.0, "source_country_logp": -9.954638481140137, "target_country_logp": -25.92338752746582, "source_code_first_rank": 26871.0, "target_code_first_rank": 68640.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 2577.0, "target_country_rank": 11345.0, "source_country_logp": -16.564224243164062, "target_country_logp": -19.017349243164062, "source_code_first_rank": 4911.5, "target_code_first_rank": 70250.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 35.0, "target_country_rank": 9823.0, "source_country_logp": -5.53804349899292, "target_country_logp": -13.612262725830078, "source_code_first_rank": 27110.0, "target_code_first_rank": 44301.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 18237.5, "target_country_rank": 31697.0, "source_country_logp": -14.367311477661133, "target_country_logp": -15.691530227661133, "source_code_first_rank": 3869.5, "target_code_first_rank": 2947.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 43.0, "target_country_rank": 5202.5, "source_country_logp": -8.381811141967773, "target_country_logp": -22.631811141967773, "source_code_first_rank": 27730.0, "target_code_first_rank": 48651.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 85843.5, "target_country_rank": 87530.0, "source_country_logp": -14.715272903442383, "target_country_logp": -14.777772903442383, "source_code_first_rank": 4076.0, "target_code_first_rank": 47323.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 27.5, "target_country_rank": 3219.5, "source_country_logp": -9.9130277633667, "target_country_logp": -18.975528717041016, "source_code_first_rank": 39065.0, "target_code_first_rank": 55671.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 575.0, "target_country_rank": 101152.0, "source_country_logp": -15.801287651062012, "target_country_logp": -30.207536697387695, "source_code_first_rank": 24.5, "target_code_first_rank": 45.0}], "actual_source_country_rank": 575.0, "actual_target_country_rank": 101152.0, "array_path": "results/readout-arrays/eval-24-main.npz"}
{"case_id": "eval-24", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 24.0, "target_country_rank": 4060.0, "source_country_logp": -4.976551055908203, "target_country_logp": -12.460926055908203, "source_code_first_rank": 69786.5, "target_code_first_rank": 70713.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 4045.5, "target_country_rank": 38002.5, "source_country_logp": -11.843804359436035, "target_country_logp": -16.85943031311035, "source_code_first_rank": 62977.0, "target_code_first_rank": 2533.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 30.0, "target_country_rank": 6305.0, "source_country_logp": -7.296819686889648, "target_country_logp": -25.67181968688965, "source_code_first_rank": 55867.0, "target_code_first_rank": 61406.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 2435.5, "target_country_rank": 6945.0, "source_country_logp": -28.133756637573242, "target_country_logp": -30.344694137573242, "source_code_first_rank": 27187.5, "target_code_first_rank": 85627.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 70.0, "target_country_rank": 8113.5, "source_country_logp": -7.029338836669922, "target_country_logp": -14.222698211669922, "source_code_first_rank": 70435.0, "target_code_first_rank": 83915.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 8686.5, "target_country_rank": 18114.5, "source_country_logp": -12.437777519226074, "target_country_logp": -13.883090019226074, "source_code_first_rank": 31729.5, "target_code_first_rank": 4393.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 79.0, "target_country_rank": 12340.0, "source_country_logp": -11.194541931152344, "target_country_logp": -27.757041931152344, "source_code_first_rank": 39001.0, "target_code_first_rank": 72168.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 1628.0, "target_country_rank": 5695.0, "source_country_logp": -17.869850158691406, "target_country_logp": -19.987037658691406, "source_code_first_rank": 9788.0, "target_code_first_rank": 59788.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 25.0, "target_country_rank": 7906.0, "source_country_logp": -5.271498680114746, "target_country_logp": -13.195326805114746, "source_code_first_rank": 36590.0, "target_code_first_rank": 48229.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 27346.5, "target_country_rank": 29719.5, "source_country_logp": -16.82877540588379, "target_country_logp": -17.04752540588379, "source_code_first_rank": 4673.5, "target_code_first_rank": 4389.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 25.5, "target_country_rank": 4655.0, "source_country_logp": -9.446887016296387, "target_country_logp": -24.44688606262207, "source_code_first_rank": 29221.0, "target_code_first_rank": 50828.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 93168.5, "target_country_rank": 105356.0, "source_country_logp": -14.971680641174316, "target_country_logp": -15.346680641174316, "source_code_first_rank": 10344.5, "target_code_first_rank": 77861.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 31.5, "target_country_rank": 3521.5, "source_country_logp": -9.474994659423828, "target_country_logp": -18.537494659423828, "source_code_first_rank": 44090.5, "target_code_first_rank": 72550.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 546.0, "target_country_rank": 23160.5, "source_country_logp": -27.6250057220459, "target_country_logp": -38.03125762939453, "source_code_first_rank": 122.0, "target_code_first_rank": 84.5}], "actual_source_country_rank": 546.0, "actual_target_country_rank": 23160.5, "array_path": "results/readout-arrays/eval-24-paraphrase.npz"}
{"case_id": "eval-25", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 11.0, "target_country_rank": 14.0, "source_country_logp": -4.272421836853027, "target_country_logp": -4.584921836853027, "source_code_first_rank": 20058.5, "target_code_first_rank": 54133.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 44954.5, "target_country_rank": 7886.5, "source_country_logp": -17.35651969909668, "target_country_logp": -13.05183219909668, "source_code_first_rank": 3184.5, "target_code_first_rank": 92226.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 49.0, "source_country_logp": -5.259810447692871, "target_country_logp": -20.384811401367188, "source_code_first_rank": 113060.5, "target_code_first_rank": 92180.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 5610.0, "target_country_rank": 1203.0, "source_country_logp": -27.920984268188477, "target_country_logp": -25.051843643188477, "source_code_first_rank": 94592.5, "target_code_first_rank": 65523.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 48.5, "target_country_rank": 10.0, "source_country_logp": -6.616104602813721, "target_country_logp": -4.834854602813721, "source_code_first_rank": 28425.0, "target_code_first_rank": 38168.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 16510.5, "target_country_rank": 6845.0, "source_country_logp": -14.270487785339355, "target_country_logp": -12.524394035339355, "source_code_first_rank": 6788.0, "target_code_first_rank": 22004.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 25.5, "source_country_logp": -4.343735218048096, "target_country_logp": -14.468735694885254, "source_code_first_rank": 115207.0, "target_code_first_rank": 59510.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 9837.0, "target_country_rank": 1548.5, "source_country_logp": -18.141357421875, "target_country_logp": -15.242918968200684, "source_code_first_rank": 70873.5, "target_code_first_rank": 21606.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 28.0, "target_country_rank": 40.0, "source_country_logp": -6.008292198181152, "target_country_logp": -6.352042198181152, "source_code_first_rank": 38380.0, "target_code_first_rank": 27808.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 22893.0, "target_country_rank": 9777.0, "source_country_logp": -15.265724182128906, "target_country_logp": -13.500099182128906, "source_code_first_rank": 2966.0, "target_code_first_rank": 4345.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 44.5, "source_country_logp": -4.865888595581055, "target_country_logp": -17.865888595581055, "source_code_first_rank": 103107.0, "target_code_first_rank": 57182.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 82554.5, "target_country_rank": 31884.0, "source_country_logp": -14.473650932312012, "target_country_logp": -12.598650932312012, "source_code_first_rank": 48961.0, "target_code_first_rank": 7280.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 18.5, "target_country_rank": 217.0, "source_country_logp": -7.409341812133789, "target_country_logp": -12.534341812133789, "source_code_first_rank": 11797.5, "target_code_first_rank": 42678.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 107.0, "target_country_rank": 12310.5, "source_country_logp": -23.12531280517578, "target_country_logp": -35.09406280517578, "source_code_first_rank": 1.0, "target_code_first_rank": 131.5}], "actual_source_country_rank": 107.0, "actual_target_country_rank": 12310.5, "array_path": "results/readout-arrays/eval-25-main.npz"}
{"case_id": "eval-25", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 9.5, "target_country_rank": 9.5, "source_country_logp": -4.406931400299072, "target_country_logp": -4.406931400299072, "source_code_first_rank": 31370.0, "target_code_first_rank": 54963.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 38883.0, "target_country_rank": 9296.0, "source_country_logp": -16.947538375854492, "target_country_logp": -13.408475875854492, "source_code_first_rank": 2145.5, "target_code_first_rank": 87528.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 40.0, "source_country_logp": -5.956113815307617, "target_country_logp": -19.081113815307617, "source_code_first_rank": 122926.5, "target_code_first_rank": 99022.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 8356.0, "target_country_rank": 2288.0, "source_country_logp": -30.283945083618164, "target_country_logp": -27.596445083618164, "source_code_first_rank": 86104.0, "target_code_first_rank": 67824.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 57.5, "target_country_rank": 10.0, "source_country_logp": -7.30253791809082, "target_country_logp": -5.02128791809082, "source_code_first_rank": 35394.0, "target_code_first_rank": 32655.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 19896.5, "target_country_rank": 5103.0, "source_country_logp": -14.079742431640625, "target_country_logp": -11.493804931640625, "source_code_first_rank": 4092.0, "target_code_first_rank": 14663.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 25.0, "source_country_logp": -4.714981555938721, "target_country_logp": -13.464982032775879, "source_code_first_rank": 120575.0, "target_code_first_rank": 64104.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 6731.5, "target_country_rank": 1738.0, "source_country_logp": -20.7501277923584, "target_country_logp": -18.4376277923584, "source_code_first_rank": 56105.5, "target_code_first_rank": 30817.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 17.5, "target_country_rank": 23.0, "source_country_logp": -6.00343132019043, "target_country_logp": -6.25343132019043, "source_code_first_rank": 46722.5, "target_code_first_rank": 29712.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 24881.5, "target_country_rank": 7794.0, "source_country_logp": -16.829570770263672, "target_country_logp": -14.341289520263672, "source_code_first_rank": 3447.5, "target_code_first_rank": 2928.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 36.5, "source_country_logp": -4.926703453063965, "target_country_logp": -17.05170440673828, "source_code_first_rank": 107931.0, "target_code_first_rank": 63300.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 101491.5, "target_country_rank": 35494.0, "source_country_logp": -15.386167526245117, "target_country_logp": -13.198667526245117, "source_code_first_rank": 76570.5, "target_code_first_rank": 11252.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 18.0, "target_country_rank": 237.0, "source_country_logp": -6.850130081176758, "target_country_logp": -12.912630081176758, "source_code_first_rank": 8130.0, "target_code_first_rank": 41261.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 26.0, "target_country_rank": 997.5, "source_country_logp": -17.504547119140625, "target_country_logp": -27.692047119140625, "source_code_first_rank": 2.0, "target_code_first_rank": 156.0}], "actual_source_country_rank": 26.0, "actual_target_country_rank": 997.5, "array_path": "results/readout-arrays/eval-25-paraphrase.npz"}
{"case_id": "eval-26", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 5.0, "target_country_rank": 287.5, "source_country_logp": -3.236748695373535, "target_country_logp": -8.049248695373535, "source_code_first_rank": 34000.5, "target_code_first_rank": 58418.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 45350.0, "target_country_rank": 9204.0, "source_country_logp": -17.458206176757812, "target_country_logp": -13.426957130432129, "source_code_first_rank": 3667.0, "target_code_first_rank": 94254.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 113.0, "source_country_logp": -1.440546989440918, "target_country_logp": -13.940546989440918, "source_code_first_rank": 113387.5, "target_code_first_rank": 116496.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 7178.5, "target_country_rank": 1529.5, "source_country_logp": -27.54335594177246, "target_country_logp": -24.70351219177246, "source_code_first_rank": 95672.0, "target_code_first_rank": 65305.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 38.0, "target_country_rank": 133.0, "source_country_logp": -5.727634906768799, "target_country_logp": -7.102634906768799, "source_code_first_rank": 37247.0, "target_code_first_rank": 20071.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 15320.0, "target_country_rank": 7401.0, "source_country_logp": -14.263581275939941, "target_country_logp": -12.796784400939941, "source_code_first_rank": 5359.5, "target_code_first_rank": 22900.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 4.0, "target_country_rank": 80.5, "source_country_logp": -2.2928555011749268, "target_country_logp": -8.667855262756348, "source_code_first_rank": 103614.0, "target_code_first_rank": 102648.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 11512.5, "target_country_rank": 2071.0, "source_country_logp": -18.972057342529297, "target_country_logp": -16.167369842529297, "source_code_first_rank": 65724.5, "target_code_first_rank": 19638.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 87.0, "target_country_rank": 336.0, "source_country_logp": -7.3091278076171875, "target_country_logp": -9.121627807617188, "source_code_first_rank": 24352.5, "target_code_first_rank": 22628.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 19571.5, "target_country_rank": 7576.0, "source_country_logp": -14.605905532836914, "target_country_logp": -12.760202407836914, "source_code_first_rank": 2794.5, "target_code_first_rank": 4987.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 108.5, "source_country_logp": -2.3931021690368652, "target_country_logp": -11.143102645874023, "source_code_first_rank": 71454.0, "target_code_first_rank": 97522.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 67198.5, "target_country_rank": 21206.0, "source_country_logp": -14.087104797363281, "target_country_logp": -12.180854797363281, "source_code_first_rank": 52443.0, "target_code_first_rank": 8158.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 7.0, "target_country_rank": 180.5, "source_country_logp": -6.716705322265625, "target_country_logp": -13.466705322265625, "source_code_first_rank": 11314.5, "target_code_first_rank": 61588.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 179.0, "target_country_rank": 16307.0, "source_country_logp": -23.625263214111328, "target_country_logp": -34.32838821411133, "source_code_first_rank": 1.0, "target_code_first_rank": 93.0}], "actual_source_country_rank": 179.0, "actual_target_country_rank": 16307.0, "array_path": "results/readout-arrays/eval-26-main.npz"}
{"case_id": "eval-26", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 5.0, "target_country_rank": 569.5, "source_country_logp": -3.4557313919067383, "target_country_logp": -8.955731391906738, "source_code_first_rank": 37961.0, "target_code_first_rank": 51951.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 38158.0, "target_country_rank": 9593.5, "source_country_logp": -17.00303077697754, "target_country_logp": -13.588969230651855, "source_code_first_rank": 2689.5, "target_code_first_rank": 83913.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 103.0, "source_country_logp": -1.5023002624511719, "target_country_logp": -15.377300262451172, "source_code_first_rank": 117768.5, "target_code_first_rank": 117021.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 8210.0, "target_country_rank": 2311.5, "source_country_logp": -30.034759521484375, "target_country_logp": -27.378509521484375, "source_code_first_rank": 86819.5, "target_code_first_rank": 65306.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 22.0, "target_country_rank": 172.5, "source_country_logp": -5.583901405334473, "target_country_logp": -7.583901405334473, "source_code_first_rank": 42508.5, "target_code_first_rank": 20903.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 20048.5, "target_country_rank": 5557.5, "source_country_logp": -14.056438446044922, "target_country_logp": -11.595500946044922, "source_code_first_rank": 3677.5, "target_code_first_rank": 13089.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 5.5, "target_country_rank": 71.0, "source_country_logp": -2.56833553314209, "target_country_logp": -8.69333553314209, "source_code_first_rank": 111218.5, "target_code_first_rank": 104302.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 6588.5, "target_country_rank": 1739.5, "source_country_logp": -20.071008682250977, "target_country_logp": -17.821008682250977, "source_code_first_rank": 58309.5, "target_code_first_rank": 30714.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 54.5, "target_country_rank": 271.5, "source_country_logp": -7.130857944488525, "target_country_logp": -9.255857467651367, "source_code_first_rank": 30517.5, "target_code_first_rank": 24877.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 21842.5, "target_country_rank": 8592.0, "source_country_logp": -16.30328369140625, "target_country_logp": -14.32574462890625, "source_code_first_rank": 3470.0, "target_code_first_rank": 3158.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 59.5, "source_country_logp": -2.0776071548461914, "target_country_logp": -10.452607154846191, "source_code_first_rank": 71867.0, "target_code_first_rank": 92364.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 93042.5, "target_country_rank": 36630.5, "source_country_logp": -15.046992301940918, "target_country_logp": -13.109492301940918, "source_code_first_rank": 76357.0, "target_code_first_rank": 11329.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 11.5, "target_country_rank": 233.5, "source_country_logp": -6.252182483673096, "target_country_logp": -13.189682006835938, "source_code_first_rank": 14199.5, "target_code_first_rank": 57489.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 47.0, "target_country_rank": 1133.0, "source_country_logp": -20.751195907592773, "target_country_logp": -28.376195907592773, "source_code_first_rank": 2.0, "target_code_first_rank": 85.5}], "actual_source_country_rank": 47.0, "actual_target_country_rank": 1133.0, "array_path": "results/readout-arrays/eval-26-paraphrase.npz"}
{"case_id": "eval-27", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 3.0, "target_country_rank": 6370.5, "source_country_logp": -3.212402820587158, "target_country_logp": -12.970215797424316, "source_code_first_rank": 76668.0, "target_code_first_rank": 27898.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 8431.0, "target_country_rank": 17352.5, "source_country_logp": -13.138002395629883, "target_country_logp": -14.690736770629883, "source_code_first_rank": 93886.5, "target_code_first_rank": 61855.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 6803.0, "source_country_logp": -0.13450519740581512, "target_country_logp": -29.697004318237305, "source_code_first_rank": 103511.5, "target_code_first_rank": 129848.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 1262.0, "target_country_rank": 23260.5, "source_country_logp": -25.313528060913086, "target_country_logp": -31.078176498413086, "source_code_first_rank": 67404.0, "target_code_first_rank": 22947.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 10.0, "target_country_rank": 978.5, "source_country_logp": -4.3324360847473145, "target_country_logp": -9.676185607910156, "source_code_first_rank": 82097.5, "target_code_first_rank": 36516.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 7029.0, "target_country_rank": 17709.0, "source_country_logp": -12.583466529846191, "target_country_logp": -14.435029029846191, "source_code_first_rank": 21724.5, "target_code_first_rank": 74431.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 2465.0, "source_country_logp": -0.04900013282895088, "target_country_logp": -25.486499786376953, "source_code_first_rank": 96928.5, "target_code_first_rank": 117122.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 2313.0, "target_country_rank": 25740.5, "source_country_logp": -15.571808815002441, "target_country_logp": -19.673370361328125, "source_code_first_rank": 23849.5, "target_code_first_rank": 15716.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 564.5, "source_country_logp": -2.2430481910705566, "target_country_logp": -9.399298667907715, "source_code_first_rank": 75140.5, "target_code_first_rank": 28796.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 8039.5, "target_country_rank": 16338.5, "source_country_logp": -13.116536140441895, "target_country_logp": -14.466145515441895, "source_code_first_rank": 4623.0, "target_code_first_rank": 13019.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 1677.0, "source_country_logp": -0.08695906400680542, "target_country_logp": -25.399459838867188, "source_code_first_rank": 98638.5, "target_code_first_rank": 126641.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 25488.5, "target_country_rank": 119700.5, "source_country_logp": -12.182432174682617, "target_country_logp": -15.869932174682617, "source_code_first_rank": 8750.0, "target_code_first_rank": 9032.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 8.0, "target_country_rank": 4388.5, "source_country_logp": -6.697752475738525, "target_country_logp": -19.604001998901367, "source_code_first_rank": 950.5, "target_code_first_rank": 82434.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 90.0, "target_country_rank": 118797.0, "source_country_logp": -22.62507438659668, "target_country_logp": -43.93757247924805, "source_code_first_rank": 1.0, "target_code_first_rank": 992.5}], "actual_source_country_rank": 90.0, "actual_target_country_rank": 118797.0, "array_path": "results/readout-arrays/eval-27-main.npz"}
{"case_id": "eval-27", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 3.0, "target_country_rank": 6863.0, "source_country_logp": -2.9395523071289062, "target_country_logp": -13.181739807128906, "source_code_first_rank": 66989.5, "target_code_first_rank": 36404.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 9504.5, "target_country_rank": 22709.0, "source_country_logp": -13.4122953414917, "target_country_logp": -15.3810453414917, "source_code_first_rank": 85211.5, "target_code_first_rank": 59204.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 7720.5, "source_country_logp": -0.11328887939453125, "target_country_logp": -31.80078887939453, "source_code_first_rank": 102214.0, "target_code_first_rank": 127330.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 2396.5, "target_country_rank": 39378.5, "source_country_logp": -27.422250747680664, "target_country_logp": -33.7425651550293, "source_code_first_rank": 70477.5, "target_code_first_rank": 29228.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 8.0, "target_country_rank": 1322.0, "source_country_logp": -4.459600925445557, "target_country_logp": -10.334600448608398, "source_code_first_rank": 81522.5, "target_code_first_rank": 37977.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 4548.0, "target_country_rank": 10459.5, "source_country_logp": -11.290119171142578, "target_country_logp": -12.747150421142578, "source_code_first_rank": 14421.0, "target_code_first_rank": 55756.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 3106.0, "source_country_logp": -0.03774368390440941, "target_country_logp": -26.787744522094727, "source_code_first_rank": 102368.0, "target_code_first_rank": 116024.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 1781.5, "target_country_rank": 14525.5, "source_country_logp": -17.560779571533203, "target_country_logp": -21.232654571533203, "source_code_first_rank": 32239.5, "target_code_first_rank": 20846.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 633.5, "source_country_logp": -2.4992012977600098, "target_country_logp": -9.530451774597168, "source_code_first_rank": 82612.0, "target_code_first_rank": 29800.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 7682.0, "target_country_rank": 13996.0, "source_country_logp": -14.106180191040039, "target_country_logp": -15.256570816040039, "source_code_first_rank": 2360.5, "target_code_first_rank": 15001.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 2660.5, "source_country_logp": -0.0437738336622715, "target_country_logp": -25.606273651123047, "source_code_first_rank": 94118.5, "target_code_first_rank": 117400.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 32027.0, "target_country_rank": 127545.5, "source_country_logp": -12.955095291137695, "target_country_logp": -16.611345291137695, "source_code_first_rank": 11045.5, "target_code_first_rank": 14496.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 11.5, "target_country_rank": 7880.0, "source_country_logp": -7.4401726722717285, "target_country_logp": -20.87767219543457, "source_code_first_rank": 1930.0, "target_code_first_rank": 90031.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 28.0, "target_country_rank": 60709.5, "source_country_logp": -20.750749588012695, "target_country_logp": -42.38356399536133, "source_code_first_rank": 2.0, "target_code_first_rank": 3987.0}], "actual_source_country_rank": 28.0, "actual_target_country_rank": 60709.5, "array_path": "results/readout-arrays/eval-27-paraphrase.npz"}
{"case_id": "eval-28", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 7.5, "target_country_rank": 18567.5, "source_country_logp": -3.9688711166381836, "target_country_logp": -14.632933616638184, "source_code_first_rank": 32154.0, "target_code_first_rank": 32754.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 8886.0, "target_country_rank": 16917.0, "source_country_logp": -13.392498970031738, "target_country_logp": -14.804608345031738, "source_code_first_rank": 92778.5, "target_code_first_rank": 60860.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 31081.0, "source_country_logp": -0.726797878742218, "target_country_logp": -27.820547103881836, "source_code_first_rank": 93734.5, "target_code_first_rank": 134371.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 1349.0, "target_country_rank": 24860.0, "source_country_logp": -24.602298736572266, "target_country_logp": -30.213626861572266, "source_code_first_rank": 64572.0, "target_code_first_rank": 21998.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 10.0, "target_country_rank": 1288.5, "source_country_logp": -4.201889514923096, "target_country_logp": -10.389389038085938, "source_code_first_rank": 19518.5, "target_code_first_rank": 57412.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 8232.5, "target_country_rank": 17998.0, "source_country_logp": -12.954362869262695, "target_country_logp": -14.548112869262695, "source_code_first_rank": 21905.5, "target_code_first_rank": 79297.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 5787.5, "source_country_logp": -0.4572460353374481, "target_country_logp": -24.394746780395508, "source_code_first_rank": 88341.5, "target_code_first_rank": 126898.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 2325.5, "target_country_rank": 28391.0, "source_country_logp": -15.345059394836426, "target_country_logp": -19.649747848510742, "source_code_first_rank": 19698.5, "target_code_first_rank": 13196.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 9.0, "target_country_rank": 802.5, "source_country_logp": -5.616682529449463, "target_country_logp": -11.710433006286621, "source_code_first_rank": 14980.5, "target_code_first_rank": 21787.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 9468.5, "target_country_rank": 12735.0, "source_country_logp": -13.373239517211914, "target_country_logp": -13.917184829711914, "source_code_first_rank": 3892.5, "target_code_first_rank": 8830.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 3327.0, "source_country_logp": -0.38034313917160034, "target_country_logp": -24.380342483520508, "source_code_first_rank": 62926.0, "target_code_first_rank": 128817.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 25235.0, "target_country_rank": 113670.0, "source_country_logp": -12.261155128479004, "target_country_logp": -15.729905128479004, "source_code_first_rank": 6699.0, "target_code_first_rank": 5466.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 10.0, "target_country_rank": 6421.0, "source_country_logp": -7.209638595581055, "target_country_logp": -19.490888595581055, "source_code_first_rank": 1991.5, "target_code_first_rank": 78093.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 62.0, "target_country_rank": 122615.5, "source_country_logp": -21.250045776367188, "target_country_logp": -44.03129577636719, "source_code_first_rank": 1.0, "target_code_first_rank": 962.0}], "actual_source_country_rank": 62.0, "actual_target_country_rank": 122615.5, "array_path": "results/readout-arrays/eval-28-main.npz"}
{"case_id": "eval-28", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 6.5, "target_country_rank": 20690.5, "source_country_logp": -4.170341968536377, "target_country_logp": -14.795341491699219, "source_code_first_rank": 34776.0, "target_code_first_rank": 33916.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 9068.5, "target_country_rank": 21904.0, "source_country_logp": -13.365241050720215, "target_country_logp": -15.337897300720215, "source_code_first_rank": 84206.5, "target_code_first_rank": 59537.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 18496.0, "source_country_logp": -0.7046031355857849, "target_country_logp": -28.57960319519043, "source_code_first_rank": 87258.0, "target_code_first_rank": 130225.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 1933.0, "target_country_rank": 34391.5, "source_country_logp": -27.249635696411133, "target_country_logp": -33.6090087890625, "source_code_first_rank": 67066.0, "target_code_first_rank": 28957.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 8.0, "target_country_rank": 1090.5, "source_country_logp": -4.086954593658447, "target_country_logp": -9.977580070495605, "source_code_first_rank": 18995.5, "target_code_first_rank": 58833.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 4308.0, "target_country_rank": 10299.5, "source_country_logp": -11.24458122253418, "target_country_logp": -12.76313591003418, "source_code_first_rank": 11901.0, "target_code_first_rank": 51336.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 3034.5, "source_country_logp": -0.5642406344413757, "target_country_logp": -23.189241409301758, "source_code_first_rank": 89843.5, "target_code_first_rank": 125153.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 1461.5, "target_country_rank": 13705.0, "source_country_logp": -17.739219665527344, "target_country_logp": -21.700157165527344, "source_code_first_rank": 29919.0, "target_code_first_rank": 18607.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 7.5, "target_country_rank": 507.5, "source_country_logp": -4.460517406463623, "target_country_logp": -9.726142883300781, "source_code_first_rank": 16968.5, "target_code_first_rank": 24253.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 7332.0, "target_country_rank": 14914.0, "source_country_logp": -13.957498550415039, "target_country_logp": -15.359842300415039, "source_code_first_rank": 2849.5, "target_code_first_rank": 14747.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 2586.5, "source_country_logp": -0.28719696402549744, "target_country_logp": -23.72469711303711, "source_code_first_rank": 64073.5, "target_code_first_rank": 126706.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 32827.0, "target_country_rank": 126948.0, "source_country_logp": -12.941963195800781, "target_country_logp": -16.50446319580078, "source_code_first_rank": 11981.5, "target_code_first_rank": 15280.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 13.0, "target_country_rank": 7893.5, "source_country_logp": -6.929172039031982, "target_country_logp": -19.71042251586914, "source_code_first_rank": 2458.0, "target_code_first_rank": 83506.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 19.5, "target_country_rank": 70390.0, "source_country_logp": -21.75005531311035, "target_country_logp": -44.859432220458984, "source_code_first_rank": 2.0, "target_code_first_rank": 4747.5}], "actual_source_country_rank": 19.5, "actual_target_country_rank": 70390.0, "array_path": "results/readout-arrays/eval-28-paraphrase.npz"}
{"case_id": "eval-29", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 5.0, "source_country_logp": -0.46228155493736267, "target_country_logp": -4.087281703948975, "source_code_first_rank": 24847.5, "target_code_first_rank": 118601.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 15425.0, "target_country_rank": 1876.5, "source_country_logp": -14.481792449951172, "target_country_logp": -10.568828582763672, "source_code_first_rank": 62764.0, "target_code_first_rank": 62274.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.5, "target_country_rank": 32.0, "source_country_logp": -2.4868266582489014, "target_country_logp": -14.73682689666748, "source_code_first_rank": 108805.0, "target_code_first_rank": 136711.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 21167.5, "target_country_rank": 53442.0, "source_country_logp": -30.637746810913086, "target_country_logp": -33.13774871826172, "source_code_first_rank": 22659.0, "target_code_first_rank": 60593.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 13.0, "source_country_logp": -0.935209333896637, "target_country_logp": -5.435209274291992, "source_code_first_rank": 23118.0, "target_code_first_rank": 60983.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 14143.5, "target_country_rank": 4600.5, "source_country_logp": -13.843770980834961, "target_country_logp": -11.714864730834961, "source_code_first_rank": 68941.5, "target_code_first_rank": 4490.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 16.5, "source_country_logp": -0.29400965571403503, "target_country_logp": -13.2940092086792, "source_code_first_rank": 74779.5, "target_code_first_rank": 77072.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 21331.5, "target_country_rank": 51946.0, "source_country_logp": -19.479942321777344, "target_country_logp": -21.573692321777344, "source_code_first_rank": 15915.5, "target_code_first_rank": 19761.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 34.0, "source_country_logp": -0.75487220287323, "target_country_logp": -6.8486223220825195, "source_code_first_rank": 13481.0, "target_code_first_rank": 49171.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 11805.5, "target_country_rank": 12576.5, "source_country_logp": -13.809913635253906, "target_country_logp": -13.939308166503906, "source_code_first_rank": 15903.0, "target_code_first_rank": 1422.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 13.0, "source_country_logp": -0.4920794367790222, "target_country_logp": -14.367079734802246, "source_code_first_rank": 52293.0, "target_code_first_rank": 101388.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 111971.0, "target_country_rank": 108117.5, "source_country_logp": -15.472204208374023, "target_country_logp": -15.284704208374023, "source_code_first_rank": 8752.0, "target_code_first_rank": 9040.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 9.5, "target_country_rank": 63.5, "source_country_logp": -6.588970184326172, "target_country_logp": -11.213970184326172, "source_code_first_rank": 19348.0, "target_code_first_rank": 21990.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 159.5, "target_country_rank": 1791.0, "source_country_logp": -15.308667182922363, "target_country_logp": -21.80866813659668, "source_code_first_rank": 3.0, "target_code_first_rank": 4.0}], "actual_source_country_rank": 159.5, "actual_target_country_rank": 1791.0, "array_path": "results/readout-arrays/eval-29-main.npz"}
{"case_id": "eval-29", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 7.0, "source_country_logp": -0.3665528893470764, "target_country_logp": -4.804052829742432, "source_code_first_rank": 26825.0, "target_code_first_rank": 116987.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 19426.0, "target_country_rank": 2901.0, "source_country_logp": -15.020711898803711, "target_country_logp": -11.239461898803711, "source_code_first_rank": 56480.0, "target_code_first_rank": 74822.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 2.5, "target_country_rank": 31.0, "source_country_logp": -2.293590545654297, "target_country_logp": -15.668590545654297, "source_code_first_rank": 104616.5, "target_code_first_rank": 136028.5}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 29963.5, "target_country_rank": 76827.0, "source_country_logp": -32.715999603271484, "target_country_logp": -35.653499603271484, "source_code_first_rank": 25832.5, "target_code_first_rank": 63811.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 11.5, "source_country_logp": -1.3221004009246826, "target_country_logp": -6.1346001625061035, "source_code_first_rank": 23961.0, "target_code_first_rank": 56422.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 8901.5, "target_country_rank": 3326.5, "source_country_logp": -12.45111083984375, "target_country_logp": -10.79876708984375, "source_code_first_rank": 49178.5, "target_code_first_rank": 11882.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 20.0, "source_country_logp": -0.31379076838493347, "target_country_logp": -13.938790321350098, "source_code_first_rank": 74575.0, "target_code_first_rank": 79315.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 12284.5, "target_country_rank": 30173.0, "source_country_logp": -21.966148376464844, "target_country_logp": -23.903648376464844, "source_code_first_rank": 18591.5, "target_code_first_rank": 34645.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 28.0, "source_country_logp": -0.792609453201294, "target_country_logp": -7.167609214782715, "source_code_first_rank": 15148.0, "target_code_first_rank": 40321.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 11859.0, "target_country_rank": 10966.5, "source_country_logp": -15.13340950012207, "target_country_logp": -14.96739387512207, "source_code_first_rank": 15152.0, "target_code_first_rank": 3572.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 14.0, "source_country_logp": -0.5921605825424194, "target_country_logp": -14.71716022491455, "source_code_first_rank": 55103.0, "target_code_first_rank": 92267.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 122934.0, "target_country_rank": 106842.5, "source_country_logp": -16.380401611328125, "target_country_logp": -15.567901611328125, "source_code_first_rank": 14535.5, "target_code_first_rank": 17312.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 14.5, "target_country_rank": 83.5, "source_country_logp": -6.6953654289245605, "target_country_logp": -11.632865905761719, "source_code_first_rank": 23479.5, "target_code_first_rank": 28795.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 116.5, "target_country_rank": 747.0, "source_country_logp": -24.37500762939453, "target_country_logp": -29.00000762939453, "source_code_first_rank": 30.0, "target_code_first_rank": 46.0}], "actual_source_country_rank": 116.5, "actual_target_country_rank": 747.0, "array_path": "results/readout-arrays/eval-29-paraphrase.npz"}
{"case_id": "eval-30", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 2.0, "target_country_rank": 999.5, "source_country_logp": -1.6115120649337769, "target_country_logp": -10.799012184143066, "source_code_first_rank": 27432.5, "target_code_first_rank": 128752.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 14927.0, "target_country_rank": 1836.0, "source_country_logp": -14.380866050720215, "target_country_logp": -10.544928550720215, "source_code_first_rank": 60433.0, "target_code_first_rank": 62831.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 6.5, "target_country_rank": 991.5, "source_country_logp": -5.183146953582764, "target_country_logp": -18.683147430419922, "source_code_first_rank": 96173.5, "target_code_first_rank": 137485.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 16455.5, "target_country_rank": 47000.0, "source_country_logp": -30.543773651123047, "target_country_logp": -33.26252365112305, "source_code_first_rank": 21690.0, "target_code_first_rank": 58436.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 4.0, "target_country_rank": 853.0, "source_country_logp": -2.9333372116088867, "target_country_logp": -10.464587211608887, "source_code_first_rank": 34664.5, "target_code_first_rank": 71836.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 14542.0, "target_country_rank": 4764.5, "source_country_logp": -14.05213737487793, "target_country_logp": -11.89979362487793, "source_code_first_rank": 71903.0, "target_code_first_rank": 4104.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 305.0, "source_country_logp": -0.4409445524215698, "target_country_logp": -16.81594467163086, "source_code_first_rank": 76918.0, "target_code_first_rank": 120623.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 20265.0, "target_country_rank": 55193.0, "source_country_logp": -19.443050384521484, "target_country_logp": -21.818050384521484, "source_code_first_rank": 14547.0, "target_code_first_rank": 17203.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 1355.0, "source_country_logp": -0.8364315032958984, "target_country_logp": -11.086431503295898, "source_code_first_rank": 29767.0, "target_code_first_rank": 81161.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 9544.0, "target_country_rank": 12143.5, "source_country_logp": -13.24952507019043, "target_country_logp": -13.66749382019043, "source_code_first_rank": 10336.0, "target_code_first_rank": 1569.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 64.0, "source_country_logp": -0.1941847950220108, "target_country_logp": -15.444185256958008, "source_code_first_rank": 85415.0, "target_code_first_rank": 137118.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 107270.5, "target_country_rank": 103086.5, "source_country_logp": -15.360304832458496, "target_country_logp": -15.172804832458496, "source_code_first_rank": 5826.5, "target_code_first_rank": 6715.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 17.5, "target_country_rank": 1674.0, "source_country_logp": -9.65972900390625, "target_country_logp": -19.59722900390625, "source_code_first_rank": 18216.5, "target_code_first_rank": 30767.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 291.0, "target_country_rank": 1511.0, "source_country_logp": -14.598331451416016, "target_country_logp": -19.785831451416016, "source_code_first_rank": 3.0, "target_code_first_rank": 4.0}], "actual_source_country_rank": 291.0, "actual_target_country_rank": 1511.0, "array_path": "results/readout-arrays/eval-30-main.npz"}
{"case_id": "eval-30", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 1202.5, "source_country_logp": -1.6272754669189453, "target_country_logp": -11.127275466918945, "source_code_first_rank": 25565.0, "target_code_first_rank": 122389.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 21627.5, "target_country_rank": 3722.0, "source_country_logp": -15.282386779785156, "target_country_logp": -11.669105529785156, "source_code_first_rank": 53765.0, "target_code_first_rank": 78756.0}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 5.0, "target_country_rank": 725.0, "source_country_logp": -2.337836265563965, "target_country_logp": -16.21283721923828, "source_code_first_rank": 74325.5, "target_code_first_rank": 132325.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 32679.5, "target_country_rank": 81352.5, "source_country_logp": -33.595191955566406, "target_country_logp": -36.626441955566406, "source_code_first_rank": 25505.0, "target_code_first_rank": 62253.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 4.0, "target_country_rank": 934.0, "source_country_logp": -3.2679052352905273, "target_country_logp": -10.799155235290527, "source_code_first_rank": 34712.0, "target_code_first_rank": 65611.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 8844.0, "target_country_rank": 3034.5, "source_country_logp": -12.450504302978516, "target_country_logp": -10.677066802978516, "source_code_first_rank": 49165.0, "target_code_first_rank": 11496.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 241.0, "source_country_logp": -0.6981711387634277, "target_country_logp": -16.948171615600586, "source_code_first_rank": 65037.5, "target_code_first_rank": 119936.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 11873.0, "target_country_rank": 30608.0, "source_country_logp": -21.34428596496582, "target_country_logp": -23.34428596496582, "source_code_first_rank": 19898.5, "target_code_first_rank": 36567.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 1142.0, "source_country_logp": -0.4851683974266052, "target_country_logp": -11.21954345703125, "source_code_first_rank": 23574.0, "target_code_first_rank": 73707.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 12193.5, "target_country_rank": 10654.5, "source_country_logp": -14.933964729309082, "target_country_logp": -14.657597541809082, "source_code_first_rank": 14999.5, "target_code_first_rank": 3311.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 52.5, "source_country_logp": -0.2971571385860443, "target_country_logp": -14.922157287597656, "source_code_first_rank": 73339.5, "target_code_first_rank": 132517.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 126581.5, "target_country_rank": 107633.0, "source_country_logp": -16.482746124267578, "target_country_logp": -15.482747077941895, "source_code_first_rank": 13678.5, "target_code_first_rank": 15435.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 31.0, "target_country_rank": 2322.0, "source_country_logp": -10.370179176330566, "target_country_logp": -20.15142822265625, "source_code_first_rank": 19924.0, "target_code_first_rank": 30542.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 589.5, "target_country_rank": 742.5, "source_country_logp": -31.3125, "target_country_logp": -31.875, "source_code_first_rank": 22.0, "target_code_first_rank": 10.5}], "actual_source_country_rank": 589.5, "actual_target_country_rank": 742.5, "array_path": "results/readout-arrays/eval-30-paraphrase.npz"}
{"case_id": "eval-31", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 85.0, "source_country_logp": -1.1345903873443604, "target_country_logp": -7.634590148925781, "source_code_first_rank": 112415.0, "target_code_first_rank": 17684.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 1553.0, "target_country_rank": 12993.0, "source_country_logp": -10.23072624206543, "target_country_logp": -14.02174186706543, "source_code_first_rank": 65501.0, "target_code_first_rank": 79652.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 4.0, "target_country_rank": 52.5, "source_country_logp": -2.4470877647399902, "target_country_logp": -20.94708824157715, "source_code_first_rank": 121564.0, "target_code_first_rank": 6664.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 43581.0, "target_country_rank": 18911.0, "source_country_logp": -32.596412658691406, "target_country_logp": -30.440162658691406, "source_code_first_rank": 61740.0, "target_code_first_rank": 77969.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 7.0, "target_country_rank": 92.0, "source_country_logp": -4.012600898742676, "target_country_logp": -7.543850898742676, "source_code_first_rank": 61811.5, "target_code_first_rank": 8115.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 3975.5, "target_country_rank": 10376.0, "source_country_logp": -11.463237762451172, "target_country_logp": -13.211284637451172, "source_code_first_rank": 4834.0, "target_code_first_rank": 32407.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 34.5, "source_country_logp": -1.3524322509765625, "target_country_logp": -15.977432250976562, "source_code_first_rank": 74369.0, "target_code_first_rank": 8864.0}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 47205.5, "target_country_rank": 14596.5, "source_country_logp": -20.636444091796875, "target_country_logp": -18.136444091796875, "source_code_first_rank": 20164.0, "target_code_first_rank": 6820.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 62.5, "target_country_rank": 174.5, "source_country_logp": -6.906759738922119, "target_country_logp": -8.344260215759277, "source_code_first_rank": 42837.5, "target_code_first_rank": 22181.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 8728.0, "target_country_rank": 27868.5, "source_country_logp": -13.28076457977295, "target_country_logp": -15.71045207977295, "source_code_first_rank": 1553.5, "target_code_first_rank": 5820.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 49.0, "source_country_logp": -0.5409404635429382, "target_country_logp": -16.29094123840332, "source_code_first_rank": 110175.0, "target_code_first_rank": 12427.0}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 101413.0, "target_country_rank": 124906.5, "source_country_logp": -14.98444938659668, "target_country_logp": -16.23444938659668, "source_code_first_rank": 9000.5, "target_code_first_rank": 1487.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 9.0, "target_country_rank": 332.5, "source_country_logp": -6.384927749633789, "target_country_logp": -13.822427749633789, "source_code_first_rank": 8634.0, "target_code_first_rank": 23383.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 123.5, "target_country_rank": 83773.0, "source_country_logp": -21.875486373901367, "target_country_logp": -40.48486328125, "source_code_first_rank": 1.0, "target_code_first_rank": 25.0}], "actual_source_country_rank": 123.5, "actual_target_country_rank": 83773.0, "array_path": "results/readout-arrays/eval-31-main.npz"}
{"case_id": "eval-31", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 1.0, "target_country_rank": 161.5, "source_country_logp": -1.1333991289138794, "target_country_logp": -8.13339900970459, "source_code_first_rank": 104756.0, "target_code_first_rank": 20721.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 2696.5, "target_country_rank": 11846.0, "source_country_logp": -11.067866325378418, "target_country_logp": -13.845210075378418, "source_code_first_rank": 77685.0, "target_code_first_rank": 81846.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 4.0, "target_country_rank": 46.5, "source_country_logp": -2.6856584548950195, "target_country_logp": -20.810659408569336, "source_code_first_rank": 116430.5, "target_code_first_rank": 6410.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 68375.0, "target_country_rank": 30166.0, "source_country_logp": -35.65752029418945, "target_country_logp": -33.06377029418945, "source_code_first_rank": 65172.0, "target_code_first_rank": 69495.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 10.0, "target_country_rank": 149.5, "source_country_logp": -4.821724891662598, "target_country_logp": -8.446724891662598, "source_code_first_rank": 56653.0, "target_code_first_rank": 10683.5}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 3000.0, "target_country_rank": 3852.5, "source_country_logp": -10.633716583251953, "target_country_logp": -11.008716583251953, "source_code_first_rank": 11313.5, "target_code_first_rank": 29900.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 52.0, "source_country_logp": -1.8693451881408691, "target_country_logp": -16.24434471130371, "source_code_first_rank": 80243.0, "target_code_first_rank": 15851.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 29345.0, "target_country_rank": 7497.0, "source_country_logp": -23.581289291381836, "target_country_logp": -20.800039291381836, "source_code_first_rank": 35853.5, "target_code_first_rank": 9541.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 72.5, "target_country_rank": 252.0, "source_country_logp": -7.627348899841309, "target_country_logp": -9.158598899841309, "source_code_first_rank": 37910.5, "target_code_first_rank": 21956.5}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 10120.5, "target_country_rank": 24843.0, "source_country_logp": -14.704391479492188, "target_country_logp": -16.693649291992188, "source_code_first_rank": 3339.0, "target_code_first_rank": 17938.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 1.0, "target_country_rank": 46.0, "source_country_logp": -0.5724337697029114, "target_country_logp": -15.697433471679688, "source_code_first_rank": 105295.0, "target_code_first_rank": 20878.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 101509.5, "target_country_rank": 116897.5, "source_country_logp": -15.319143295288086, "target_country_logp": -16.006643295288086, "source_code_first_rank": 15483.5, "target_code_first_rank": 8925.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 15.0, "target_country_rank": 332.5, "source_country_logp": -6.51940393447876, "target_country_logp": -13.706904411315918, "source_code_first_rank": 7470.0, "target_code_first_rank": 33543.5}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 63.0, "target_country_rank": 27204.5, "source_country_logp": -18.89858627319336, "target_country_logp": -34.97671127319336, "source_code_first_rank": 2.0, "target_code_first_rank": 6.0}], "actual_source_country_rank": 63.0, "actual_target_country_rank": 27204.5, "array_path": "results/readout-arrays/eval-31-paraphrase.npz"}
{"case_id": "eval-32", "template_id": "main", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 11.5, "target_country_rank": 1419.0, "source_country_logp": -3.925705909729004, "target_country_logp": -10.410080909729004, "source_code_first_rank": 132497.5, "target_code_first_rank": 33749.5}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 1643.0, "target_country_rank": 14059.0, "source_country_logp": -10.323983192443848, "target_country_logp": -14.211678504943848, "source_code_first_rank": 62297.5, "target_code_first_rank": 80372.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 602.0, "source_country_logp": -2.33325457572937, "target_country_logp": -15.77075481414795, "source_code_first_rank": 121786.0, "target_code_first_rank": 75498.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 44313.5, "target_country_rank": 19542.0, "source_country_logp": -32.981868743896484, "target_country_logp": -30.825620651245117, "source_code_first_rank": 58669.5, "target_code_first_rank": 79942.5}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 61.0, "target_country_rank": 657.5, "source_country_logp": -6.134786128997803, "target_country_logp": -9.306660652160645, "source_code_first_rank": 106736.0, "target_code_first_rank": 17589.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 4309.0, "target_country_rank": 12754.0, "source_country_logp": -11.654180526733398, "target_country_logp": -13.681524276733398, "source_code_first_rank": 3560.5, "target_code_first_rank": 35693.5}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 320.5, "source_country_logp": -2.0673351287841797, "target_country_logp": -14.31733512878418, "source_code_first_rank": 102793.0, "target_code_first_rank": 50241.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 43448.5, "target_country_rank": 13241.5, "source_country_logp": -22.791400909423828, "target_country_logp": -20.135150909423828, "source_code_first_rank": 15846.0, "target_code_first_rank": 6460.5}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 77.5, "target_country_rank": 1677.5, "source_country_logp": -7.3333659172058105, "target_country_logp": -11.286491394042969, "source_code_first_rank": 96809.5, "target_code_first_rank": 19585.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 11243.5, "target_country_rank": 27119.5, "source_country_logp": -13.574745178222656, "target_country_logp": -15.471229553222656, "source_code_first_rank": 1272.5, "target_code_first_rank": 4202.5}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 405.5, "source_country_logp": -0.9891131520271301, "target_country_logp": -18.614112854003906, "source_code_first_rank": 126525.0, "target_code_first_rank": 31590.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 90379.0, "target_country_rank": 116607.5, "source_country_logp": -14.812420845031738, "target_country_logp": -15.937420845031738, "source_code_first_rank": 6816.0, "target_code_first_rank": 715.0}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 21.0, "target_country_rank": 1388.5, "source_country_logp": -9.163018226623535, "target_country_logp": -17.22551727294922, "source_code_first_rank": 14944.0, "target_code_first_rank": 38546.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 227.5, "target_country_rank": 138596.5, "source_country_logp": -21.750930786132812, "target_country_logp": -42.25093078613281, "source_code_first_rank": 1.0, "target_code_first_rank": 43.5}], "actual_source_country_rank": 227.5, "actual_target_country_rank": 138596.5, "array_path": "results/readout-arrays/eval-32-main.npz"}
{"case_id": "eval-32", "template_id": "paraphrase", "readouts": [{"layer": 9, "method": "ordinary", "position": "city", "source_country_rank": 11.0, "target_country_rank": 1968.0, "source_country_logp": -4.14136266708374, "target_country_logp": -11.094488143920898, "source_code_first_rank": 128544.0, "target_code_first_rank": 35890.0}, {"layer": 9, "method": "ordinary", "position": "final", "source_country_rank": 2880.5, "target_country_rank": 12743.5, "source_country_logp": -11.185334205627441, "target_country_logp": -13.980256080627441, "source_code_first_rank": 79016.0, "target_code_first_rank": 90838.5}, {"layer": 9, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 465.5, "source_country_logp": -2.223323345184326, "target_country_logp": -15.473322868347168, "source_code_first_rank": 123693.5, "target_code_first_rank": 74956.0}, {"layer": 9, "method": "j", "position": "final", "source_country_rank": 70870.5, "target_country_rank": 29401.5, "source_country_logp": -36.718177795410156, "target_country_logp": -33.874427795410156, "source_code_first_rank": 61327.5, "target_code_first_rank": 67654.0}, {"layer": 13, "method": "ordinary", "position": "city", "source_country_rank": 34.0, "target_country_rank": 472.5, "source_country_logp": -5.786881923675537, "target_country_logp": -9.068131446838379, "source_code_first_rank": 107679.0, "target_code_first_rank": 19922.0}, {"layer": 13, "method": "ordinary", "position": "final", "source_country_rank": 3185.0, "target_country_rank": 3383.0, "source_country_logp": -10.748482704162598, "target_country_logp": -10.826607704162598, "source_code_first_rank": 11842.0, "target_code_first_rank": 34321.0}, {"layer": 13, "method": "j", "position": "city", "source_country_rank": 3.0, "target_country_rank": 224.0, "source_country_logp": -2.151021718978882, "target_country_logp": -13.901021957397461, "source_code_first_rank": 108714.5, "target_code_first_rank": 55412.5}, {"layer": 13, "method": "j", "position": "final", "source_country_rank": 25622.0, "target_country_rank": 5915.0, "source_country_logp": -24.239879608154297, "target_country_logp": -21.271129608154297, "source_code_first_rank": 31528.5, "target_code_first_rank": 10010.0}, {"layer": 17, "method": "ordinary", "position": "city", "source_country_rank": 40.0, "target_country_rank": 1179.0, "source_country_logp": -6.8118743896484375, "target_country_logp": -11.030624389648438, "source_code_first_rank": 86639.0, "target_code_first_rank": 18879.0}, {"layer": 17, "method": "ordinary", "position": "final", "source_country_rank": 11218.0, "target_country_rank": 26498.0, "source_country_logp": -14.82801628112793, "target_country_logp": -16.79676628112793, "source_code_first_rank": 2945.5, "target_code_first_rank": 18053.0}, {"layer": 17, "method": "j", "position": "city", "source_country_rank": 2.0, "target_country_rank": 262.0, "source_country_logp": -1.0962073802947998, "target_country_logp": -19.221206665039062, "source_code_first_rank": 121801.5, "target_code_first_rank": 34685.5}, {"layer": 17, "method": "j", "position": "final", "source_country_rank": 101088.5, "target_country_rank": 117848.0, "source_country_logp": -15.315260887145996, "target_country_logp": -16.065261840820312, "source_code_first_rank": 13962.5, "target_code_first_rank": 7153.5}, {"layer": 27, "method": "ordinary", "position": "city", "source_country_rank": 27.0, "target_country_rank": 1413.5, "source_country_logp": -9.691207885742188, "target_country_logp": -17.628707885742188, "source_code_first_rank": 16855.5, "target_code_first_rank": 64094.0}, {"layer": 27, "method": "ordinary", "position": "final", "source_country_rank": 126.0, "target_country_rank": 60375.5, "source_country_logp": -23.250038146972656, "target_country_logp": -41.429725646972656, "source_code_first_rank": 2.0, "target_code_first_rank": 9.5}], "actual_source_country_rank": 126.0, "actual_target_country_rank": 60375.5, "array_path": "results/readout-arrays/eval-32-paraphrase.npz"}